From d1935bc6adbf9c2bc3b7fa9dab99fad0d9cf05e3 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 11 Nov 2020 10:50:39 +0100 Subject: [PATCH 001/324] Added taxonomy to database's schema --- .../controllers/v1/TheHiveQueryExecutor.scala | 2 +- .../org/thp/thehive/models/Organisation.scala | 3 ++ .../models/TheHiveSchemaDefinition.scala | 32 ++++++++++++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index 2ee8a82cf2..bbc3b86b81 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -2,7 +2,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.controllers.{FObject, FieldsParser} -import org.thp.scalligraph.models.{Database, UMapping} +import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ case class OutputParam(from: Long, to: Long, extraData: Set[String]) diff --git a/thehive/app/org/thp/thehive/models/Organisation.scala b/thehive/app/org/thp/thehive/models/Organisation.scala index 41ca8dd5c2..b7ad03b1fc 100644 --- a/thehive/app/org/thp/thehive/models/Organisation.scala +++ b/thehive/app/org/thp/thehive/models/Organisation.scala @@ -20,6 +20,9 @@ case class OrganisationShare() @BuildEdgeEntity[Organisation, Organisation] case class OrganisationOrganisation() +@BuildEdgeEntity[Organisation, Taxonomy] +case class OrganisationTaxonomy() + case class RichOrganisation(organisation: Organisation with Entity, links: Seq[Organisation with Entity]) { def name: String = organisation.name def description: String = organisation.description diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 62683434d6..38953a7a51 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -9,9 +9,11 @@ import org.janusgraph.graphdb.types.TypeDefinitionCategory import org.reflections.Reflections import org.reflections.scanners.SubTypesScanner import org.reflections.util.ConfigurationBuilder +import org.thp.scalligraph.{EntityId, RichSeq} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.janus.JanusDatabase import org.thp.scalligraph.models._ +import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import play.api.Logger @@ -74,8 +76,36 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { Success(()) } .addProperty[Option[Boolean]]("Observable", "ignoreSimilarity") + // Taxonomies + .addVertexModel[String]("Taxonomy", Seq("namespace")) + .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => + db.tryTransaction { g => + db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => + val taxoVertex = g.addVertex("Taxonomy") + taxoVertex.property("namespace", "Custom") + o.addEdge("OrganisationTaxonomy", taxoVertex) + Success(()) + } + }.map(_ => ()) + } + .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => + db.tryTransaction { implicit g => + db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => + val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "Custom").head + Traversal.V(EntityId(o.id())).unionFlat( + _.out("OrganisationShare").out("ShareCase").out("CaseTag"), + _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), + _.in("AlertOrganisation").out("AlertTag"), + _.in("CaseTemplateOrganisation").out("CaseTemplateTag") + ).toSeq.foreach(tag => + customTaxo.addEdge("TaxonomyTag", tag) + ) + Success(()) + } + }.map(_ => ()) + } - val reflectionClasses = new Reflections( + val reflectionClasses = new Reflections( new ConfigurationBuilder() .forPackages("org.thp.thehive.models") .addClassLoader(getClass.getClassLoader) From 5f9d4b991f664d02d4c254c40295d04edd5a06a4 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 12 Nov 2020 11:13:26 +0100 Subject: [PATCH 002/324] WIP adding taxonomy routes & mapping --- .../org/thp/thehive/dto/v0/Taxonomy.scala | 43 +++++++++++ .../org/thp/thehive/dto/v1/Taxonomy.scala | 43 +++++++++++ .../thp/thehive/controllers/v0/CaseCtrl.scala | 23 ++++-- .../thehive/controllers/v0/Conversion.scala | 21 ++++++ .../thehive/controllers/v0/TaxonomyCtrl.scala | 74 +++++++++++++++++++ .../thehive/controllers/v1/Conversion.scala | 22 ++++++ .../org/thp/thehive/models/Permissions.scala | 2 + .../app/org/thp/thehive/models/Taxonomy.scala | 36 +++++++++ .../thehive/services/OrganisationSrv.scala | 2 + .../thp/thehive/services/TaxonomySrv.scala | 66 +++++++++++++++++ 10 files changed, 324 insertions(+), 8 deletions(-) create mode 100644 dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala create mode 100644 dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala create mode 100644 thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala create mode 100644 thehive/app/org/thp/thehive/models/Taxonomy.scala create mode 100644 thehive/app/org/thp/thehive/services/TaxonomySrv.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala new file mode 100644 index 0000000000..a5af5ff61e --- /dev/null +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala @@ -0,0 +1,43 @@ +package org.thp.thehive.dto.v0 + +import java.util.Date + +import play.api.libs.json.{Json, OFormat, OWrites} + +case class InputTaxonomy ( + namespace: String, + description: String, + version: Int, + predicates: Seq[InputPredicate], + values: Option[Seq[InputValue]] +) + +case class InputPredicate(value: String, expanded: String) + +case class InputValue(predicate: String, entry: Seq[InputPredicate]) + +object InputTaxonomy { + implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] +} + +case class OutputTaxonomy( + _id: String, + _type: String, + _createdBy: String, + _updatedBy: Option[String] = None, + _createdAt: Date, + _updatedAt: Option[Date] = None, + namespace: String, + description: String, + version: Int, + predicates: Seq[OutputPredicate], + values: Option[Seq[OutputValue]] +) + +case class OutputPredicate(value: String, expanded: String) + +case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) + +object OutputTaxonomy { + implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] +} \ No newline at end of file diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala new file mode 100644 index 0000000000..1c6a1b2bc9 --- /dev/null +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -0,0 +1,43 @@ +package org.thp.thehive.dto.v1 + +import java.util.Date + +import play.api.libs.json.{Json, OFormat, OWrites} + +case class InputTaxonomy ( + namespace: String, + description: String, + version: Int, + predicates: Seq[InputPredicate], + values: Option[Seq[InputValue]] +) + +case class InputPredicate(value: String, expanded: String) + +case class InputValue(predicate: String, entry: Seq[InputPredicate]) + +object InputTaxonomy { + implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] +} + +case class OutputTaxonomy( + _id: String, + _type: String, + _createdBy: String, + _updatedBy: Option[String] = None, + _createdAt: Date, + _updatedAt: Option[Date] = None, + namespace: String, + description: String, + version: Int, + predicates: Seq[OutputPredicate], + values: Option[Seq[OutputValue]] +) + +case class OutputPredicate(value: String, expanded: String) + +case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) + +object OutputTaxonomy { + implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] +} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index aa94cfbd82..7c6b8db99e 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -195,13 +195,17 @@ class PublicCase @Inject() ( with CaseRenderer { override val entityName: String = "case" override val initialQuery: Query = - Query.init[Traversal.V[Case]]("listCase", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).cases) - override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Case]]( - "getCase", - FieldsParser[EntityIdOrName], - (idOrName, graph, authContext) => caseSrv.get(idOrName)(graph).visible(authContext) - ) - override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Case], IteratorOutput]( + Query.init[Traversal.V[Case]]("listCase", (graph, authContext) => + organisationSrv.get(authContext.organisation)(graph).cases + ) + override val getQuery: ParamQuery[EntityIdOrName] = + Query.initWithParam[EntityIdOrName, Traversal.V[Case]]( + "getCase", + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => caseSrv.get(idOrName)(graph).visible(authContext) + ) + override val pageQuery: ParamQuery[OutputParam] = + Query.withParam[OutputParam, Traversal.V[Case], IteratorOutput]( "page", FieldsParser[OutputParam], { @@ -215,7 +219,10 @@ class PublicCase @Inject() ( } } ) - override val outputQuery: Query = Query.outputWithContext[RichCase, Traversal.V[Case]]((caseSteps, authContext) => caseSteps.richCase(authContext)) + override val outputQuery: + Query = Query.outputWithContext[RichCase, Traversal.V[Case]]((caseSteps, authContext) => + caseSteps.richCase(authContext) + ) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Case], Traversal.V[Observable]]("observables", (caseSteps, authContext) => caseSteps.observables(authContext)), Query[Traversal.V[Case], Traversal.V[Task]]("tasks", (caseSteps, authContext) => caseSteps.tasks(authContext)) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index f972afd972..345a2a643e 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -571,6 +571,27 @@ object Conversion { .transform } + implicit class InputTaxonomyOps(inputTaxonomy: InputTaxonomy) { + + def toTaxonomy: Taxonomy = + inputTaxonomy + .into[Taxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .transform + } + + implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( + _.into[OutputTaxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .withFieldComputed(_.predicates, _.predicates) + .withFieldComputed(_.values, _.values) + .transform + ) + implicit class InputUserOps(inputUser: InputUser) { def toUser: User = diff --git a/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala new file mode 100644 index 0000000000..ac6622e6d2 --- /dev/null +++ b/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala @@ -0,0 +1,74 @@ +package org.thp.thehive.controllers.v0 + +import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName +import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} +import org.thp.scalligraph.models.{Database, UMapping} +import org.thp.scalligraph.query.{ParamQuery, PublicProperties, PublicPropertyListBuilder, Query, QueryExecutor} +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.thehive.controllers.v0.Conversion.taxonomyOutput +import org.thp.thehive.dto.v1.InputTaxonomy +import org.thp.thehive.models.{Permissions, RichTaxonomy, Taxonomy} +import org.thp.thehive.services.{OrganisationSrv, TaxonomySrv} +import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.TaxonomyOps._ +import play.api.mvc.{Action, AnyContent} + +class TaxonomyCtrl @Inject() ( + override val entrypoint: Entrypoint, + @Named("with-thehive-schema") override val db: Database, + @Named("v0") override val queryExecutor: QueryExecutor, + override val publicData: PublicTaxonomy +) extends QueryCtrl { + def importTaxonomy: Action[AnyContent] = + entrypoint("import taxonomy") + .extract("file", FieldsParser.file.optional.on("file")) + .extract("taxonomy", FieldsParser[InputTaxonomy]) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val file: Option[FFile] = request.body("file") + val taxonomy: InputTaxonomy = request.body("taxonomy") + + // Create Taxonomy vertex + // Create Tags associated + // Add edge orgaTaxo + + ??? + } + +} + +@Singleton +class PublicTaxonomy @Inject() ( + taxonomySrv: TaxonomySrv, + organisationSrv: OrganisationSrv +) extends PublicData { + override val entityName: String = "taxonomy" + override val initialQuery: Query = + Query.init[Traversal.V[Taxonomy]]("listTaxonomy", (graph, authContext) => + organisationSrv.get(authContext.organisation)(graph).taxonomies + ) + override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( + "page", + FieldsParser[OutputParam], + (range, taxoSteps, _) => taxoSteps.page(range.from, range.to, withTotal = true)(???) + ) + override val outputQuery: Query = Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((taxonomySteps, authContext) => + taxonomySteps.richTaxonomy(authContext) + ) + override val getQuery: ParamQuery[EntityIdOrName] = + Query.initWithParam[EntityIdOrName, Traversal.V[Taxonomy]]( + "getTaxonomy", + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => taxonomySrv.get(idOrName)(graph).visible(authContext) + ) + override val publicProperties: PublicProperties = + PublicPropertyListBuilder[Taxonomy] + .property("namespace", UMapping.string)(_.field.readonly) + .property("description", UMapping.string)(_.field.readonly) + .property("version", UMapping.int)(_.field.readonly) + // Predicates ? + // Values ? + .build + +} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 5c569fb0d0..759df983db 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -5,6 +5,7 @@ import java.util.Date import io.scalaland.chimney.dsl._ import org.thp.scalligraph.controllers.Renderer import org.thp.scalligraph.models.Entity +import org.thp.thehive.dto.v0.{InputTaxonomy, OutputTaxonomy} import org.thp.thehive.dto.v1._ import org.thp.thehive.models._ import play.api.libs.json.{JsObject, JsValue, Json} @@ -251,6 +252,27 @@ object Conversion { .transform } + implicit class InputTaxonomyOps(inputTaxonomy: InputTaxonomy) { + + def toTaxonomy: Taxonomy = + inputTaxonomy + .into[Taxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .transform + } + + implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( + _.into[OutputTaxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .withFieldComputed(_.predicates, _.predicates) + .withFieldComputed(_.values, _.values) + .transform + ) + implicit class InputUserOps(inputUser: InputUser) { def toUser: User = diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index bf10a22dde..81ea621740 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -14,6 +14,7 @@ object Permissions extends Perms { lazy val manageAction: PermissionDesc = PermissionDesc("manageAction", "Run Cortex responders ", "organisation") lazy val manageConfig: PermissionDesc = PermissionDesc("manageConfig", "Manage configurations", "organisation", "admin") lazy val manageProfile: PermissionDesc = PermissionDesc("manageProfile", "Manage user profiles", "admin") + lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "admin") lazy val manageTag: PermissionDesc = PermissionDesc("manageTag", "Manage tags", "admin") lazy val manageCustomField: PermissionDesc = PermissionDesc("manageCustomField", "Manage custom fields", "admin") lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") @@ -34,6 +35,7 @@ object Permissions extends Perms { manageAction, manageConfig, manageProfile, + manageTaxonomy, manageTag, manageCustomField, manageShare, diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala new file mode 100644 index 0000000000..3ed3c6f0cb --- /dev/null +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -0,0 +1,36 @@ +package org.thp.thehive.models + +import java.util.Date + +import org.thp.scalligraph.models.Entity +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} + +@BuildVertexEntity +case class Taxonomy( + namespace: String, + description: String, + version: Int +) + +case class Predicate(value: String) + +case class Value(predicate: String, entry: Seq[String]) + +@BuildEdgeEntity[Taxonomy, Tag] +case class TaxonomyTag() + +case class RichTaxonomy( + taxonomy: Taxonomy with Entity, + predicates: Seq[Predicate], + values: Seq[Value] +) { + def _id: EntityId = taxonomy._id + def _createdBy: String = taxonomy._createdBy + def _updatedBy: Option[String] = taxonomy._updatedBy + def _createdAt: Date = taxonomy._createdAt + def _updatedAt: Option[Date] = taxonomy._updatedAt + def namespace: String = taxonomy.namespace + def description: String = taxonomy.description + def version: Int = taxonomy.version + +} diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 69af5f84de..e74b249a54 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -138,6 +138,8 @@ object OrganisationOps { def shares: Traversal.V[Share] = traversal.out[OrganisationShare].v[Share] + def taxonomies: Traversal.V[Taxonomy] = traversal.out[OrganisationTaxonomy].v[Taxonomy] + def caseTemplates: Traversal.V[CaseTemplate] = traversal.in[CaseTemplateOrganisation].v[CaseTemplate] def users(requiredPermission: Permission): Traversal.V[User] = diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala new file mode 100644 index 0000000000..7bd16db5c6 --- /dev/null +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -0,0 +1,66 @@ +package org.thp.thehive.services + +import java.util.{Map => JMap} + +import javax.inject.{Inject, Named} +import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.auth.AuthContext +import org.thp.scalligraph.models.{Database, Entity} +import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} +import org.thp.scalligraph.traversal.{Converter, Traversal} +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.thehive.models.{Organisation, OrganisationTaxonomy, Predicate, RichTaxonomy, Tag, Taxonomy, TaxonomyTag, Value} +import org.thp.thehive.services.OrganisationOps._ + +import scala.util.Try + +@Singleton +class TaxonomySrv @Inject() ( +)(implicit + @Named("with-thehive-schema") db: Database +) extends VertexSrv[Taxonomy] { + + val taxonomyTagSrv = new EdgeSrv[TaxonomyTag, Taxonomy, Tag] + + def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = + for { + taxonomy <- createEntity(taxo) + _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) + richTaxonomy <- RichTaxonomy(taxonomy, ???, ???) + } yield richTaxonomy +} + +object TaxonomyOps { + implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { + + def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = visible(authContext.organisation) + + def visible(organisationIdOrName: EntityIdOrName): Traversal.V[Taxonomy] = + traversal.filter(_.organisations.get(organisationIdOrName)) + + def organisations: Traversal.V[Organisation] = traversal.in[OrganisationTaxonomy].v[Organisation] + + def tags: Traversal.V[Tag] = traversal.out[TaxonomyTag].v[Tag] + + def richTaxonomy(implicit authContext: AuthContext): Traversal[RichTaxonomy, JMap[String, Any], Converter[RichTaxonomy, JMap[String, Any]]] = + traversal + .project( + _.by + .by(_.tags.fold) + ) + .domainMap { + case (taxonomy, tags) => + val predicates = tags.map(t => Predicate(t.predicate)).distinct + val values = predicates.map { p => + val tagValues = tags + .filter(_.predicate == p.value) + .filter(_.value.isDefined) + .map(_.value.get) + Value(p.value, tagValues) + } + RichTaxonomy(taxonomy, predicates, values) + } + + } +} From 95a425dea7e76c5ba96d1b68e9ea793c33ab5718 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 12 Nov 2020 18:52:40 +0100 Subject: [PATCH 003/324] WIP Continued mapping for taxonomies --- .../org/thp/thehive/dto/v0/Taxonomy.scala | 43 -------- .../org/thp/thehive/dto/v1/Taxonomy.scala | 17 +-- .../thehive/controllers/v0/Conversion.scala | 21 ---- .../thehive/controllers/v0/TaxonomyCtrl.scala | 74 ------------- .../thehive/controllers/v1/Conversion.scala | 26 +++-- .../thehive/controllers/v1/Properties.scala | 10 ++ .../thehive/controllers/v1/TaxonomyCtrl.scala | 103 ++++++++++++++++++ .../app/org/thp/thehive/models/Taxonomy.scala | 8 +- .../app/org/thp/thehive/services/TagSrv.scala | 12 +- .../thp/thehive/services/TaxonomySrv.scala | 42 +++---- 10 files changed, 168 insertions(+), 188 deletions(-) delete mode 100644 dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala delete mode 100644 thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala create mode 100644 thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala deleted file mode 100644 index a5af5ff61e..0000000000 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.thp.thehive.dto.v0 - -import java.util.Date - -import play.api.libs.json.{Json, OFormat, OWrites} - -case class InputTaxonomy ( - namespace: String, - description: String, - version: Int, - predicates: Seq[InputPredicate], - values: Option[Seq[InputValue]] -) - -case class InputPredicate(value: String, expanded: String) - -case class InputValue(predicate: String, entry: Seq[InputPredicate]) - -object InputTaxonomy { - implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] -} - -case class OutputTaxonomy( - _id: String, - _type: String, - _createdBy: String, - _updatedBy: Option[String] = None, - _createdAt: Date, - _updatedAt: Option[Date] = None, - namespace: String, - description: String, - version: Int, - predicates: Seq[OutputPredicate], - values: Option[Seq[OutputValue]] -) - -case class OutputPredicate(value: String, expanded: String) - -case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) - -object OutputTaxonomy { - implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] -} \ No newline at end of file diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 1c6a1b2bc9..a2d05e879c 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -4,17 +4,18 @@ import java.util.Date import play.api.libs.json.{Json, OFormat, OWrites} +// TODO make sure of input format case class InputTaxonomy ( namespace: String, description: String, version: Int, - predicates: Seq[InputPredicate], - values: Option[Seq[InputValue]] + predicates: Seq[String], + values: Option[Seq[InputEntry]] ) -case class InputPredicate(value: String, expanded: String) +case class InputEntry(predicate: String, entry: Seq[InputValue]) -case class InputValue(predicate: String, entry: Seq[InputPredicate]) +case class InputValue(value: String, expanded: String, colour: Option[String]) object InputTaxonomy { implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] @@ -30,13 +31,13 @@ case class OutputTaxonomy( namespace: String, description: String, version: Int, - predicates: Seq[OutputPredicate], - values: Option[Seq[OutputValue]] + predicates: Seq[String], + values: Option[Seq[OutputEntry]] ) -case class OutputPredicate(value: String, expanded: String) +case class OutputEntry(predicate: String, entry: Seq[OutputValue]) -case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) +case class OutputValue(value: String, expanded: String) object OutputTaxonomy { implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index 345a2a643e..f972afd972 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -571,27 +571,6 @@ object Conversion { .transform } - implicit class InputTaxonomyOps(inputTaxonomy: InputTaxonomy) { - - def toTaxonomy: Taxonomy = - inputTaxonomy - .into[Taxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .transform - } - - implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( - _.into[OutputTaxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldComputed(_.predicates, _.predicates) - .withFieldComputed(_.values, _.values) - .transform - ) - implicit class InputUserOps(inputUser: InputUser) { def toUser: User = diff --git a/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala deleted file mode 100644 index ac6622e6d2..0000000000 --- a/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala +++ /dev/null @@ -1,74 +0,0 @@ -package org.thp.thehive.controllers.v0 - -import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.EntityIdOrName -import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} -import org.thp.scalligraph.models.{Database, UMapping} -import org.thp.scalligraph.query.{ParamQuery, PublicProperties, PublicPropertyListBuilder, Query, QueryExecutor} -import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs -import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.thehive.controllers.v0.Conversion.taxonomyOutput -import org.thp.thehive.dto.v1.InputTaxonomy -import org.thp.thehive.models.{Permissions, RichTaxonomy, Taxonomy} -import org.thp.thehive.services.{OrganisationSrv, TaxonomySrv} -import org.thp.thehive.services.OrganisationOps._ -import org.thp.thehive.services.TaxonomyOps._ -import play.api.mvc.{Action, AnyContent} - -class TaxonomyCtrl @Inject() ( - override val entrypoint: Entrypoint, - @Named("with-thehive-schema") override val db: Database, - @Named("v0") override val queryExecutor: QueryExecutor, - override val publicData: PublicTaxonomy -) extends QueryCtrl { - def importTaxonomy: Action[AnyContent] = - entrypoint("import taxonomy") - .extract("file", FieldsParser.file.optional.on("file")) - .extract("taxonomy", FieldsParser[InputTaxonomy]) - .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - val file: Option[FFile] = request.body("file") - val taxonomy: InputTaxonomy = request.body("taxonomy") - - // Create Taxonomy vertex - // Create Tags associated - // Add edge orgaTaxo - - ??? - } - -} - -@Singleton -class PublicTaxonomy @Inject() ( - taxonomySrv: TaxonomySrv, - organisationSrv: OrganisationSrv -) extends PublicData { - override val entityName: String = "taxonomy" - override val initialQuery: Query = - Query.init[Traversal.V[Taxonomy]]("listTaxonomy", (graph, authContext) => - organisationSrv.get(authContext.organisation)(graph).taxonomies - ) - override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( - "page", - FieldsParser[OutputParam], - (range, taxoSteps, _) => taxoSteps.page(range.from, range.to, withTotal = true)(???) - ) - override val outputQuery: Query = Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((taxonomySteps, authContext) => - taxonomySteps.richTaxonomy(authContext) - ) - override val getQuery: ParamQuery[EntityIdOrName] = - Query.initWithParam[EntityIdOrName, Traversal.V[Taxonomy]]( - "getTaxonomy", - FieldsParser[EntityIdOrName], - (idOrName, graph, authContext) => taxonomySrv.get(idOrName)(graph).visible(authContext) - ) - override val publicProperties: PublicProperties = - PublicPropertyListBuilder[Taxonomy] - .property("namespace", UMapping.string)(_.field.readonly) - .property("description", UMapping.string)(_.field.readonly) - .property("version", UMapping.int)(_.field.readonly) - // Predicates ? - // Values ? - .build - -} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 759df983db..27e11ab729 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -5,7 +5,7 @@ import java.util.Date import io.scalaland.chimney.dsl._ import org.thp.scalligraph.controllers.Renderer import org.thp.scalligraph.models.Entity -import org.thp.thehive.dto.v0.{InputTaxonomy, OutputTaxonomy} +import org.thp.thehive.dto.v1.{InputTaxonomy, OutputTaxonomy} import org.thp.thehive.dto.v1._ import org.thp.thehive.models._ import play.api.libs.json.{JsObject, JsValue, Json} @@ -263,15 +263,20 @@ object Conversion { .transform } - implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( - _.into[OutputTaxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldComputed(_.predicates, _.predicates) - .withFieldComputed(_.values, _.values) - .transform - ) + implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = + Renderer.toJson[RichTaxonomy, OutputTaxonomy]( + _.into[OutputTaxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) + .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { + val outputValues = entryMap.getOrElse(tag.predicate, Seq()) + val value = OutputValue(tag.value.getOrElse(""), tag.description.getOrElse("")) + entryMap + (tag.predicate -> (outputValues :+ value)) + }).map(e => OutputEntry(e._1, e._2))) + .transform + ) implicit class InputUserOps(inputUser: InputUser) { @@ -357,6 +362,7 @@ object Conversion { .withFieldComputed(_.tlp, _.tlp.getOrElse(2)) .transform } + implicit val observableOutput: Renderer.Aux[RichObservable, OutputObservable] = Renderer.toJson[RichObservable, OutputObservable](richObservable => richObservable .into[OutputObservable] diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index f435c9b3d3..a7d78bc02a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -375,4 +375,14 @@ class Properties @Inject() ( .property("data", UMapping.string.optional)(_.select(_.data.value(_.data)).readonly) // TODO add attachment ? .build + + lazy val taxonomy: PublicProperties = + PublicPropertyListBuilder[Taxonomy] + .property("namespace", UMapping.string)(_.field.readonly) + .property("description", UMapping.string)(_.field.readonly) + .property("version", UMapping.int)(_.field.readonly) + // Predicates ? + // Values ? + .build + } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala new file mode 100644 index 0000000000..d60db4d141 --- /dev/null +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -0,0 +1,103 @@ +package org.thp.thehive.controllers.v1 + +import javax.inject.{Inject, Named} +import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} +import org.thp.scalligraph.models.Database +import org.thp.scalligraph.query._ +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.thehive.controllers.v1.Conversion._ +import org.thp.thehive.dto.v1.InputTaxonomy +import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} +import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.TaxonomyOps._ +import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} +import play.api.mvc.{Action, AnyContent, Results} + +import scala.+: + +class TaxonomyCtrl @Inject() ( + entrypoint: Entrypoint, + properties: Properties, + taxonomySrv: TaxonomySrv, + organisationSrv: OrganisationSrv, + tagSrv: TagSrv, + @Named("with-thehive-schema") implicit val db: Database +) extends QueryableCtrl { + + override val entityName: String = "taxonomy" + override val publicProperties: PublicProperties = properties.taxonomy + override val initialQuery: Query = + Query.init[Traversal.V[Taxonomy]]("listTaxonomy", (graph, authContext) => + organisationSrv.get(authContext.organisation)(graph).taxonomies + ) + override val getQuery: ParamQuery[EntityIdOrName] = + Query.initWithParam[EntityIdOrName, Traversal.V[Taxonomy]]( + "getTaxonomy", + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => taxonomySrv.get(idOrName)(graph).visible(authContext) + ) + override val pageQuery: ParamQuery[OutputParam] = + Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( + "page", + FieldsParser[OutputParam], + (range, traversal, authContext) => + traversal.richPage(range.from, range.to, range.extraData.contains("total"))(_.richTaxonomy(authContext)) + ) + override val outputQuery: Query = + Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((traversal, authContext) => + traversal.richTaxonomy(authContext) + ) + override val extraQueries: Seq[ParamQuery[_]] = Seq( + Query[Traversal.V[Taxonomy], Traversal.V[Tag]]("tags", (traversal, _) => traversal.tags) + ) + + def importTaxonomy: Action[AnyContent] = + entrypoint("import taxonomy") + .extract("file", FieldsParser.file.optional.on("file")) + .extract("taxonomy", FieldsParser[InputTaxonomy]) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val file: Option[FFile] = request.body("file") + val inputTaxo: InputTaxonomy = request.body("taxonomy") + + // TODO Parse file & combine with body + + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) + + // Create tags + val tagValues = inputTaxo.values.getOrElse(Seq()) + val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { + all ++ value.entry.map(e => + Tag(inputTaxo.namespace, + value.predicate, + Some(e.value), + Some(e.expanded), + e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour)) + + ) + } + ) + + // Create a tag for predicates with no tags associated + val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + tags ++ predicateWithNoTags.map(p => + Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) + ) + + for { + tagsEntities <- tags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield Results.Created(richTaxonomy.toJson) + } + + def delete(namespace: String): Action[AnyContent] = + entrypoint("delete taxonomy") + .authTransaction(db) { implicit request => implicit graph => + for { + t <- taxonomySrv.getByNamespace(namespace) + + } yield Results.Nocontent + } + +} diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index 3ed3c6f0cb..925956d9cd 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -12,17 +12,12 @@ case class Taxonomy( version: Int ) -case class Predicate(value: String) - -case class Value(predicate: String, entry: Seq[String]) - @BuildEdgeEntity[Taxonomy, Tag] case class TaxonomyTag() case class RichTaxonomy( taxonomy: Taxonomy with Entity, - predicates: Seq[Predicate], - values: Seq[Value] + tags: Seq[Tag with Entity] ) { def _id: EntityId = taxonomy._id def _createdBy: String = taxonomy._createdBy @@ -32,5 +27,4 @@ case class RichTaxonomy( def namespace: String = taxonomy.namespace def description: String = taxonomy.description def version: Int = taxonomy.version - } diff --git a/thehive/app/org/thp/thehive/services/TagSrv.scala b/thehive/app/org/thp/thehive/services/TagSrv.scala index 7b78029f4f..a1ade9e7a3 100644 --- a/thehive/app/org/thp/thehive/services/TagSrv.scala +++ b/thehive/app/org/thp/thehive/services/TagSrv.scala @@ -19,27 +19,31 @@ class TagSrv @Inject() (appConfig: ApplicationConfig, @Named("integrity-check-ac @Named("with-thehive-schema") db: Database ) extends VertexSrv[Tag] { - val autoCreateConfig: ConfigItem[Boolean, Boolean] = + private val autoCreateConfig: ConfigItem[Boolean, Boolean] = appConfig.item[Boolean]("tags.autocreate", "If true, create automatically tag if it doesn't exist") def autoCreate: Boolean = autoCreateConfig.get - val defaultNamespaceConfig: ConfigItem[String, String] = + private val defaultNamespaceConfig: ConfigItem[String, String] = appConfig.item[String]("tags.defaultNamespace", "Default namespace of the automatically created tags") def defaultNamespace: String = defaultNamespaceConfig.get - val defaultColourConfig: ConfigItem[String, Int] = + private val defaultColourConfig: ConfigItem[String, Int] = appConfig.mapItem[String, Int]( "tags.defaultColour", "Default colour of the automatically created tags", { - case s if s(0) == '#' => Try(Integer.parseUnsignedInt(s.tail, 16)).getOrElse(defaultColour) + case s if s(0) == '#' => parseTagColour(s.tail) case _ => defaultColour } ) + def defaultColour: Int = defaultColourConfig.get + // TODO Duplication in Tag.scala + def parseTagColour(c: String) = Try(Integer.parseUnsignedInt(c, 16)).getOrElse(defaultColour) + def parseString(tagName: String): Tag = Tag.fromString(tagName, defaultNamespace, defaultColour) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 7bd16db5c6..8a7e906979 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -4,36 +4,48 @@ import java.util.{Map => JMap} import javax.inject.{Inject, Named} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.{EntityIdOrName, RichSeq} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} -import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs -import org.thp.thehive.models.{Organisation, OrganisationTaxonomy, Predicate, RichTaxonomy, Tag, Taxonomy, TaxonomyTag, Value} +import org.thp.scalligraph.traversal.{Converter, Traversal} +import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ import scala.util.Try @Singleton class TaxonomySrv @Inject() ( -)(implicit - @Named("with-thehive-schema") db: Database + organisationSrv: OrganisationSrv, + tagSrv: TagSrv +)(implicit @Named("with-thehive-schema") db: Database ) extends VertexSrv[Taxonomy] { val taxonomyTagSrv = new EdgeSrv[TaxonomyTag, Taxonomy, Tag] + val organisationTaxonomySrv = new EdgeSrv[OrganisationTaxonomy, Organisation, Taxonomy] def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { - taxonomy <- createEntity(taxo) - _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- RichTaxonomy(taxonomy, ???, ???) + taxonomy <- createEntity(taxo) + organisation <- organisationSrv.getOrFail(authContext.organisation) + _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) + _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy +/* + + def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = + Try(startTraversal.getByNamespace(namespace)).getOrElse(startTraversal.limit(0)) +*/ + } object TaxonomyOps { implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { + def getByNamespace(namespace: String): Traversal.V[Taxonomy] = traversal.has(_.namespace, namespace) + def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = visible(authContext.organisation) def visible(organisationIdOrName: EntityIdOrName): Traversal.V[Taxonomy] = @@ -49,18 +61,6 @@ object TaxonomyOps { _.by .by(_.tags.fold) ) - .domainMap { - case (taxonomy, tags) => - val predicates = tags.map(t => Predicate(t.predicate)).distinct - val values = predicates.map { p => - val tagValues = tags - .filter(_.predicate == p.value) - .filter(_.value.isDefined) - .map(_.value.get) - Value(p.value, tagValues) - } - RichTaxonomy(taxonomy, predicates, values) - } - + .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags) } } } From 08c0a5c5280a5fc78cc0b8aa8f94296c5503c6a6 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 15:18:24 +0100 Subject: [PATCH 004/324] WIP create taxonomiy / get works --- .../thp/thehive/dto/v1/CustomFieldValue.scala | 1 + .../org/thp/thehive/dto/v1/Taxonomy.scala | 30 +++++- .../thehive/controllers/v1/Conversion.scala | 10 +- .../thp/thehive/controllers/v1/Router.scala | 8 ++ .../thehive/controllers/v1/TaxonomyCtrl.scala | 93 +++++++++++-------- .../org/thp/thehive/models/Permissions.scala | 2 +- .../models/TheHiveSchemaDefinition.scala | 17 +++- .../thehive/services/OrganisationSrv.scala | 10 +- .../thp/thehive/services/TaxonomySrv.scala | 8 +- 9 files changed, 125 insertions(+), 54 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index 6e72438d06..06d6fb16e4 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -70,6 +70,7 @@ object InputCustomFieldValue { } case _ => Good(Nil) } + implicit val writes: Writes[Seq[InputCustomFieldValue]] = Writes[Seq[InputCustomFieldValue]] { icfv => val fields = icfv.map { case InputCustomFieldValue(name, Some(s: String), _) => name -> JsString(s) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index a2d05e879c..f0ebfb9659 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -2,7 +2,11 @@ package org.thp.thehive.dto.v1 import java.util.Date -import play.api.libs.json.{Json, OFormat, OWrites} +import org.scalactic.Accumulation.convertGenTraversableOnceToValidatable +import org.scalactic.{Bad, Good, One} +import org.thp.scalligraph.InvalidFormatAttributeError +import org.thp.scalligraph.controllers.{FObject, FSeq, FieldsParser, WithParser} +import play.api.libs.json.{JsArray, JsObject, JsString, Json, OFormat, OWrites, Writes} // TODO make sure of input format case class InputTaxonomy ( @@ -17,6 +21,20 @@ case class InputEntry(predicate: String, entry: Seq[InputValue]) case class InputValue(value: String, expanded: String, colour: Option[String]) +object InputEntry { + implicitly[FieldsParser[Option[Seq[InputEntry]]]] + + implicit val parser: FieldsParser[InputEntry] = FieldsParser[InputEntry] + + implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] +} + +object InputValue { + implicit val parser: FieldsParser[InputValue] = FieldsParser[InputValue] + + implicit val writes: Writes[InputValue] = Json.writes[InputValue] +} + object InputTaxonomy { implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] } @@ -32,7 +50,7 @@ case class OutputTaxonomy( description: String, version: Int, predicates: Seq[String], - values: Option[Seq[OutputEntry]] + values: Seq[OutputEntry] ) case class OutputEntry(predicate: String, entry: Seq[OutputValue]) @@ -41,4 +59,12 @@ case class OutputValue(value: String, expanded: String) object OutputTaxonomy { implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] +} + +object OutputEntry { + implicit val format: OFormat[OutputEntry] = Json.format[OutputEntry] +} + +object OutputValue { + implicit val format: OFormat[OutputValue] = Json.format[OutputValue] } \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 27e11ab729..2ee833e562 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -266,15 +266,19 @@ object Conversion { implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( _.into[OutputTaxonomy] + .withFieldComputed(_._id, _._id.toString) + .withFieldConst(_._type, "Taxonomy") .withFieldComputed(_.namespace, _.namespace) .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { val outputValues = entryMap.getOrElse(tag.predicate, Seq()) - val value = OutputValue(tag.value.getOrElse(""), tag.description.getOrElse("")) - entryMap + (tag.predicate -> (outputValues :+ value)) - }).map(e => OutputEntry(e._1, e._2))) + if (tag.value.isDefined) + entryMap + (tag.predicate -> (outputValues :+ OutputValue(tag.value.get, tag.description.getOrElse("")))) + else + entryMap + (tag.predicate -> outputValues) + }).map(e => OutputEntry(e._1, e._2)).toSeq) .transform ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index feffe865bb..1683c010ad 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -14,6 +14,7 @@ class Router @Inject() ( taskCtrl: TaskCtrl, customFieldCtrl: CustomFieldCtrl, alertCtrl: AlertCtrl, + taxonomyCtrl: TaxonomyCtrl, auditCtrl: AuditCtrl, statusCtrl: StatusCtrl, authenticationCtrl: AuthenticationCtrl, @@ -90,6 +91,13 @@ class Router @Inject() ( // DELETE /alert/:alertId controllers.AlertCtrl.delete(alertId) // POST /alert/:alertId/merge/:caseId controllers.AlertCtrl.mergeWithCase(alertId, caseId) + case GET(p"/taxonomy") => taxonomyCtrl.list + case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) + case POST(p"/taxonomy") => taxonomyCtrl.create + // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip + // case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.activate + // case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.deactivate + case GET(p"/audit") => auditCtrl.flow // GET /flow controllers.AuditCtrl.flow(rootId: Option[String], count: Option[Int]) // GET /audit controllers.AuditCtrl.find() diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index d60db4d141..6a8220514c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -2,7 +2,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} import org.thp.scalligraph.{EntityIdOrName, RichSeq} -import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} +import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs @@ -15,7 +15,7 @@ import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} import play.api.mvc.{Action, AnyContent, Results} -import scala.+: +import scala.util.Success class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, @@ -42,55 +42,73 @@ class TaxonomyCtrl @Inject() ( Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( "page", FieldsParser[OutputParam], - (range, traversal, authContext) => - traversal.richPage(range.from, range.to, range.extraData.contains("total"))(_.richTaxonomy(authContext)) + (range, traversal, _) => + traversal.richPage(range.from, range.to, range.extraData.contains("total"))(_.richTaxonomy) ) override val outputQuery: Query = - Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((traversal, authContext) => - traversal.richTaxonomy(authContext) + Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((traversal, _) => + traversal.richTaxonomy ) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Taxonomy], Traversal.V[Tag]]("tags", (traversal, _) => traversal.tags) ) - def importTaxonomy: Action[AnyContent] = - entrypoint("import taxonomy") - .extract("file", FieldsParser.file.optional.on("file")) - .extract("taxonomy", FieldsParser[InputTaxonomy]) - .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - val file: Option[FFile] = request.body("file") - val inputTaxo: InputTaxonomy = request.body("taxonomy") - - // TODO Parse file & combine with body + def list: Action[AnyContent] = + entrypoint("list taxonomies") + .authRoTransaction(db) { implicit request => implicit graph => + val taxos = taxonomySrv + .startTraversal + .visible + .richTaxonomy + .toSeq + Success(Results.Ok(taxos.toJson)) + } - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) + def create: Action[AnyContent] = + entrypoint("import taxonomy") + .extract("taxonomy", FieldsParser[InputTaxonomy]) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val inputTaxo: InputTaxonomy = request.body("taxonomy") - // Create tags - val tagValues = inputTaxo.values.getOrElse(Seq()) - val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { - all ++ value.entry.map(e => - Tag(inputTaxo.namespace, - value.predicate, - Some(e.value), - Some(e.expanded), - e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour)) + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) - ) - } - ) + // Create tags + val tagValues = inputTaxo.values.getOrElse(Seq()) + val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { + all ++ value.entry.map(e => + Tag(inputTaxo.namespace, + value.predicate, + Some(e.value), + Some(e.expanded), + e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour) + ) + ) + }) - // Create a tag for predicates with no tags associated - val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) - tags ++ predicateWithNoTags.map(p => - Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) - ) + // Create a tag for predicates with no tags associated + val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + val allTags = tags ++ predicateWithNoTags.map(p => + Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) + ) - for { - tagsEntities <- tags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) - } yield Results.Created(richTaxonomy.toJson) + for { + tagsEntities <- allTags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield Results.Created(richTaxonomy.toJson) } + def get(taxonomyId: String): Action[AnyContent] = + entrypoint("get taxonomy") + .authRoTransaction(db) { implicit request => implicit graph => + taxonomySrv + .get(EntityIdOrName(taxonomyId)) + .visible + .richTaxonomy + .getOrFail("Taxonomy") + .map(taxonomy => Results.Ok(taxonomy.toJson)) + } + +/* def delete(namespace: String): Action[AnyContent] = entrypoint("delete taxonomy") .authTransaction(db) { implicit request => implicit graph => @@ -99,5 +117,6 @@ class TaxonomyCtrl @Inject() ( } yield Results.Nocontent } +*/ } diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index 81ea621740..7c079860de 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -14,7 +14,7 @@ object Permissions extends Perms { lazy val manageAction: PermissionDesc = PermissionDesc("manageAction", "Run Cortex responders ", "organisation") lazy val manageConfig: PermissionDesc = PermissionDesc("manageConfig", "Manage configurations", "organisation", "admin") lazy val manageProfile: PermissionDesc = PermissionDesc("manageProfile", "Manage user profiles", "admin") - lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "admin") + lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "organisation", "admin") lazy val manageTag: PermissionDesc = PermissionDesc("manageTag", "Manage tags", "admin") lazy val manageCustomField: PermissionDesc = PermissionDesc("manageCustomField", "Manage custom fields", "admin") lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 38953a7a51..0baa6d6961 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -1,6 +1,7 @@ package org.thp.thehive.models import java.lang.reflect.Modifier +import java.util.Date import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph @@ -82,7 +83,12 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { db.tryTransaction { g => db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => val taxoVertex = g.addVertex("Taxonomy") - taxoVertex.property("namespace", "Custom") + taxoVertex.property("_label", "Taxonomy") + taxoVertex.property("_createdBy", "???") // TODO What user should be used ? + taxoVertex.property("_createdAt", new Date()) + taxoVertex.property("namespace", "custom") + taxoVertex.property("description", "Custom taxonomy") + taxoVertex.property("version", 1) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) } @@ -97,13 +103,18 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), _.in("AlertOrganisation").out("AlertTag"), _.in("CaseTemplateOrganisation").out("CaseTemplateTag") - ).toSeq.foreach(tag => + ).toSeq.foreach { tag => + tag.property("namespace", "custom") customTaxo.addEdge("TaxonomyTag", tag) - ) + } Success(()) } }.map(_ => ()) } + .updateGraph("Add manageTaxonomy to org-admin profile", "Profile") { traversal => + Try(traversal.unsafeHas("name", "org-admin").raw.property("permissions", "manageTaxonomy").iterate()) + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index e74b249a54..026f8a9554 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -3,7 +3,7 @@ package org.thp.thehive.services import java.util.{Map => JMap} import akka.actor.ActorRef -import javax.inject.{Inject, Named, Singleton} +import javax.inject.{Inject, Named, Provider, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.models._ @@ -23,6 +23,7 @@ import scala.util.{Failure, Success, Try} @Singleton class OrganisationSrv @Inject() ( + taxonomySrvProvider: Provider[TaxonomySrv], roleSrv: RoleSrv, profileSrv: ProfileSrv, auditSrv: AuditSrv, @@ -31,9 +32,9 @@ class OrganisationSrv @Inject() ( )(implicit @Named("with-thehive-schema") db: Database ) extends VertexSrv[Organisation] { - - val organisationOrganisationSrv = new EdgeSrv[OrganisationOrganisation, Organisation, Organisation] - val organisationShareSrv = new EdgeSrv[OrganisationShare, Organisation, Share] + lazy val taxonomySrv: TaxonomySrv = taxonomySrvProvider.get + val organisationOrganisationSrv = new EdgeSrv[OrganisationOrganisation, Organisation, Organisation] + val organisationShareSrv = new EdgeSrv[OrganisationShare, Organisation, Share] override def createEntity(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("Organisation") @@ -51,6 +52,7 @@ class OrganisationSrv @Inject() ( def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = for { createdOrganisation <- createEntity(e) + _ <- taxonomySrv.create(Taxonomy("custom", "Custom taxonomy", 1), Seq()) _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) } yield createdOrganisation diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 8a7e906979..55117defea 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -2,7 +2,7 @@ package org.thp.thehive.services import java.util.{Map => JMap} -import javax.inject.{Inject, Named} +import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} @@ -17,8 +17,7 @@ import scala.util.Try @Singleton class TaxonomySrv @Inject() ( - organisationSrv: OrganisationSrv, - tagSrv: TagSrv + organisationSrv: OrganisationSrv )(implicit @Named("with-thehive-schema") db: Database ) extends VertexSrv[Taxonomy] { @@ -33,6 +32,7 @@ class TaxonomySrv @Inject() ( _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy + /* def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = @@ -55,7 +55,7 @@ object TaxonomyOps { def tags: Traversal.V[Tag] = traversal.out[TaxonomyTag].v[Tag] - def richTaxonomy(implicit authContext: AuthContext): Traversal[RichTaxonomy, JMap[String, Any], Converter[RichTaxonomy, JMap[String, Any]]] = + def richTaxonomy: Traversal[RichTaxonomy, JMap[String, Any], Converter[RichTaxonomy, JMap[String, Any]]] = traversal .project( _.by From a6cfdb3eb970e6f6a2cd2313976aae480c7f4891 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 16:47:17 +0100 Subject: [PATCH 005/324] WIP Custom taxonomy when new organisation is created --- .../main/scala/org/thp/thehive/dto/v1/Taxonomy.scala | 2 -- .../thp/thehive/models/TheHiveSchemaDefinition.scala | 2 +- .../app/org/thp/thehive/services/OrganisationSrv.scala | 2 +- thehive/app/org/thp/thehive/services/TaxonomySrv.scala | 10 +++++++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index f0ebfb9659..a73243a59e 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -22,8 +22,6 @@ case class InputEntry(predicate: String, entry: Seq[InputValue]) case class InputValue(value: String, expanded: String, colour: Option[String]) object InputEntry { - implicitly[FieldsParser[Option[Seq[InputEntry]]]] - implicit val parser: FieldsParser[InputEntry] = FieldsParser[InputEntry] implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 0baa6d6961..5bf2b0ec70 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -84,7 +84,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => val taxoVertex = g.addVertex("Taxonomy") taxoVertex.property("_label", "Taxonomy") - taxoVertex.property("_createdBy", "???") // TODO What user should be used ? + taxoVertex.property("_createdBy", "system@thehive.local") taxoVertex.property("_createdAt", new Date()) taxoVertex.property("namespace", "custom") taxoVertex.property("description", "Custom taxonomy") diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 026f8a9554..e2a8c61068 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -52,7 +52,7 @@ class OrganisationSrv @Inject() ( def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = for { createdOrganisation <- createEntity(e) - _ <- taxonomySrv.create(Taxonomy("custom", "Custom taxonomy", 1), Seq()) + _ <- taxonomySrv.createWithOrg(Taxonomy("custom", "Custom taxonomy", 1), Seq(), createdOrganisation) _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) } yield createdOrganisation diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 55117defea..28734aefb9 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -26,8 +26,16 @@ class TaxonomySrv @Inject() ( def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { - taxonomy <- createEntity(taxo) organisation <- organisationSrv.getOrFail(authContext.organisation) + richTaxonomy <- createWithOrg(taxo, tags, organisation) + } yield richTaxonomy + + def createWithOrg(taxo: Taxonomy, + tags: Seq[Tag with Entity], + organisation: Organisation with Entity) + (implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = + for { + taxonomy <- createEntity(taxo) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) From 80ffdd2aa17888fb295b5c0d9efd5f56375f769b Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 19:02:21 +0100 Subject: [PATCH 006/324] WIP Added taxonomy activate / deactivate --- .../main/scala/org/thp/thehive/dto/v1/Taxonomy.scala | 2 +- .../org/thp/thehive/controllers/v1/Conversion.scala | 2 ++ .../app/org/thp/thehive/controllers/v1/Router.scala | 6 +++--- .../org/thp/thehive/controllers/v1/TaxonomyCtrl.scala | 10 +++++++++- thehive/app/org/thp/thehive/models/Taxonomy.scala | 4 +++- .../thp/thehive/models/TheHiveSchemaDefinition.scala | 3 ++- .../app/org/thp/thehive/services/OrganisationSrv.scala | 6 ++++-- thehive/app/org/thp/thehive/services/TaxonomySrv.scala | 10 +++++++++- 8 files changed, 33 insertions(+), 10 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index a73243a59e..fe8eaa467c 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -8,7 +8,6 @@ import org.thp.scalligraph.InvalidFormatAttributeError import org.thp.scalligraph.controllers.{FObject, FSeq, FieldsParser, WithParser} import play.api.libs.json.{JsArray, JsObject, JsString, Json, OFormat, OWrites, Writes} -// TODO make sure of input format case class InputTaxonomy ( namespace: String, description: String, @@ -47,6 +46,7 @@ case class OutputTaxonomy( namespace: String, description: String, version: Int, + enabled: Boolean, predicates: Seq[String], values: Seq[OutputEntry] ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 2ee833e562..78f235c031 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -260,6 +260,7 @@ object Conversion { .withFieldComputed(_.namespace, _.namespace) .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) + .withFieldConst(_.enabled, false) // TODO always false when importing a taxonomy ? .transform } @@ -271,6 +272,7 @@ object Conversion { .withFieldComputed(_.namespace, _.namespace) .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) + .withFieldComputed(_.enabled, _.enabled) .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { val outputValues = entryMap.getOrElse(tag.predicate, Seq()) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 1683c010ad..7fd69f6291 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -94,9 +94,9 @@ class Router @Inject() ( case GET(p"/taxonomy") => taxonomyCtrl.list case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case POST(p"/taxonomy") => taxonomyCtrl.create - // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip - // case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.activate - // case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.deactivate + // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip< + case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) + case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) case GET(p"/audit") => auditCtrl.flow // GET /flow controllers.AuditCtrl.flow(rootId: Option[String], count: Option[Int]) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 6a8220514c..b2b3ff7136 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -70,7 +70,7 @@ class TaxonomyCtrl @Inject() ( .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => val inputTaxo: InputTaxonomy = request.body("taxonomy") - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) // Create tags val tagValues = inputTaxo.values.getOrElse(Seq()) @@ -108,6 +108,14 @@ class TaxonomyCtrl @Inject() ( .map(taxonomy => Results.Ok(taxonomy.toJson)) } + def setEnabled(taxonomyId: String, isEnabled: Boolean): Action[AnyContent] = + entrypoint("toggle taxonomy") + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + taxonomySrv + .setEnabled(EntityIdOrName(taxonomyId), isEnabled) + .map(_ => Results.NoContent) + } + /* def delete(namespace: String): Action[AnyContent] = entrypoint("delete taxonomy") diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index 925956d9cd..7a8f9a46c2 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -9,7 +9,8 @@ import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} case class Taxonomy( namespace: String, description: String, - version: Int + version: Int, + enabled: Boolean ) @BuildEdgeEntity[Taxonomy, Tag] @@ -27,4 +28,5 @@ case class RichTaxonomy( def namespace: String = taxonomy.namespace def description: String = taxonomy.description def version: Int = taxonomy.version + def enabled: Boolean = taxonomy.enabled } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 5bf2b0ec70..3cca9b9b9f 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -89,6 +89,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("namespace", "custom") taxoVertex.property("description", "Custom taxonomy") taxoVertex.property("version", 1) + taxoVertex.property("enabled", true) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) } @@ -97,7 +98,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => db.tryTransaction { implicit g => db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => - val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "Custom").head + val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "custom").head Traversal.V(EntityId(o.id())).unionFlat( _.out("OrganisationShare").out("ShareCase").out("CaseTag"), _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index e2a8c61068..05f3889499 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -49,12 +49,14 @@ class OrganisationSrv @Inject() ( _ <- roleSrv.create(user, createdOrganisation, profileSrv.orgAdmin) } yield createdOrganisation - def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = + def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { + val customTaxo = Taxonomy("custom", "Custom taxonomy", 1, enabled = true) for { createdOrganisation <- createEntity(e) - _ <- taxonomySrv.createWithOrg(Taxonomy("custom", "Custom taxonomy", 1), Seq(), createdOrganisation) + _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), createdOrganisation) _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) } yield createdOrganisation + } def current(implicit graph: Graph, authContext: AuthContext): Traversal.V[Organisation] = get(authContext.organisation) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 28734aefb9..42003e4f54 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -9,7 +9,7 @@ import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs import org.thp.scalligraph.traversal.{Converter, Traversal} -import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.{EntityId, EntityIdOrName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ @@ -41,6 +41,11 @@ class TaxonomySrv @Inject() ( richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy + def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = + for { + _ <- get(taxonomyId).update(_.enabled, isEnabled).getOrFail("Taxonomy") + } yield () + /* def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = @@ -52,6 +57,9 @@ class TaxonomySrv @Inject() ( object TaxonomyOps { implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { + def get(idOrName: EntityId): Traversal.V[Taxonomy] = + traversal.getByIds(idOrName) + def getByNamespace(namespace: String): Traversal.V[Taxonomy] = traversal.has(_.namespace, namespace) def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = visible(authContext.organisation) From 279f8af88c8410dc692814c93664c7de844366c5 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 19:08:08 +0100 Subject: [PATCH 007/324] Fixed taxonomy values parsing --- .../main/scala/org/thp/thehive/dto/v1/Taxonomy.scala | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index fe8eaa467c..20890915a5 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -2,11 +2,7 @@ package org.thp.thehive.dto.v1 import java.util.Date -import org.scalactic.Accumulation.convertGenTraversableOnceToValidatable -import org.scalactic.{Bad, Good, One} -import org.thp.scalligraph.InvalidFormatAttributeError -import org.thp.scalligraph.controllers.{FObject, FSeq, FieldsParser, WithParser} -import play.api.libs.json.{JsArray, JsObject, JsString, Json, OFormat, OWrites, Writes} +import play.api.libs.json.{Json, OFormat, OWrites, Writes} case class InputTaxonomy ( namespace: String, @@ -21,14 +17,10 @@ case class InputEntry(predicate: String, entry: Seq[InputValue]) case class InputValue(value: String, expanded: String, colour: Option[String]) object InputEntry { - implicit val parser: FieldsParser[InputEntry] = FieldsParser[InputEntry] - implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] } object InputValue { - implicit val parser: FieldsParser[InputValue] = FieldsParser[InputValue] - implicit val writes: Writes[InputValue] = Json.writes[InputValue] } From 2e25d591ade69eadf3f3081ea338f4849fa7a6e7 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 11:42:50 +0100 Subject: [PATCH 008/324] Idempotent TheHive schema --- .../thp/thehive/controllers/v1/Router.scala | 2 +- .../models/TheHiveSchemaDefinition.scala | 27 +++++++++++-------- .../thehive/controllers/v1/UserCtrlTest.scala | 1 + 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 7fd69f6291..f3184dffdb 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -94,7 +94,7 @@ class Router @Inject() ( case GET(p"/taxonomy") => taxonomyCtrl.list case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case POST(p"/taxonomy") => taxonomyCtrl.create - // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip< + // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 3cca9b9b9f..9edc5c41f1 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -81,17 +81,22 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .addVertexModel[String]("Taxonomy", Seq("namespace")) .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => db.tryTransaction { g => - db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => - val taxoVertex = g.addVertex("Taxonomy") - taxoVertex.property("_label", "Taxonomy") - taxoVertex.property("_createdBy", "system@thehive.local") - taxoVertex.property("_createdAt", new Date()) - taxoVertex.property("namespace", "custom") - taxoVertex.property("description", "Custom taxonomy") - taxoVertex.property("version", 1) - taxoVertex.property("enabled", true) - o.addEdge("OrganisationTaxonomy", taxoVertex) - Success(()) + // If there are no taxonomies in database, add a custom one for each organisation + db.labelFilter("Taxonomy")(Traversal.V()(g)).headOption match { + case None => + db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => + val taxoVertex = g.addVertex("Taxonomy") + taxoVertex.property("_label", "Taxonomy") + taxoVertex.property("_createdBy", "system@thehive.local") + taxoVertex.property("_createdAt", new Date()) + taxoVertex.property("namespace", "custom") + taxoVertex.property("description", "Custom taxonomy") + taxoVertex.property("version", 1) + taxoVertex.property("enabled", true) + o.addEdge("OrganisationTaxonomy", taxoVertex) + Success(()) + } + case _ => Success(()) } }.map(_ => ()) } diff --git a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala index e7ac8f762c..dd68b7d3a9 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala @@ -109,6 +109,7 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { Permissions.managePage, Permissions.manageObservable, Permissions.manageAlert, + Permissions.manageTaxonomy, Permissions.manageAction, Permissions.manageConfig ), From 49e9e0f65edbd43cf3defa91f0c1cc2229c1336b Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 14:23:35 +0100 Subject: [PATCH 009/324] Checked if taxonomy namespace is present before creating --- .../thp/thehive/controllers/v1/TaxonomyCtrl.scala | 15 +++++++++------ .../org/thp/thehive/services/TaxonomySrv.scala | 9 +++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index b2b3ff7136..844dbc311b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -1,7 +1,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} -import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ @@ -15,7 +15,7 @@ import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} import play.api.mvc.{Action, AnyContent, Results} -import scala.util.Success +import scala.util.{Failure, Success} class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, @@ -91,10 +91,13 @@ class TaxonomyCtrl @Inject() ( Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) ) - for { - tagsEntities <- allTags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) - } yield Results.Created(richTaxonomy.toJson) + if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) + Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) + else + for { + tagsEntities <- allTags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield Results.Created(richTaxonomy.toJson) } def get(taxonomyId: String): Action[AnyContent] = diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 42003e4f54..49ee30d3d3 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -24,6 +24,15 @@ class TaxonomySrv @Inject() ( val taxonomyTagSrv = new EdgeSrv[TaxonomyTag, Taxonomy, Tag] val organisationTaxonomySrv = new EdgeSrv[OrganisationTaxonomy, Organisation, Taxonomy] + def existsInOrganisation(namespace: String)(implicit graph: Graph, authContext: AuthContext): Boolean = { + startTraversal + .has(_.namespace, namespace) + .in[OrganisationTaxonomy] + .v[Organisation] + .has(_.name, authContext.organisation.toString) // TODO not great + .exists + } + def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { organisation <- organisationSrv.getOrFail(authContext.organisation) From cda163ddd0545697f814f2817119aa2ed3438742 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 15:09:28 +0100 Subject: [PATCH 010/324] Correct output format for taxonomies --- .../scala/org/thp/thehive/dto/v1/Tag.scala | 15 ++++++++++++++ .../org/thp/thehive/dto/v1/Taxonomy.scala | 15 +------------- .../thehive/controllers/v1/Conversion.scala | 20 +++++++++++-------- thehive/app/org/thp/thehive/models/Tag.scala | 19 ++++++++++++++++-- .../app/org/thp/thehive/models/Taxonomy.scala | 2 +- .../thp/thehive/services/TaxonomySrv.scala | 10 ++-------- 6 files changed, 48 insertions(+), 33 deletions(-) create mode 100644 dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala new file mode 100644 index 0000000000..3b536c867c --- /dev/null +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala @@ -0,0 +1,15 @@ +package org.thp.thehive.dto.v1 + +import play.api.libs.json.{Json, OFormat} + +case class OutputTag( + namespace: String, + predicate: String, + value: Option[String], + description: Option[String], + colour: Int +) + +object OutputTag { + implicit val format: OFormat[OutputTag] = Json.format[OutputTag] +} diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 20890915a5..70f0b23208 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -39,22 +39,9 @@ case class OutputTaxonomy( description: String, version: Int, enabled: Boolean, - predicates: Seq[String], - values: Seq[OutputEntry] + tags: Seq[OutputTag] ) -case class OutputEntry(predicate: String, entry: Seq[OutputValue]) - -case class OutputValue(value: String, expanded: String) - object OutputTaxonomy { implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] } - -object OutputEntry { - implicit val format: OFormat[OutputEntry] = Json.format[OutputEntry] -} - -object OutputValue { - implicit val format: OFormat[OutputValue] = Json.format[OutputValue] -} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 78f235c031..05e63d0bdf 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -273,14 +273,18 @@ object Conversion { .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) .withFieldComputed(_.enabled, _.enabled) - .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) - .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { - val outputValues = entryMap.getOrElse(tag.predicate, Seq()) - if (tag.value.isDefined) - entryMap + (tag.predicate -> (outputValues :+ OutputValue(tag.value.get, tag.description.getOrElse("")))) - else - entryMap + (tag.predicate -> outputValues) - }).map(e => OutputEntry(e._1, e._2)).toSeq) + .withFieldComputed(_.tags, _.tags.map(_.toOutput)) + .transform + ) + + implicit val tagOutput: Renderer.Aux[RichTag, OutputTag] = + Renderer.toJson[RichTag, OutputTag]( + _.into[OutputTag] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.predicate, _.predicate) + .withFieldComputed(_.value, _.value) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.colour, _.colour) .transform ) diff --git a/thehive/app/org/thp/thehive/models/Tag.scala b/thehive/app/org/thp/thehive/models/Tag.scala index e188ee45c2..cc97dc317e 100644 --- a/thehive/app/org/thp/thehive/models/Tag.scala +++ b/thehive/app/org/thp/thehive/models/Tag.scala @@ -1,7 +1,9 @@ package org.thp.thehive.models -import org.thp.scalligraph.BuildVertexEntity -import org.thp.scalligraph.models.{DefineIndex, IndexType} +import java.util.Date + +import org.thp.scalligraph.{BuildVertexEntity, EntityId} +import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType} import play.api.Logger import scala.util.Try @@ -54,3 +56,16 @@ object Tag { } } } + +case class RichTag(tag: Tag with Entity) { + def _id: EntityId = tag._id + def _createdBy: String = tag._createdBy + def _updatedBy: Option[String] = tag._updatedBy + def _createdAt: Date = tag._createdAt + def _updatedAt: Option[Date] = tag._updatedAt + def namespace: String = tag.namespace + def predicate: String = tag.predicate + def value: Option[String] = tag.value + def description: Option[String] = tag.description + def colour: Int = tag.colour +} diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index 7a8f9a46c2..a7815963e6 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -18,7 +18,7 @@ case class TaxonomyTag() case class RichTaxonomy( taxonomy: Taxonomy with Entity, - tags: Seq[Tag with Entity] + tags: Seq[RichTag] ) { def _id: EntityId = taxonomy._id def _createdBy: String = taxonomy._createdBy diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 49ee30d3d3..1b29f20081 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -47,7 +47,7 @@ class TaxonomySrv @Inject() ( taxonomy <- createEntity(taxo) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags.map(RichTag))) } yield richTaxonomy def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = @@ -55,12 +55,6 @@ class TaxonomySrv @Inject() ( _ <- get(taxonomyId).update(_.enabled, isEnabled).getOrFail("Taxonomy") } yield () -/* - - def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = - Try(startTraversal.getByNamespace(namespace)).getOrElse(startTraversal.limit(0)) -*/ - } object TaxonomyOps { @@ -86,6 +80,6 @@ object TaxonomyOps { _.by .by(_.tags.fold) ) - .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags) } + .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags.map(RichTag)) } } } From eb91a5b1df0d231e98b7c64f240b19fd84fd5907 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 15:31:29 +0100 Subject: [PATCH 011/324] Query for taxonomies --- thehive/app/org/thp/thehive/controllers/v1/Router.scala | 3 +-- .../org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index f3184dffdb..1d49363c75 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -91,10 +91,9 @@ class Router @Inject() ( // DELETE /alert/:alertId controllers.AlertCtrl.delete(alertId) // POST /alert/:alertId/merge/:caseId controllers.AlertCtrl.mergeWithCase(alertId, caseId) - case GET(p"/taxonomy") => taxonomyCtrl.list - case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case POST(p"/taxonomy") => taxonomyCtrl.create // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip + case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index bbc3b86b81..27fed93b12 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -32,6 +32,7 @@ class TheHiveQueryExecutor @Inject() ( profileCtrl: ProfileCtrl, taskCtrl: TaskCtrl, userCtrl: UserCtrl, + taxonomyCtrl: TaxonomyCtrl, // dashboardCtrl: DashboardCtrl, properties: Properties, @Named("with-thehive-schema") implicit val db: Database @@ -53,7 +54,8 @@ class TheHiveQueryExecutor @Inject() ( profileCtrl, // tagCtrl, taskCtrl, - userCtrl + userCtrl, + taxonomyCtrl ) override val version: (Int, Int) = 1 -> 1 From 858ba48a0b6e206800f1e4b32c029bfc474880c5 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 18 Nov 2020 12:13:01 +0100 Subject: [PATCH 012/324] Basic zip import --- .../org/thp/thehive/dto/v1/Taxonomy.scala | 49 +++++++--- .../thp/thehive/controllers/v1/Router.scala | 2 +- .../thehive/controllers/v1/TaxonomyCtrl.scala | 91 +++++++++++-------- 3 files changed, 90 insertions(+), 52 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 70f0b23208..576683127a 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -2,30 +2,57 @@ package org.thp.thehive.dto.v1 import java.util.Date -import play.api.libs.json.{Json, OFormat, OWrites, Writes} +import play.api.libs.json.{Json, OFormat} -case class InputTaxonomy ( +/* +Format based on : +https://tools.ietf.org/id/draft-dulaunoy-misp-taxonomy-format-04.html +*/ + +case class InputTaxonomy( namespace: String, description: String, version: Int, - predicates: Seq[String], - values: Option[Seq[InputEntry]] + `type`: Option[Seq[String]], + exclusive: Option[Boolean], + predicates: Seq[InputPredicate], + values: Option[Seq[InputValue]] +) + +case class InputPredicate( + value: String, + expanded: Option[String], + exclusive: Option[Boolean], + description: Option[String] ) -case class InputEntry(predicate: String, entry: Seq[InputValue]) +case class InputValue( + predicate: String, + entry: Seq[InputEntry] +) -case class InputValue(value: String, expanded: String, colour: Option[String]) +case class InputEntry( + value: String, + expanded: Option[String], + colour: Option[String], + description: Option[String], + numerical_value: Option[Int] +) -object InputEntry { - implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] +object InputTaxonomy { + implicit val format: OFormat[InputTaxonomy] = Json.format[InputTaxonomy] +} + +object InputPredicate { + implicit val format: OFormat[InputPredicate] = Json.format[InputPredicate] } object InputValue { - implicit val writes: Writes[InputValue] = Json.writes[InputValue] + implicit val format: OFormat[InputValue] = Json.format[InputValue] } -object InputTaxonomy { - implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] +object InputEntry { + implicit val format: OFormat[InputEntry] = Json.format[InputEntry] } case class OutputTaxonomy( diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 1d49363c75..324df8eb8d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -92,7 +92,7 @@ class Router @Inject() ( // POST /alert/:alertId/merge/:caseId controllers.AlertCtrl.mergeWithCase(alertId, caseId) case POST(p"/taxonomy") => taxonomyCtrl.create - // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip + case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 844dbc311b..5946240540 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -1,21 +1,25 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} -import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} -import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} +import net.lingala.zip4j.ZipFile +import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.auth.AuthContext +import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ -import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.scalligraph.traversal.TraversalOps.{TraversalOpsDefs, logger} import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputTaxonomy import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} +import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, Results} -import scala.util.{Failure, Success} +import scala.util.{Failure, Success, Try} class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, @@ -68,36 +72,54 @@ class TaxonomyCtrl @Inject() ( entrypoint("import taxonomy") .extract("taxonomy", FieldsParser[InputTaxonomy]) .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - val inputTaxo: InputTaxonomy = request.body("taxonomy") + for { + richTaxonomy <- createFromInput(request.body("taxonomy")) + } yield Results.Created(richTaxonomy.toJson) + } - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) + def importZip: Action[AnyContent] = + entrypoint("import taxonomies zip") + .extract("file", FieldsParser.file.on("file")) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val file: FFile = request.body("file") + val zipFile = new ZipFile(file.filepath.toString) + zipFile.getFileHeaders.stream.forEach { fileHeader => + val json = Json.parse(zipFile.getInputStream(fileHeader)) + createFromInput(json.as[InputTaxonomy]) + } + + Success(Results.NoContent) + } - // Create tags - val tagValues = inputTaxo.values.getOrElse(Seq()) - val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { - all ++ value.entry.map(e => - Tag(inputTaxo.namespace, - value.predicate, - Some(e.value), - Some(e.expanded), - e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour) - ) - ) - }) + private def createFromInput(inputTaxo: InputTaxonomy)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) - // Create a tag for predicates with no tags associated - val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) - val allTags = tags ++ predicateWithNoTags.map(p => - Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) + // Create tags + val tagValues = inputTaxo.values.getOrElse(Seq()) + val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { + all ++ value.entry.map(e => + Tag(inputTaxo.namespace, + value.predicate, + Some(e.value), + e.expanded, + e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour) ) + ) + }) - if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) - Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) - else - for { - tagsEntities <- allTags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) - } yield Results.Created(richTaxonomy.toJson) + // Create a tag for predicates with no tags associated + val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + val allTags = tags ++ predicateWithNoTags.map(p => + Tag(inputTaxo.namespace, p.value, None, None, tagSrv.defaultColour) + ) + + if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) + Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) + else + for { + tagsEntities <- allTags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield richTaxonomy } def get(taxonomyId: String): Action[AnyContent] = @@ -119,15 +141,4 @@ class TaxonomyCtrl @Inject() ( .map(_ => Results.NoContent) } -/* - def delete(namespace: String): Action[AnyContent] = - entrypoint("delete taxonomy") - .authTransaction(db) { implicit request => implicit graph => - for { - t <- taxonomySrv.getByNamespace(namespace) - - } yield Results.Nocontent - } -*/ - } From 366ef7b0e0ec857f6fc83fd4463926bbd9e94f57 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 18 Nov 2020 17:32:51 +0100 Subject: [PATCH 013/324] Handled zip import errors --- .../thehive/controllers/v1/Conversion.scala | 14 +-------- .../thehive/controllers/v1/Properties.scala | 3 +- .../thehive/controllers/v1/TaxonomyCtrl.scala | 31 +++++++++++++------ .../models/TheHiveSchemaDefinition.scala | 2 +- .../thp/thehive/services/TaxonomySrv.scala | 2 +- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 05e63d0bdf..154f7cb790 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -257,10 +257,7 @@ object Conversion { def toTaxonomy: Taxonomy = inputTaxonomy .into[Taxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldConst(_.enabled, false) // TODO always false when importing a taxonomy ? + .withFieldConst(_.enabled, false) .transform } @@ -269,10 +266,6 @@ object Conversion { _.into[OutputTaxonomy] .withFieldComputed(_._id, _._id.toString) .withFieldConst(_._type, "Taxonomy") - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldComputed(_.enabled, _.enabled) .withFieldComputed(_.tags, _.tags.map(_.toOutput)) .transform ) @@ -280,11 +273,6 @@ object Conversion { implicit val tagOutput: Renderer.Aux[RichTag, OutputTag] = Renderer.toJson[RichTag, OutputTag]( _.into[OutputTag] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.predicate, _.predicate) - .withFieldComputed(_.value, _.value) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.colour, _.colour) .transform ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index a7d78bc02a..f3b2bb997d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -381,8 +381,7 @@ class Properties @Inject() ( .property("namespace", UMapping.string)(_.field.readonly) .property("description", UMapping.string)(_.field.readonly) .property("version", UMapping.int)(_.field.readonly) - // Predicates ? - // Values ? + .property("enabled", UMapping.boolean)(_.field.readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 5946240540..5a48e28d8a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -2,14 +2,15 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} import net.lingala.zip4j.ZipFile +import net.lingala.zip4j.model.FileHeader import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ -import org.thp.scalligraph.traversal.TraversalOps.{TraversalOpsDefs, logger} +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} +import org.thp.scalligraph.{BadRequestError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputTaxonomy import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} @@ -19,6 +20,7 @@ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, Results} +import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} class TaxonomyCtrl @Inject() ( @@ -80,17 +82,28 @@ class TaxonomyCtrl @Inject() ( def importZip: Action[AnyContent] = entrypoint("import taxonomies zip") .extract("file", FieldsParser.file.on("file")) - .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + .authPermitted(Permissions.manageTaxonomy) { implicit request => val file: FFile = request.body("file") val zipFile = new ZipFile(file.filepath.toString) - zipFile.getFileHeaders.stream.forEach { fileHeader => - val json = Json.parse(zipFile.getInputStream(fileHeader)) - createFromInput(json.as[InputTaxonomy]) - } + val headers = zipFile + .getFileHeaders + .iterator() + .asScala - Success(Results.NoContent) + for { + inputTaxos <- headers.toTry(h => parseJsonFile(zipFile, h)) + richTaxos <- db.tryTransaction { implicit graph => + inputTaxos.toTry(inputTaxo => createFromInput(inputTaxo)).map(_.toJson) + } + } yield Results.Created(richTaxos) } + private def parseJsonFile(zipFile: ZipFile, h: FileHeader): Try[InputTaxonomy] = { + Try(Json.parse(zipFile.getInputStream(h)).as[InputTaxonomy]).recoverWith { + case _ => Failure(BadRequestError(s"File '${h.getFileName}' does not comply with the MISP taxonomy formatting")) + } + } + private def createFromInput(inputTaxo: InputTaxonomy)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) @@ -114,7 +127,7 @@ class TaxonomyCtrl @Inject() ( ) if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) - Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) + Failure(BadRequestError(s"A taxonomy with namespace '${inputTaxo.namespace}' already exists in this organisation")) else for { tagsEntities <- allTags.toTry(t => tagSrv.create(t)) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 9edc5c41f1..e109b6ffd7 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -122,7 +122,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { Success(()) } - val reflectionClasses = new Reflections( + val reflectionClasses = new Reflections( new ConfigurationBuilder() .forPackages("org.thp.thehive.models") .addClassLoader(getClass.getClassLoader) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 1b29f20081..062d96b4e4 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -29,7 +29,7 @@ class TaxonomySrv @Inject() ( .has(_.namespace, namespace) .in[OrganisationTaxonomy] .v[Organisation] - .has(_.name, authContext.organisation.toString) // TODO not great + .current .exists } From 2c7e887ea369b43a2d3b000edff53b6b215159bc Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 19 Nov 2020 11:18:59 +0100 Subject: [PATCH 014/324] Review changes --- .../thp/thehive/controllers/v1/Conversion.scala | 4 ++-- .../thp/thehive/controllers/v1/TaxonomyCtrl.scala | 8 +++----- thehive/app/org/thp/thehive/models/Tag.scala | 13 ------------- thehive/app/org/thp/thehive/models/Taxonomy.scala | 2 +- .../thehive/models/TheHiveSchemaDefinition.scala | 14 +++++++------- .../app/org/thp/thehive/services/TaxonomySrv.scala | 4 ++-- 6 files changed, 15 insertions(+), 30 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 154f7cb790..5ef735620f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -270,8 +270,8 @@ object Conversion { .transform ) - implicit val tagOutput: Renderer.Aux[RichTag, OutputTag] = - Renderer.toJson[RichTag, OutputTag]( + implicit val tagOutput: Renderer.Aux[Tag, OutputTag] = + Renderer.toJson[Tag, OutputTag]( _.into[OutputTag] .transform ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 5a48e28d8a..10c91d6974 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -105,12 +105,10 @@ class TaxonomyCtrl @Inject() ( } private def createFromInput(inputTaxo: InputTaxonomy)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) - // Create tags val tagValues = inputTaxo.values.getOrElse(Seq()) - val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { - all ++ value.entry.map(e => + val tags = tagValues.flatMap(value => { + value.entry.map(e => Tag(inputTaxo.namespace, value.predicate, Some(e.value), @@ -131,7 +129,7 @@ class TaxonomyCtrl @Inject() ( else for { tagsEntities <- allTags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + richTaxonomy <- taxonomySrv.create(inputTaxo.toTaxonomy, tagsEntities) } yield richTaxonomy } diff --git a/thehive/app/org/thp/thehive/models/Tag.scala b/thehive/app/org/thp/thehive/models/Tag.scala index cc97dc317e..3ad58979a5 100644 --- a/thehive/app/org/thp/thehive/models/Tag.scala +++ b/thehive/app/org/thp/thehive/models/Tag.scala @@ -56,16 +56,3 @@ object Tag { } } } - -case class RichTag(tag: Tag with Entity) { - def _id: EntityId = tag._id - def _createdBy: String = tag._createdBy - def _updatedBy: Option[String] = tag._updatedBy - def _createdAt: Date = tag._createdAt - def _updatedAt: Option[Date] = tag._updatedAt - def namespace: String = tag.namespace - def predicate: String = tag.predicate - def value: Option[String] = tag.value - def description: Option[String] = tag.description - def colour: Int = tag.colour -} diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index a7815963e6..bc4fb1a6d4 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -18,7 +18,7 @@ case class TaxonomyTag() case class RichTaxonomy( taxonomy: Taxonomy with Entity, - tags: Seq[RichTag] + tags: Seq[Tag] ) { def _id: EntityId = taxonomy._id def _createdBy: String = taxonomy._createdBy diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index e109b6ffd7..100ce9967b 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -80,11 +80,11 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { // Taxonomies .addVertexModel[String]("Taxonomy", Seq("namespace")) .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => - db.tryTransaction { g => - // If there are no taxonomies in database, add a custom one for each organisation - db.labelFilter("Taxonomy")(Traversal.V()(g)).headOption match { - case None => - db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => + db.tryTransaction { implicit g => + // For each organisation, if there is no custom taxonomy, create it + db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => + Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", "custom").headOption match { + case None => val taxoVertex = g.addVertex("Taxonomy") taxoVertex.property("_label", "Taxonomy") taxoVertex.property("_createdBy", "system@thehive.local") @@ -95,8 +95,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("enabled", true) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) - } - case _ => Success(()) + case _ => Success(()) + } } }.map(_ => ()) } diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 062d96b4e4..4b4b7c28a2 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -47,7 +47,7 @@ class TaxonomySrv @Inject() ( taxonomy <- createEntity(taxo) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- Try(RichTaxonomy(taxonomy, tags.map(RichTag))) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = @@ -80,6 +80,6 @@ object TaxonomyOps { _.by .by(_.tags.fold) ) - .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags.map(RichTag)) } + .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags) } } } From 114601067ef9da32209e2806ebd4f6304599ce9e Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 09:46:17 +0100 Subject: [PATCH 015/324] Added (de)activation & deletion --- .../org/thp/thehive/dto/v1/Taxonomy.scala | 1 - .../thehive/controllers/v1/Conversion.scala | 1 - .../thehive/controllers/v1/Properties.scala | 1 - .../thp/thehive/controllers/v1/Router.scala | 5 +-- .../thehive/controllers/v1/TaxonomyCtrl.scala | 31 ++++++++++++++----- .../org/thp/thehive/models/Permissions.scala | 2 +- .../app/org/thp/thehive/models/Taxonomy.scala | 4 +-- .../models/TheHiveSchemaDefinition.scala | 19 ++++++++++-- .../thehive/services/OrganisationSrv.scala | 13 +++++--- .../thp/thehive/services/TaxonomySrv.scala | 27 +++++++++++++--- 10 files changed, 76 insertions(+), 28 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 576683127a..7081347184 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -65,7 +65,6 @@ case class OutputTaxonomy( namespace: String, description: String, version: Int, - enabled: Boolean, tags: Seq[OutputTag] ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 5ef735620f..850521f156 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -257,7 +257,6 @@ object Conversion { def toTaxonomy: Taxonomy = inputTaxonomy .into[Taxonomy] - .withFieldConst(_.enabled, false) .transform } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index f3b2bb997d..e8af8ccd96 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -381,7 +381,6 @@ class Properties @Inject() ( .property("namespace", UMapping.string)(_.field.readonly) .property("description", UMapping.string)(_.field.readonly) .property("version", UMapping.int)(_.field.readonly) - .property("enabled", UMapping.boolean)(_.field.readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 324df8eb8d..f3bd9e882b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -94,8 +94,9 @@ class Router @Inject() ( case POST(p"/taxonomy") => taxonomyCtrl.create case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) - case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) - case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) + case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.toggleActivation(taxoId, isActive = true) + case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.toggleActivation(taxoId, isActive = false) + case DELETE(p"/taxonomy/$taxoId") => taxonomyCtrl.delete(taxoId) case GET(p"/audit") => auditCtrl.flow // GET /flow controllers.AuditCtrl.flow(rootId: Option[String], count: Option[Int]) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 10c91d6974..e81c47a098 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -119,12 +119,16 @@ class TaxonomyCtrl @Inject() ( }) // Create a tag for predicates with no tags associated - val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + val predicateWithNoTags = inputTaxo.predicates.map(_.value).diff(tagValues.map(_.predicate)) val allTags = tags ++ predicateWithNoTags.map(p => - Tag(inputTaxo.namespace, p.value, None, None, tagSrv.defaultColour) + Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) ) - if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) + if (inputTaxo.namespace.isEmpty) + Failure(BadRequestError(s"A taxonomy with no namespace cannot be imported")) + else if (inputTaxo.namespace == "_freetags") + Failure(BadRequestError(s"Namespace _freetags is restricted for TheHive")) + else if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) Failure(BadRequestError(s"A taxonomy with namespace '${inputTaxo.namespace}' already exists in this organisation")) else for { @@ -144,12 +148,25 @@ class TaxonomyCtrl @Inject() ( .map(taxonomy => Results.Ok(taxonomy.toJson)) } - def setEnabled(taxonomyId: String, isEnabled: Boolean): Action[AnyContent] = + def toggleActivation(taxonomyId: String, isActive: Boolean): Action[AnyContent] = entrypoint("toggle taxonomy") .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - taxonomySrv - .setEnabled(EntityIdOrName(taxonomyId), isEnabled) - .map(_ => Results.NoContent) + val toggleF = if (isActive) taxonomySrv.activate _ else taxonomySrv.deactivate _ + toggleF(EntityIdOrName(taxonomyId)).map(_ => Results.NoContent) + } + + def delete(taxoId: String): Action[AnyContent] = + entrypoint("delete taxonomy") + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + for { + taxo <- taxonomySrv + .get(EntityIdOrName(taxoId)) + .visible + .getOrFail("Taxonomy") + tags <- Try(taxonomySrv.get(taxo).tags.toSeq) + _ <- tags.toTry(t => tagSrv.delete(t)) + _ <- taxonomySrv.delete(taxo) + } yield Results.NoContent } } diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index 7c079860de..81ea621740 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -14,7 +14,7 @@ object Permissions extends Perms { lazy val manageAction: PermissionDesc = PermissionDesc("manageAction", "Run Cortex responders ", "organisation") lazy val manageConfig: PermissionDesc = PermissionDesc("manageConfig", "Manage configurations", "organisation", "admin") lazy val manageProfile: PermissionDesc = PermissionDesc("manageProfile", "Manage user profiles", "admin") - lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "organisation", "admin") + lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "admin") lazy val manageTag: PermissionDesc = PermissionDesc("manageTag", "Manage tags", "admin") lazy val manageCustomField: PermissionDesc = PermissionDesc("manageCustomField", "Manage custom fields", "admin") lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index bc4fb1a6d4..e5fcdb0c03 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -9,8 +9,7 @@ import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} case class Taxonomy( namespace: String, description: String, - version: Int, - enabled: Boolean + version: Int ) @BuildEdgeEntity[Taxonomy, Tag] @@ -28,5 +27,4 @@ case class RichTaxonomy( def namespace: String = taxonomy.namespace def description: String = taxonomy.description def version: Int = taxonomy.version - def enabled: Boolean = taxonomy.enabled } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 100ce9967b..246d149474 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -89,7 +89,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("_label", "Taxonomy") taxoVertex.property("_createdBy", "system@thehive.local") taxoVertex.property("_createdAt", new Date()) - taxoVertex.property("namespace", "custom") + taxoVertex.property("namespace", "_freetags") taxoVertex.property("description", "Custom taxonomy") taxoVertex.property("version", 1) taxoVertex.property("enabled", true) @@ -103,14 +103,22 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => db.tryTransaction { implicit g => db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => - val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "custom").head + val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "_freetags").head Traversal.V(EntityId(o.id())).unionFlat( _.out("OrganisationShare").out("ShareCase").out("CaseTag"), _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), _.in("AlertOrganisation").out("AlertTag"), _.in("CaseTemplateOrganisation").out("CaseTemplateTag") ).toSeq.foreach { tag => - tag.property("namespace", "custom") + // Create a freetext tag and store it into predicate + val tagStr = tagString( + tag.property("namespace").value().toString, + tag.property("predicate").value().toString, + tag.property("value").value().toString + ) + tag.property("namespace", "_freetags") + tag.property("predicate", tagStr) + tag.property("value").remove() customTaxo.addEdge("TaxonomyTag", tag) } Success(()) @@ -148,5 +156,10 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { case vertexModel: VertexModel => vertexModel.getInitialValues }.flatten + private def tagString(namespace: String, predicate: String, value: String): String = + (if (namespace.headOption.getOrElse('_') == '_') "" else namespace + ':') + + (if (predicate.headOption.getOrElse('_') == '_') "" else predicate) + + (if (value.isEmpty) "" else f"""="$value"""") + override def init(db: Database)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = Success(()) } diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 05f3889499..0a30d74c74 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -35,6 +35,7 @@ class OrganisationSrv @Inject() ( lazy val taxonomySrv: TaxonomySrv = taxonomySrvProvider.get val organisationOrganisationSrv = new EdgeSrv[OrganisationOrganisation, Organisation, Organisation] val organisationShareSrv = new EdgeSrv[OrganisationShare, Organisation, Share] + val organisationTaxonomySrv = new EdgeSrv[OrganisationTaxonomy, Organisation, Taxonomy] override def createEntity(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("Organisation") @@ -50,12 +51,14 @@ class OrganisationSrv @Inject() ( } yield createdOrganisation def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { - val customTaxo = Taxonomy("custom", "Custom taxonomy", 1, enabled = true) + val customTaxo = Taxonomy("_freetags", "Custom taxonomy", 1) + val activeTaxos = getByName("admin").taxonomies.toSeq for { - createdOrganisation <- createEntity(e) - _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), createdOrganisation) - _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) - } yield createdOrganisation + newOrga <- createEntity(e) + _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), newOrga) + _ <- activeTaxos.toTry(t => organisationTaxonomySrv.create(OrganisationTaxonomy(), newOrga, t)) + _ <- auditSrv.organisation.create(newOrga, newOrga.toJson) + } yield newOrga } def current(implicit graph: Graph, authContext: AuthContext): Traversal.V[Organisation] = get(authContext.organisation) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 4b4b7c28a2..b172b44c6b 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -12,8 +12,9 @@ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.scalligraph.{EntityId, EntityIdOrName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.TaxonomyOps._ -import scala.util.Try +import scala.util.{Success, Try} @Singleton class TaxonomySrv @Inject() ( @@ -50,10 +51,28 @@ class TaxonomySrv @Inject() ( richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy - def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Taxonomy] = + Try(startTraversal.getByNamespace(name)).getOrElse(startTraversal.limit(0)) + + def activate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + for { + taxo <- get(taxonomyId).getOrFail("Taxonomy") + organisations <- Try(organisationSrv.startTraversal.filter(_ + .out[OrganisationTaxonomy] + .filter(_.unsafeHas("namespace", taxo.namespace)) + ).toSeq) + _ <- organisations.toTry(o => organisationTaxonomySrv.create(OrganisationTaxonomy(), o, taxo)) + } yield Success(()) + + def deactivate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { - _ <- get(taxonomyId).update(_.enabled, isEnabled).getOrFail("Taxonomy") - } yield () + taxo <- get(taxonomyId).getOrFail("Taxonomy") + _ <- Try(organisationSrv + .get(authContext.organisation) + .outE[OrganisationTaxonomy] + .filter(_.otherV().unsafeHas("namespace", taxo.namespace)) + .remove()) + } yield Success(()) } From 5e88dba40ab07158a1964e80c44ec2420b412e9b Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 10:26:59 +0100 Subject: [PATCH 016/324] Used correct Scalligraph commit --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index e95b44aafa..1346ea3588 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit e95b44aafa9269a723903204f9a1676fbbfab698 +Subproject commit 1346ea3588009e8393c7f79180af35be01efa097 From bc1b507880bbd51aea2695d1b8d5030464a31890 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 10:39:38 +0100 Subject: [PATCH 017/324] Edit drone.yml --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 16354f6ca6..2b2ebfffeb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,7 +8,7 @@ steps: - name: submodules image: alpine/git commands: - - git submodule update --recursive --init --remote + - git submodule update --recursive --init # Restore cache of downloaded dependencies - name: restore-cache From e4d43cbe29271fdf56a38a0b2aaa351252a3dbde Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 17:07:18 +0100 Subject: [PATCH 018/324] Fixed schema erros & (de)activation --- ScalliGraph | 2 +- .../models/TheHiveSchemaDefinition.scala | 11 +- .../thehive/services/OrganisationSrv.scala | 3 +- .../thp/thehive/services/TaxonomySrv.scala | 24 +-- .../org/thp/thehive/DatabaseBuilder.scala | 5 + .../controllers/v1/TaxonomyCtrlTest.scala | 204 ++++++++++++++++++ .../resources/data/OrganisationTaxonomy.json | 5 + thehive/test/resources/data/Tag.json | 7 + thehive/test/resources/data/Taxonomy.json | 8 + thehive/test/resources/data/TaxonomyTag.json | 3 + .../test/resources/machinetag-badformat.zip | Bin 0 -> 4274 bytes .../test/resources/machinetag-otherfiles.zip | Bin 0 -> 3841 bytes thehive/test/resources/machinetag-present.zip | Bin 0 -> 3941 bytes thehive/test/resources/machinetag.zip | Bin 0 -> 4076 bytes 14 files changed, 252 insertions(+), 20 deletions(-) create mode 100644 thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala create mode 100644 thehive/test/resources/data/OrganisationTaxonomy.json create mode 100644 thehive/test/resources/data/Taxonomy.json create mode 100644 thehive/test/resources/data/TaxonomyTag.json create mode 100644 thehive/test/resources/machinetag-badformat.zip create mode 100644 thehive/test/resources/machinetag-otherfiles.zip create mode 100644 thehive/test/resources/machinetag-present.zip create mode 100644 thehive/test/resources/machinetag.zip diff --git a/ScalliGraph b/ScalliGraph index 1346ea3588..1a55a0db73 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 1346ea3588009e8393c7f79180af35be01efa097 +Subproject commit 1a55a0db730460c6f548695251248934196b6ecc diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 246d149474..a509a89059 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -4,6 +4,7 @@ import java.lang.reflect.Modifier import java.util.Date import javax.inject.{Inject, Singleton} +import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.Graph import org.janusgraph.core.schema.ConsistencyModifier import org.janusgraph.graphdb.types.TypeDefinitionCategory @@ -82,8 +83,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => db.tryTransaction { implicit g => // For each organisation, if there is no custom taxonomy, create it - db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => - Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", "custom").headOption match { + db.labelFilter("Organisation")(Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o => + Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", "_freetags").headOption match { case None => val taxoVertex = g.addVertex("Taxonomy") taxoVertex.property("_label", "Taxonomy") @@ -114,7 +115,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { val tagStr = tagString( tag.property("namespace").value().toString, tag.property("predicate").value().toString, - tag.property("value").value().toString + tag.property ("value").orElse("") ) tag.property("namespace", "_freetags") tag.property("predicate", tagStr) @@ -125,8 +126,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { } }.map(_ => ()) } - .updateGraph("Add manageTaxonomy to org-admin profile", "Profile") { traversal => - Try(traversal.unsafeHas("name", "org-admin").raw.property("permissions", "manageTaxonomy").iterate()) + .updateGraph("Add manageTaxonomy to admin profile", "Profile") { traversal => + Try(traversal.unsafeHas("name", "admin").raw.property("permissions", "manageTaxonomy").iterate()) Success(()) } diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 0a30d74c74..c567696848 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -51,11 +51,10 @@ class OrganisationSrv @Inject() ( } yield createdOrganisation def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { - val customTaxo = Taxonomy("_freetags", "Custom taxonomy", 1) val activeTaxos = getByName("admin").taxonomies.toSeq for { newOrga <- createEntity(e) - _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), newOrga) + _ <- taxonomySrv.createFreetag(newOrga) _ <- activeTaxos.toTry(t => organisationTaxonomySrv.create(OrganisationTaxonomy(), newOrga, t)) _ <- auditSrv.organisation.create(newOrga, newOrga.toJson) } yield newOrga diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index b172b44c6b..2051c64930 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -36,20 +36,20 @@ class TaxonomySrv @Inject() ( def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { - organisation <- organisationSrv.getOrFail(authContext.organisation) - richTaxonomy <- createWithOrg(taxo, tags, organisation) + taxonomy <- createEntity(taxo) + _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) + _ <- activate(richTaxonomy._id) } yield richTaxonomy - def createWithOrg(taxo: Taxonomy, - tags: Seq[Tag with Entity], - organisation: Organisation with Entity) - (implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = + def createFreetag(organisation: Organisation with Entity)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { + val customTaxo = Taxonomy("_freetags", "Custom taxonomy", 1) for { - taxonomy <- createEntity(taxo) + taxonomy <- createEntity(customTaxo) + richTaxonomy <- Try(RichTaxonomy(taxonomy, Seq())) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) - _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy + } override def getByName(name: String)(implicit graph: Graph): Traversal.V[Taxonomy] = Try(startTraversal.getByNamespace(name)).getOrElse(startTraversal.limit(0)) @@ -57,7 +57,7 @@ class TaxonomySrv @Inject() ( def activate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { taxo <- get(taxonomyId).getOrFail("Taxonomy") - organisations <- Try(organisationSrv.startTraversal.filter(_ + organisations <- Try(organisationSrv.startTraversal.filterNot(_ .out[OrganisationTaxonomy] .filter(_.unsafeHas("namespace", taxo.namespace)) ).toSeq) @@ -67,8 +67,8 @@ class TaxonomySrv @Inject() ( def deactivate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { taxo <- get(taxonomyId).getOrFail("Taxonomy") - _ <- Try(organisationSrv - .get(authContext.organisation) + _ <- Try(organisationSrv.startTraversal + .filterNot(_.unsafeHas("name", "admin")) .outE[OrganisationTaxonomy] .filter(_.otherV().unsafeHas("namespace", taxo.namespace)) .remove()) diff --git a/thehive/test/org/thp/thehive/DatabaseBuilder.scala b/thehive/test/org/thp/thehive/DatabaseBuilder.scala index 51767a822f..52094c2147 100644 --- a/thehive/test/org/thp/thehive/DatabaseBuilder.scala +++ b/thehive/test/org/thp/thehive/DatabaseBuilder.scala @@ -35,6 +35,7 @@ class DatabaseBuilder @Inject() ( observableSrv: ObservableSrv, observableTypeSrv: ObservableTypeSrv, taskSrv: TaskSrv, + taxonomySrv: TaxonomySrv, tagSrv: TagSrv, keyValueSrv: KeyValueSrv, dataSrv: DataSrv, @@ -82,11 +83,15 @@ class DatabaseBuilder @Inject() ( createVertex(impactStatusSrv, FieldsParser[ImpactStatus]) ++ createVertex(attachmentSrv, FieldsParser[Attachment]) ++ createVertex(tagSrv, FieldsParser[Tag]) ++ + createVertex(taxonomySrv, FieldsParser[Taxonomy]) ++ createVertex(pageSrv, FieldsParser[Page]) ++ createVertex(dashboardSrv, FieldsParser[Dashboard]) createEdge(organisationSrv.organisationOrganisationSrv, organisationSrv, organisationSrv, FieldsParser[OrganisationOrganisation], idMap) createEdge(organisationSrv.organisationShareSrv, organisationSrv, shareSrv, FieldsParser[OrganisationShare], idMap) + createEdge(organisationSrv.organisationTaxonomySrv, organisationSrv, taxonomySrv, FieldsParser[OrganisationTaxonomy], idMap) + + createEdge(taxonomySrv.taxonomyTagSrv, taxonomySrv, tagSrv, FieldsParser[TaxonomyTag], idMap) createEdge(roleSrv.userRoleSrv, userSrv, roleSrv, FieldsParser[UserRole], idMap) diff --git a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala new file mode 100644 index 0000000000..d08034f2c9 --- /dev/null +++ b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala @@ -0,0 +1,204 @@ +package org.thp.thehive.controllers.v1 + +import org.thp.scalligraph.controllers.FakeTemporaryFile +import org.thp.thehive.TestAppBuilder +import org.thp.thehive.dto.v1.{InputEntry, InputPredicate, InputTaxonomy, InputValue, OutputTag, OutputTaxonomy} +import play.api.libs.Files +import play.api.libs.json.Json +import play.api.mvc.{AnyContentAsMultipartFormData, MultipartFormData} +import play.api.mvc.MultipartFormData.FilePart +import play.api.test.{FakeRequest, PlaySpecification} + +case class TestTaxonomy( + namespace: String, + description: String, + version: Int, + tags: List[OutputTag] +) + +object TestTaxonomy { + def apply(outputTaxonomy: OutputTaxonomy): TestTaxonomy = + TestTaxonomy( + outputTaxonomy.namespace, + outputTaxonomy.description, + outputTaxonomy.version, + outputTaxonomy.tags.toList, + ) +} + +class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { + "taxonomy controller" should { + + val inputTaxo = InputTaxonomy( + "test-taxo", + "A test taxonomy", + 1, + None, + None, + List( + InputPredicate("pred1", None, None, None), + InputPredicate("pred2", None, None, None) + ), + Some(List( + InputValue("pred1", List( + InputEntry("entry1", None, None, None, None)) + ), + InputValue("pred2", List( + InputEntry("entry2", None, None, None, None), + InputEntry("entry21", None, None, None, None) + )) + )) + ) + + "create a valid taxonomy" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(inputTaxo)) + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + + val resultCase = contentAsJson(result).as[OutputTaxonomy] + + TestTaxonomy(resultCase) must_=== TestTaxonomy( + "test-taxo", + "A test taxonomy", + 1, + List( + OutputTag("test-taxo", "pred1", Some("entry1"), None, 0), + OutputTag("test-taxo", "pred2", Some("entry2"), None, 0), + OutputTag("test-taxo", "pred2", Some("entry21"), None, 0) + ) + ) + } + + "return error if not admin" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(inputTaxo)) + .withHeaders("user" -> "certuser@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(403).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("AuthorizationError") + } + + "return error if namespace is present in database" in testApp { app => + val alreadyInDatabase = inputTaxo.copy(namespace = "taxonomy1") + + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(alreadyInDatabase)) + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("already exists") + } + + "return error if namespace is empty" in testApp { app => + val emptyNamespace = inputTaxo.copy(namespace = "") + + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(emptyNamespace)) + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + + } + + "get a taxonomy present" in testApp { app => + val request = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + + val result = app[TaxonomyCtrl].get("taxonomy1")(request) + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + val resultCase = contentAsJson(result).as[OutputTaxonomy] + + TestTaxonomy(resultCase) must_=== TestTaxonomy( + "taxonomy1", + "The taxonomy 1", + 1, + List(OutputTag("taxonomy1", "pred1", Some("value1"), None, 0)) + ) + } + + "return error if taxonomy is not present in database" in testApp { app => + val request = FakeRequest("GET", "/api/v1/taxonomy/taxonomy404") + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].get("taxonomy404")(request) + status(result) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("NotFoundError") + } + + "import zip file correctly" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + + val zipTaxos = contentAsJson(result).as[Seq[OutputTaxonomy]] + zipTaxos.size must beEqualTo(2) + } + + "return error if zip file contains other files than taxonomies" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-otherfiles.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("formatting") + } + + "return error if zip file contains an already present taxonomy" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-present.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("already exists") + } + + "return error if zip file contains a bad formatted taxonomy" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-badformat.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("formatting") + } + + /* + "activate a taxonomy" in testApp { app => + + } + + "deactivate a taxonomy" in testApp { app => + + } + + "delete a taxonomy" in testApp { app => + + } + + */ + } + + def multipartZipFile(name: String): MultipartFormData[Files.TemporaryFile] = MultipartFormData( + // file must be place in test/resources/ + dataParts = Map.empty, + files = Seq(FilePart("file", name, Option("application/zip"), FakeTemporaryFile.fromResource(s"/$name"))), + badParts = Seq() + ) + +} diff --git a/thehive/test/resources/data/OrganisationTaxonomy.json b/thehive/test/resources/data/OrganisationTaxonomy.json new file mode 100644 index 0000000000..df6a1338b2 --- /dev/null +++ b/thehive/test/resources/data/OrganisationTaxonomy.json @@ -0,0 +1,5 @@ +[ + {"from": "admin", "to": "taxonomy1"}, + {"from": "cert", "to": "taxonomy1"}, + {"from": "soc", "to": "taxonomy1"} +] \ No newline at end of file diff --git a/thehive/test/resources/data/Tag.json b/thehive/test/resources/data/Tag.json index c6136decb4..094be1895a 100644 --- a/thehive/test/resources/data/Tag.json +++ b/thehive/test/resources/data/Tag.json @@ -68,5 +68,12 @@ "predicate": "testPredicate", "value": "world", "colour": 0 + }, + { + "id": "taxonomy-tag1", + "namespace": "taxonomy1", + "predicate": "pred1", + "value": "value1", + "colour": 0 } ] \ No newline at end of file diff --git a/thehive/test/resources/data/Taxonomy.json b/thehive/test/resources/data/Taxonomy.json new file mode 100644 index 0000000000..500c39c010 --- /dev/null +++ b/thehive/test/resources/data/Taxonomy.json @@ -0,0 +1,8 @@ +[ + { + "id": "taxonomy1", + "namespace": "taxonomy1", + "description": "The taxonomy 1", + "version": "1" + } +] \ No newline at end of file diff --git a/thehive/test/resources/data/TaxonomyTag.json b/thehive/test/resources/data/TaxonomyTag.json new file mode 100644 index 0000000000..80806c707c --- /dev/null +++ b/thehive/test/resources/data/TaxonomyTag.json @@ -0,0 +1,3 @@ +[ + {"from": "taxonomy1", "to": "taxonomy-tag1"} +] \ No newline at end of file diff --git a/thehive/test/resources/machinetag-badformat.zip b/thehive/test/resources/machinetag-badformat.zip new file mode 100644 index 0000000000000000000000000000000000000000..aae10498e3ed41fec6c29969e841a5ab9785e8d3 GIT binary patch literal 4274 zcmb7{Rag}4wt$BQ3F(&Zly3YbL^_56B&EA%1{jA>lys2pkWMLqA!q0wKuUfXb%GfK{x95#000UA2f)GJ$;B7! zZ4VXmfcW}=OvwR6WZzMCe~R&U1;9uFIQTnw0Koq)d8R!E8xJYbKZKtwkUk{03FKdO_?wmGEPo5+V;MLg7(%#>ptKH=Y_WvtWL6d()Gc+M>E95{ea@7IOv z*k_V3MLmpVOB+O)ja9#TiyO0vq8ITkqOt#OEVJ6br9nU>$VsArf3*g*dEh$a7exme zjBP*2+O%l5UI<=))A%fKUi#SKJe#odI-s=P&!ggCu4WkrcJ4b=S{mf=!e3!kve)sl zu7TPD^_7xi)465=fq8EFF6^bsig5)xLMHiXp*N8|i7WI~eFDCA!|tjsukgU65%M0X z0zI_;!mHx%#+sDVtccK6nMTM69%ON>9%9iuZ?Lk_?D$2~?o+ZeisVbB0${r#6$xHz zvAd0x-GSf4E?#lP&-fq#sM0_%7_OMLUnrY*7ss&dfz$e#C97$6rStH0bb(C5+6bF` z!lfaq^(yZo&Z&>lrymr!lJ_aR4_JyumL9Z6S* zU7mpSp_7GqSk>Z7 zQUCzuZ&<#)S?j4q}=~bFPZ? z^Ehq0)L}jt@XYeu;P&T6fNM^yL#EqW_^5-LPToNyrAd#CRV^G5#T6J@L+yc|0*5|F z<__QXS4~X_kUvJR3QqDbe@s`JrkJtW{i0hXKIXpDZIG;6;7bz?=smrL1G?L>%IiEjN}e-LS| z`;Tn=jH4j}BEr^jNE#;@?AW;hL@+z1>42D?<2&EqkCo{ZgH4#^=)F)yS_tIkfF0^e zEw-okO?f@u4A0ND8LuGBJ(5HL>wA(daDd;YNBjlL^DOOI=ke8e60)XFSuKuS<22}yvx!o<9@x4n z+DX`d2b}k#*OAjlKlyJ99L8M}m?n?eiA8UaY1+-b&l~*@Fr&6BJ>&^@bJvufySw;t z#Gicok=Uc}q&B=}c->ud@){x89YYblU2jKBfGkdhIZ}mN7MGF@^d6XB1 zZlQb<8ZR}LfN<`IFNJG%zg|2H7vtXOTS9*pa}G^#C^yw!Btqk(z%|X1tA0XC7WPcJ z+Vm@AtMec0D{`f6`Qi#8O{=33X06q?8IHJpC94B@Km|Wm=AQ0USU^flw)HFD@CeCh znEHe)hf5kyv>o-6Kw{D*Xz)92h-@(sKW;_3ji>CgL|gzWc6f9#mHTbHRDq$znr51^ zo16Pk#(s-BiyT(+D_!K+qB5NhNi>mo##ch9=sn062eY%(3d>R?FOO{D8->voaJFRe zx$T+eIr9KH|9ZY{xg`M=qgxU)@Ol9qb_IpJ&cVp;i9YU6&#+i&(v9gj<|X92iBi5Q zKzq#GkaUJRe{2(goj>7}RLG;JZcYlWghXAZ^HL^-b22xS+lyQ%%ncw!i3y6r;$xq~ z9i{j&2NLP-d4?R*Se|=WXsDR-{Mp|q5N?(zuwkXRUl?sfdfC0Q6@0>4q$F3G^v_;FPC_VvoiZBBwp%JGw=?sW8nAFUJF^Jt!*tj2!d zD4`y8f91l)iLGS`GGB$!zH)gaM+A>-aAI*(@uqdttYQ%QZzl#_4`>+o53Twv2QnraNN7O`=4J1C8X*#zbb}Uu6~GeX|8Ir?ACM7GmU9u zX1mY6;85ZHgln=H92`wIU&_rlyphYGeaaLiWmkKa14^{!cQjIT&R@OdIHO9Y zYQm+kWZ9HxcQ(TPa7a@un){3!J1Zt?qHmKH5Q;5U(M9&ptw9$DSW5FXaeQHFi3|*z zM3!A?N-|D;r6R82Mm_qykOqerazYYDTMq(qf~^sXq(e*|oA84YCEL06J&ABmCNfEA zJXsJR35o1bfJR8!k=4Vq!ym4?ShLWfDPJk$yft3dd^3mU#o4yh(E|C3)Y=e!M^01f=TfMVNJ0MMd1^Wu&JW6la) z$3|Oj2-e&2PzVqvvQu_Bd5eG;AW)IBH`xW8LEC+ZgN%G<2MUpJV6%xD!}A*QTPUFjbCm=)2B{p2qDIoFDeQ zF$;9=lwnzx?n|sBMg%X*%7aUjn^&KX;r~e#A^}+x`?Z zB`DM)QU2_bJb`j|9ys!&b)06XJgs!&o72t687F6V_M`<{1p~Z{sWLv(=0YgUkIwqD z-oK-O^57(P#oY)awa0nwf*O9&;1_DfOcc80s`4L>O+;%kQ@m-%qgIsCFK($g)hRY9 zpzgaWgNl3f)jkHs;u)@FaDG9 z%x=vH4)L{HT^KO3UgB)pU}uYA-EmLRxNfa1KKd_gHOri$4Yx=HEExMyqukQI=1UY` z{t+rCj)HG-CpNDrn5K1b_F#a-Cfm<4j4@xf{)>5YVAN0T0ZCVTzbFXF7L*Y8h5V7V!!XPq)R!h}%%ks8!=Pa8*TPQy z(5pu2GhH`Fe|RL#j4+Fm;_ADUfVYlK_!068V+-v({KR(S>tKas#T#M5%_=9Q0j%Z7 zHPr~`hF4^9+0+rDqC*m^SloD6PDNHqA(ycx0vt~w>9Gf0TBLi~`0KY82F4mkUok$` zALRGG;w4e(i(_}{P51oiM%D80F}^0On-$$YvFBW)8iybY@F*SDkvHC={H}NMxh&o7 zKwhg$tv#=*Tb7y*kINwe8&YJcqAgCcc=)O0(~2%JiF4MoFR0g=hA{ajZzrVf=CeB3JW{gnL(okAEi0gZ&ad@=Czy0yok7JX;(8=&dKd1cqNR z72*5_3_SFYk2OroBcGi3$aT)}HZwtXiOn!G#W~}HZbM)CW1DA^o*WLGSwB!R>v}(9 zymTP@*<~R>)4AG`oP@;k4>dgTB2wy`g2()UcXeNMLK=LwUtV4P{)HUHip#J@(CAgk zQtPFw+Ga>DR6V-QUk*LDTYu062o6Kao=opR5He0@PdWq*rbeN9GS-v<_R=;KNJuZj zj%n2$^H%WjGLvA7kM1Xl%Z_i1)y*unz5 z4Z1RH-D9<~D-xJHdHRPJtVf-0M$c$)T1wY;4_R6_$;2m9$1uIy^?WR$c~(ZbGc3A# z*tOtPo1L`0?vNbE_ap2=MoHy?FKGH*yidK2R;laCUKrKCU~a(NmvIoHwq;T~EGt`A z!x}f@E5B)utvvfV)zc3386qM}Vjk=t3c|{t4|?-69tMDZ>|LRH449V9j?C7@6XO&? zzpQOy1Gql1z_Z|7_2wc^{+@Sr=zBUWN_7h?WQH4ll_+=Vve0$hLK0mz5b}znUiEsM zs716Mx2`3sqpi^N%h^d53Ids|j%JLJY(Wtf?7Qq&N`1^h7F$^?Qv9@cFj>VA_2jBi zgUIv&pbu1tR^a_o?kZ8CB&5-JMC7Kh+KpfNq9f@zJpM5cfU}D9*_dbHjksjh(cMr? zISrQd=h+P6?zf~75x-NnYBnba0RE_%kuDA{E#7|||DQtl$MOFaP2qn||DEywv-ck* k{O8<1O8Bp63Kzu%{MUkLq)R~frvvWqZ2!Hl-~V&_A2=xanE(I) literal 0 HcmV?d00001 diff --git a/thehive/test/resources/machinetag-otherfiles.zip b/thehive/test/resources/machinetag-otherfiles.zip new file mode 100644 index 0000000000000000000000000000000000000000..cac42ffef199dfb9148db6dd29e41cdd970023df GIT binary patch literal 3841 zcmb7{XE+?%x5h^&N_1gF?=@dz%?dMo9!gM)VRbdW{w_dM{DNh#(H4 z1yM!|Mi*{!o_qfHJonpwt!MAG_Luj4KklEg9uYAkfc$qeI3i2{|64r2Cl!DT;OPT{ zJG*&0Nx%_suqicw%zUrS_MhXApac*QZx8_he_w}>swPc-ntlckVkAw3B*cXVB@GQs zgoex?>4FCziR=DYCIlGkk&#oUkdyNL*8k&S{2zBWB=kS-xiSC%?QeH4I|o-cZzs5& zi-LqZ%*Xrh1TNxe+uOZuw%mPa+y9pUYlV$lk*DwY45Eci>k}ol-|ympZs9MBK5oWH zsSL3*D~!>L!^9JB2+!s4oKo3te7oF>CQFM_jo(I zjv06`c6BaBtrTd-vl|3@@(vK4_zigLKA&BgFRC65_AJt?z3vO7%yv7;Bu?KOAR!}d z!opIysSx{4bzqXo5iNVv_>{oW_I8Y1hXi8uahCowRkVdrb{5pWrr2VAeA|@YJ$!I_ zve{$-W$vCRuDL0u0;M)uDTPn~*Xrxk1$1mLi|jlNLLtkj;mrsLn%ucBfn{N>JJ8EI zMq6|fJ?ih@X4$qKa0S9g_=Dr#!lv{skFosuVc;MrVRFBji?460r_mKt%$=wa!XQ&K z#LFAfjYxp+s(aiq@M(t5q!V^Aj)JPbLtdLJ+awh};PhCjR38edh;kJ5+kobr^f+*P z8zlW@L2zsHF+z*a?7*TY&@}b(#@Cg;+n8aSg>LHj>!}M`kIhZuSn|(4eQ4a?PfBZk zGor4}se78inXLm6J)N(I9)p06`B^ggA1Ae-p2thp#mbLq4oa=DCI;tTmDmVN2STFM zj>Uor9Y3cVh&w`m_z8us4dHu zwGoKTht)3*N0>ELUZpz__7*Mn=YSM^+1a|gQV{;h(V37JK4B5kQ3#Duc`oNvz9?Jz zhXLf2bMT-yIxzV{5OM5+Y%^cUX_2H5Tw-_c_)GQ=Y_S4MBZOg`wu_f{SI%yYK7$%j z^eatl-=aK?2~9SdfEOqtRrLBK5R0(2)DF#1q%MtY6c|P@<#9J=2)J&U<~Z?zxF0-S zw_KBgOEAqz7h+>q_xVW$E?+61<+Zy{TvBv|IDKdq zLL5EhmR87Nrf*0LDu=zgNaLqX4C7|2E433lR+#EXiIbBQgvP}@MLNg`Vs@m`+;WV# z#&LW%aPSZb%X!DQYZZO;7yYc9QB65E}*~$5Ry8P_t@yW2(6G?0uPE=jyj|e?_St1GwzG0YmnR_<>)>BCi|JSVl0ok`-csP{7M%^I1hih>TNv- zn1_iYfzC;w#W-wR*ZRf6!BtkgO7i~0#ICgV+b2z7wGrkx|#5t?XU@Tz}8n zT5V^&{o}x}^ESh+?Om&03sRXcHR57`nAb^iIi7l zfsxe0)nfX4RIJiboIgfB&JKW64#kot{kRvb*g0xPQIpG+u~nIcrq+HL7JTz9gHQxK&l3Vru8J$sqK4w7x2YGZ{%T z%fzfc^f0p;jFJ*gMfW{D4UA9GZ+KA%vs`@t%DJJU-m**IEypyvnT_Kn=bZhm7%P^; zXsiur&=CA`@r|xBsuYc)%+T?A>R$viDn=~HX3wn)1sT`%;V~_tYz$3h6_Vf{yzoH& z&G-wd2?7EwS0CEdaQ?`7@HTC-lCp~(aEG>C>t{P!e9BLwaX-C6u42?Vzi8f+Y;!Wg zZFfLRB8nH!i<^`Xe{5i#>K}qDRMA8CO)bG^`+>!|S_D1_^#m4{RWi%YR3$mb-eNHq z=tq6#t>8NQXLm)UjMr}aX9Yn}ij)Ja?yJb1A|;!tMP|E|d%(g7}CRSO1z4lbE`sNln2&!@Z$9RYQMJ%B=+6adJg@a^$67si|&zKn~qToJCd<)aZI zP2i;Mbo3Gfv%uhD`0&g;?!fh4mv%XixX7y@^5NmxHa9xhiI^(-Gsh9@nkxeDo9ocl zz|i-zn(o|QNEVLe666}#0ekMAWez&O2a&dRa2?ivU0cPE_fa!t8vkhhdV!D=kuxr# zJ0O&4S)*`~7(KG^BJ~k7w!g2@d^eEqu8|7U#PW63tEHPrk@)I|pUBv{Ve$~_a5l54 zFuW~A@%l0Oe zlcR#>B&e%&) zv<{ZvHn8p&1984q^k}|wwyNOn$Y_){8%=l%5xt^}L1AOrp>E;hJo?_V61b#$Z?ziO zru^rxN=zN3|BaH7_3`(_6Nkqd-tVby;2>jlUMZRdifLG9n|$S`1&Leg=5w6Qgaiv7 zXkvrm76n)}U1d#44c#YVQsiA}ukfz!yXp;HQEAOuJhv#L4Khc!LB-`qepyI?>RH=w z7pI}7+m{AJOVLoeLGlNr_uafNWZ*fOku6?w4kIkii}Ft2Jp~W2Z@aW2xFnaZ^bnxP zS}FXv;l>)vvcs05NzGD8T+{r7wE&|A6kwcpGe0$!c#^h+xX!Z$UNX#!djOFTF#F$$if3eD$(KD; zS*tWovGC(=DIHZY&E51RF%UM9qT3F9YLVt??Pt)G9}r^_b#_b5;FF-&89#+eZ!D){ zPnyTMD_!FqHDWD97khil=$2EJYAlM%-@SNH_nyfb?N9xKuO(@&JNL9Z)mw5Zy5#AZ zhBzQ#fTj`PTMQ%P2YzDqualY zx$_}Mw#&CW0YRZ?`GfHd7)s6&|FBiq@XIhY*`oGFt3Hx=2?Xs zrFX1EAl9oe_d?Aya9t|M$KnG(6zT&*+PW)-keuZ;-(=X3;|TTlGVDyZaf{K;EQ@FB z;OiTlfebrd4&xaJ8jF`UcY#f-RFY#UBbc7`S^;24j+Jrt1W->Ow-l6Oy^)&J6`bYp zZirLFII%R~8AG45*P)m3B7IHCGvnH4Y;~BMa`qzhHmvHqB_+%1IFpaW%Ha($<@ob2 z-7RqM0WzvYwoiQnfw+4|pS%Qb?fQdHw$9qR4O!>S_RN+g6Jiy?zaZ8z{yd+7$P6S; zt-094zouR6dmjyo(_Mk`*^oxxrAnPUE%Y4L&}64|r2LX?FM8Y$)FYbqo0bwZ+O67| z=Q9&66olTgJDA;yHIgcZ4I|U~LEdl?Mxl3e z*^6ZP(y)&vLt>Ztm9B!y$E}I`VR34F0PYIPCnFyDmy*&Id)EWer3^UAbNmGA`fq4^ zqhZxxP7MJ3K{I1L0zyWj|AWSV1Kl5J{HN-}{%rpZLjGI%4-o#_^$!sKQ}tn?zt{g3 W4~_LmNdG7h{_dvVQ-wf^0JP`2! literal 0 HcmV?d00001 diff --git a/thehive/test/resources/machinetag-present.zip b/thehive/test/resources/machinetag-present.zip new file mode 100644 index 0000000000000000000000000000000000000000..07a6812f6540fd60c1a27c077326f18742418161 GIT binary patch literal 3941 zcmb7{Wmpv4w#NsgOBxC3PDv$ILSA6VVE{?#9AajWaR@=$K~fr|Q(A`ZhG9@ac?Br} ziJ=6BM!0gHd(L^E`|bSKvvzz~d++uCw0}l=w}==4WPidP2{i`%pYmh@09pVy0Alax z=I-qbw|7;P^nm$zgH5Rcq^=TZySx1zc02=UJAVWvfPiS@769Lru6U{XA?w9h0(cuBBs!X=qEO zW~Lkmwqtq0;%31|VB^CTN*-Ibr^0q)nw^){9l(2GFf4oV#aCrn`x13pYK|)L8f*_Fjd08@9^xR zclzOEdE&x%&BO59ZLFmwBFbA^ug^1m+2}8Ao%0(6tGmKUx3Udkk~*r z?^B<7jX~vX1xk0WnZD24#qE>6Cu2|dV)RO(b^?cffEVBJtrNdtZ{3%(EAz!Q6G2|Z zdUe+W0hBrJCs{=4d&9(}BrRB2Di0N6-?;%y{AFCr0W~=-c(lD8E8iuFm@vr(zEDG3 z2zI|ymUO<_9BuQv)imO7Y4Ohyn$=TN$8`K4L>@JJ#z4Sw@ zmr-Mz5!Prjmw`lpE!6;tfJ=yAe8Q!~UX3Ylgkz<1Rn{+Rt)rp^~<>|#7QRb!WeHg}G3Dty@4 zM7azIwXTeI67$=D=AQI9@_6ef|7}5VXVV0s#cy_C(HCHvdP(thW#BGm%yyxdI^lZy zg4T0$lPHetqt5^uxA%+EM!@V=Pxtf#&7iFI;fTKO*P|vNcBg`D*@6#Ww4h$cOEx7c zCNu|SHdtf*b4V37!qSoOR%%yy!Y_dEaz1e)rkJt2$^Vv#)Ja5ODcb~9sQH%WR!F8z&AQDg(479+=5&yf9YwllrUU!cklRf&QENK zB1^M1!z67FAK$LL{Th8HHKO=;n)tp&MH&;DbRrQiSWKb>84`>`*jZ|aWGYdYMK%kL zA(--cnllC6woG%K`9VC7o~>K1Ny8U{9gD43y z;=+*l*k?#bSs~1hbeemvA@?MXp8^LDmQ?wM|GNWBm?;5jSST70>olS~?O9k0Jm4tQ z*;#uq=I;Lxb&U^5_VRx>WTe*Ih8$nc`!-FDWvuN>xZg=!Qwn0kVB~p=1Da}A4Jf8d zy3gL1`-RMIxU##8ZYJcBN9YTGHXzzso8M8lF2Z{h4a!dX;^~^Cc1Sq;kH5=(WUc&^ zPu=^|7DRTXiz1wdzgqRSnFGwjB#`Vb$)Lq}Y)8-f#lpc=c7kfk{*$Dhw9dOHEfZPa zI{D8zjD3I7!aZsqOrZA82+mHdX@ zodL}9FUmP|AKC%yQv8xAuSf$TsYPnUfqPV}GSOT=$3M&tgWn#CCx7wdS+L^ZteZf6 zS+0t!&MGps@yoQ}e~{@{-}TsEH1@d9+;D3{aIZu`jK45CjP6@PKBt0;K}69j(z&TS z)s1gW?R_>GgkMiIRtIyXAc<$0nAJz0WYvIC(juwofoG=y32%W-p+zvu#rIJzO_hz7 zJwW$d)0kE^&f8pb4tHX$SPo;bwxAIM@TS4)2lC9n7-m?4Sdz<`TNe&A zY8b#{+CtbETFNV>zUXrIp z;Db<4WMNq)wd_t+mUrqe5qE`t05Wd{H8{MuFDh-ccGo{U&>E#gIn3&@irgtyww+$y zk`CixrILZiQw0E$(CBtWc!aDSRUI-b?B1e_H9J!$ZKw*tYvY+m63UXfT+O2ut|O{y z@uI*VnjEpsGRBQu#%}A|9gb12p1G`AFbH;W%QixVp8S3}4KZ*6Gzj$pifrEkKo&(A z$Jg8#a}M}2F4}TMq|T0?Mwld#i?-VdA`WJO!Nu|6S@}Ex>;0~s@}BXLSAk?>W3wIZ zbg&a~HS|Z$Bi1!H1l~8#v7?D$;8zXZxr4AA9Lp`lJ-h?<+C9q}ad{6SY474bZ2Y>m zik}>yX38@9+5YtcAulR#R7!V1DBHG1?kYKPWC0=Z5jS#psL^^qfd9UsD$~^Rb#>Iz zZKP;I&66QgwqBTmHFY?f*>o6S&Mc27t$+L}xw~%4pNI8#Wa!ZbYqGmZi1N}g=h#zj zdMPEvnyDDvJ{{YA#~srH!<|k=#0(OwaOM7t^5ADiJyl8EcwRHzstQ#6T!wMzzetas zB~agwFtu)o%$q2Oy1b7+KrGI72 zfzg=n;RiC`y`h2g6C`%WU5cS~umW}g4Zj(P3bbM-3S4s3gm%X#qP5v*!rN}qE6M5? zHJ2ai7MbMJ_n(!*r9Ap;9)oQwe*Lb(G*}P5Q8u(W{*iRzXrkf$o{9oz{iy-+R-j{)d_w?RZZ|aFoYt`bt zLm8u=HL(pUsW|e>Mhe!<+I_z`4Kdxm)W5Y91Em`wdqnck-TOioo|_ff29b9hXK`6n zaQ@*Xbb!t1(Td=fTDsChfFkRp@skD{Yb?u-TS~^YOQrGAk8oA&(@Hjc;t_~IT)k$Q zrG54HD8anFj_f!Z!P$-2+`>SHmZ2{@gXA_@zLp_(@)YX6n>Rg=I@cMLQ4xY~YNXIS zZ%Zom$jo%+Wh#!!PheR=MQgG$d3r%Bb6-e(0JJ1ZE5F!;m@d(SNFG zkiYbMW}|L6^n)B;&&|;f8Obmu#;&Zi_$I~wwPPbu#DkgfnKphQGCS-tR54lUQjBD^ z(n)y`XE}O7H_Ef(6|KYd9YgiRvvwFjSCq9-U- zk2Q`yyYpCoNCDrR_P~p|pK2Tuj#Onx;l#6dlyf)~wCfiiYnYl#{b{0}_lV_nMuNf|r(s5lbNYL|hW@n2Hcw?dxgB^i zPden6f#)$^y0Go6(jbuObWL$~LSotbYW{d}S&dbZeWB;qwcmAv8obwEo}K;qh3v&j z$#X<704o*ffoW>C=`u5w53lm(gOBW%?{)(ML(mEblN&IUyc7OOyNJQ(F*s1(n%3W5 z&V~jJ>qFVGF1ll0i#(oZ6={~)u@bemS%rBNX{Lc2Qb9fz^#C!b4-9GNp%hGfmfw1t zVMCrX#2;eNoo?$MtDRMzz}Ce-FfwC3=5#rRXCP=US=!uXZ&{_1`t)`j)3;tH$R3<) zWt20;t_Q>|1-`Y}NX_jD%65D=$|Y))RQCJ@!+;Cq5Ms1QUt9XZsO|+@1Ln58gDAZ% ztNLzf>9RV`_ydtjcvEZz{`_-q8{B)Clq!jBXka)1_uyyLy`;i@)iR7&_7k~2iH&=)LzeXhJu0REBNW&k}WiH(odX8&o($fYK z0jZAAK92+Sh?f19r9_QRt4`+mtRxFX;S3H(vpbQT0TJa~n_Oqg{cHgiYnkk_LX6i4 z1*IU3$rn~=(Jd(wVb{9}FqPbKBkanFKF zDVfT>>*1I(1{~!%ehPIBFw!F>d!u31WKInL+@P7!zoBj23Gr`eqa^?UsQy5kFVq?4 z><#}9u&w;+u)9gzpZtHo28pky>F1`>4Cv}R$caQRey$!1jVa-k4-GYaQ+OFz&yZ|J z5l(!v8zfEv=wubqb|m)+9SnMPd!?%0QNgHbba45VEAl119O-u2{gMuQiM9)^@W#P|9R@NOng4Nmub%%;nG(<}3l01v>| z(b>b(&lTb5t|;XNf8+--rvZ>bCD9Hy``R6N`_T@8NGbpU@g@-f@SjVxX`5f)E$aDS zEKwUSn{zb}GC*4sMmnU{ML=|u$Y$~&`Q`qnVt=2Z`)rXbHxq*<(Et$^?@wNX@4{ZB z1SQyvG`k2?!k7Xg(}9(CKE#76ZAnEG;?1fLMgr`!NEvUIqW!&&T2A#j zb_cw(yjS@H_))N$LulA!OS2GTP~+h%MD%T{15mp3}9? z9gb67;u^;~DqPZgUM?_(w;f()s2+k5_nxOhq^kmgWG_J^MsoQRy_U5`Rdban{rP6b zUOzXF2}5t@?yjYn)gs+QPQxG{{vo2D0YiTJPv%w^N@~Z$d`b-Juls|jay@@$6KCuX zk&uyoz{1mbsgVb+4G@y)F&!t=x~VXLV6CDC5}FZ5zrOX=vEXIP43p8#J0HJ z6Xa_brz^gN9uEv`w`t#bdLDcPnO$ zFx1=v`HWJwIRxOp=9O>)e3+#-?TTGWprCH-RM6$gHBCbdxtghzgJ96A7#Hz?O<3N~ zUT0oE!<4_Q2ySedA$0^S4y}5F%+oF@Kd<)R#EjZ6_Ru6=&s@-ZZ*39BlYe~FkH+o) zpt2LRAnNX#QPd90ZX1f~?Rq(41_rtm=ExPkpVoo-oGjaws+!RrmfK-X4bOe6u~9b8 zghXkb6^TCpA}a+XNz5ZU5dQ#xZfbY`;H>7_d5leY`L}hSqS+IHJo*!3k8mLw?I+sv z;1K>>PsD4szMb5Pl;U6QoA3B6EIgqzlp~NAhwslsihCD9sF@B*{sNA`;>rhB(U!V|HcI zJ@brtCUF9kI7GOV>KFW9?GVB&NpQnrald$n3Ds%$;(F*IXOZ6Sy5gv3pa$w1AC%$~ z_-4>Vy}31HY$gB83=NjKt~c>c2T5%im;-~6=PL*nxuD0x%y7N-u}p5HBms*^W7dyex;8hTtGZq z^Rt@=EWjm0fNm+^r37qy_r}HI;Z;teTI#|5hK0?xyC)kn78Uojuz=D^iw9WeiKwu}CX1--jvzW513dR{7uxPJImrPk5(~H#RE?sFpOMW^-Kc4NZSMGJi%I0=cwksOf1j+-B+R3%91KMw@PsXP$X%3kMe+_q@}M zI9s-(SgbvG*a-4$>6N}JsvM1?%F^?F7+3-~E=4Zi&Yj;72{mcx$75O}IG8?ER7pd6 z@uEY8bcq+#Qv?J$9*^kPo(o1VKz8U;RaD&_fxGk_IzKwlk~0BX%?BA(@>Sz*g(VB- zWLwiwo_j+&QZf8^e%!Q_q?w^zT3|S?Sj_<4KeLRO8vvH(>kvFbY9_Im#dD*FD5ed{mfMhhf zO$iYt=RjQ_k{x+#$qfo*>7akAO7PNj_O7IgR33NpNTvI*ntFm5=$kfYT(hicBe$u? z#!kEQi)Rnr)~uL>I(g(8VZ!%+J(=+}asf05_X3LTUjx8a#hEA9JQzz(#0oCPW>vJ_ zL4a0-G>MzO%f(j$!Uji3;Gbs~@CI%4xp&BWCq!R`l8=tgwRTte~^r$2d)66 zdCp-m6A75N$mdP(8?&bH)KC3ASYLal0e!eO&_hy;kAXGUk!s6FIXTWk)rjtNbngvM zY&RTtIu#W=K(NZ4_x**J04vH`HEHv4?M#arNa<5K#;NZjBW9AYQ}oNQ_oc;S*G_rh z!tK7KN^(@_f`THfG^J_j0hTb#moP`yPd0qWTRr@`sCnSvqVpYQV#EGi)SRSHmqPWk zTgrI5r|ZDJKeO{BTjf!uhtP~cMh=#S+tr&HW*Z9dF=fd3%$y6SwcN+|XT5nvix414 z>WaS zg^*y;8%=CD+NuPvWvH$zt78}>CJnhG>-)UB=ZQVmbP3pqqyc66(+yz>~F+meFoHy`0cY-YDk6p!@s zl>rhQT`z;5G}>HeTXEi2HmzGOONhCPs|L;}+wn_8AwzNRw99QAYreh^%HMC#iKi8s z+l4W*Ba+x}r~INNrJhwm(TN41Ylo4TZNwv{tTC^?Q*EPy zsP)BjyY!}epL;Mg-?~SvgY4$)XdT~ntx=ChQ3raJ4(ltLuG9Yj9eyrL_t;g`?b2+` ztLj!@WFg{nO2mZ~*{JDBQ_SssEcv)-Ku+PBW&O4NnT~O60Zhnlbl24jFYbA&bwoH^ zohglz$k|I+NJQo8TvPCGsR;>Bz3}2b4Hin^*=Tto$0~`Wyz{ z3rL7FPRpa27=Ooi%=R)fQDL6TI5X8XIo0g{K(2Rh!iqea7eQ4Z`&o|u=S z_ZHYin`L)x#h`X;aIa$RbVx%Q_>t8+fH>?CJjB6EIh^FIpoNZUQ=Tg#(ATId!`?Ga zH@hN{qf?-NcosV9ayg1;B4{pM-r56xSfiGncs+*c-KZA=hUeLu5c^yr5+CVBO-Tt)K>rgZ5!@-B; zB&`nH4%UV2WGf|+OipKu8_`@rQ5D=<+-E9%96?s=SwJ~q=4+&aa+p?1)u>T)`T*Du zA;v87W-`+l~Jzbbs9VzpF9w&*}eb%KuFNBZdEb`G2Ro d{+7tU)BkUZXktJ@`e!P_-`)Ou=b-y@`w!^JiR1tP literal 0 HcmV?d00001 From a6d7dfc4f4638fcaac54c6f8649c758b21fbd2fe Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 17:19:23 +0100 Subject: [PATCH 019/324] Fixed schema --- .../app/org/thp/thehive/models/TheHiveSchemaDefinition.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index a509a89059..5a6ffb2a02 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -103,7 +103,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { } .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => db.tryTransaction { implicit g => - db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => + db.labelFilter("Organisation")(Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o => val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "_freetags").head Traversal.V(EntityId(o.id())).unionFlat( _.out("OrganisationShare").out("ShareCase").out("CaseTag"), From 09f0dc7adb490721b18e30cb92a623d2712bb155 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 25 Nov 2020 16:10:54 +0100 Subject: [PATCH 020/324] Fixed unit test for taxonomy --- .../controllers/v1/TaxonomyCtrlTest.scala | 93 +++++++++++++------ thehive/test/resources/data/Taxonomy.json | 8 +- 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala index d08034f2c9..6c320635ac 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala @@ -2,11 +2,11 @@ package org.thp.thehive.controllers.v1 import org.thp.scalligraph.controllers.FakeTemporaryFile import org.thp.thehive.TestAppBuilder -import org.thp.thehive.dto.v1.{InputEntry, InputPredicate, InputTaxonomy, InputValue, OutputTag, OutputTaxonomy} +import org.thp.thehive.dto.v1._ import play.api.libs.Files import play.api.libs.json.Json -import play.api.mvc.{AnyContentAsMultipartFormData, MultipartFormData} import play.api.mvc.MultipartFormData.FilePart +import play.api.mvc.{AnyContentAsMultipartFormData, MultipartFormData} import play.api.test.{FakeRequest, PlaySpecification} case class TestTaxonomy( @@ -22,7 +22,7 @@ object TestTaxonomy { outputTaxonomy.namespace, outputTaxonomy.description, outputTaxonomy.version, - outputTaxonomy.tags.toList, + outputTaxonomy.tags.toList ) } @@ -39,15 +39,18 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { InputPredicate("pred1", None, None, None), InputPredicate("pred2", None, None, None) ), - Some(List( - InputValue("pred1", List( - InputEntry("entry1", None, None, None, None)) - ), - InputValue("pred2", List( - InputEntry("entry2", None, None, None, None), - InputEntry("entry21", None, None, None, None) - )) - )) + Some( + List( + InputValue("pred1", List(InputEntry("entry1", None, None, None, None))), + InputValue( + "pred2", + List( + InputEntry("entry2", None, None, None, None), + InputEntry("entry21", None, None, None, None) + ) + ) + ) + ) ) "create a valid taxonomy" in testApp { app => @@ -113,7 +116,7 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { .withHeaders("user" -> "certuser@thehive.local") val result = app[TaxonomyCtrl].get("taxonomy1")(request) - status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + status(result) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") val resultCase = contentAsJson(result).as[OutputTaxonomy] TestTaxonomy(resultCase) must_=== TestTaxonomy( @@ -178,27 +181,65 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { (contentAsJson(result) \ "message").as[String] must contain("formatting") } - /* - "activate a taxonomy" in testApp { app => + "activate a taxonomy" in testApp { app => + val request1 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy2") + .withHeaders("user" -> "certuser@thehive.local") + val result1 = app[TaxonomyCtrl].get("taxonomy2")(request1) + status(result1) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result1)}") + + val request2 = FakeRequest("PUT", "/api/v1/taxonomy/taxonomy2") + .withHeaders("user" -> "admin@thehive.local") + val result2 = app[TaxonomyCtrl].toggleActivation("taxonomy2", isActive = true)(request2) + status(result2) must beEqualTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") + + val request3 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy2") + .withHeaders("user" -> "certuser@thehive.local") + val result3 = app[TaxonomyCtrl].get("taxonomy2")(request3) + status(result3) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result3)}") + } + + "deactivate a taxonomy" in testApp { app => + val request1 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result1 = app[TaxonomyCtrl].get("taxonomy1")(request1) + status(result1) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result1)}") - } + val request2 = FakeRequest("PUT", "/api/v1/taxonomy/taxonomy1/deactivate") + .withHeaders("user" -> "admin@thehive.local") + val result2 = app[TaxonomyCtrl].toggleActivation("taxonomy1", isActive = false)(request2) + status(result2) must beEqualTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") - "deactivate a taxonomy" in testApp { app => + val request3 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result3 = app[TaxonomyCtrl].get("taxonomy1")(request3) + status(result3) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result3)}") + } - } + "delete a taxonomy" in testApp { app => + val request1 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result1 = app[TaxonomyCtrl].get("taxonomy1")(request1) + status(result1) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result1)}") - "delete a taxonomy" in testApp { app => + val request2 = FakeRequest("DELETE", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "admin@thehive.local") + val result2 = app[TaxonomyCtrl].delete("taxonomy1")(request2) + status(result2) must beEqualTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") - } + val request3 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result3 = app[TaxonomyCtrl].get("taxonomy1")(request3) + status(result3) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result3)}") + } - */ } - def multipartZipFile(name: String): MultipartFormData[Files.TemporaryFile] = MultipartFormData( + def multipartZipFile(name: String): MultipartFormData[Files.TemporaryFile] = // file must be place in test/resources/ - dataParts = Map.empty, - files = Seq(FilePart("file", name, Option("application/zip"), FakeTemporaryFile.fromResource(s"/$name"))), - badParts = Seq() - ) + MultipartFormData( + dataParts = Map.empty, + files = Seq(FilePart("file", name, Option("application/zip"), FakeTemporaryFile.fromResource(s"/$name"))), + badParts = Seq() + ) } diff --git a/thehive/test/resources/data/Taxonomy.json b/thehive/test/resources/data/Taxonomy.json index 500c39c010..5c661448dc 100644 --- a/thehive/test/resources/data/Taxonomy.json +++ b/thehive/test/resources/data/Taxonomy.json @@ -3,6 +3,12 @@ "id": "taxonomy1", "namespace": "taxonomy1", "description": "The taxonomy 1", - "version": "1" + "version": 1 + }, + { + "id": "taxonomy2", + "namespace": "taxonomy2", + "description": "The taxonomy 2", + "version": 1 } ] \ No newline at end of file From 6f8c4fe641d647c7a942e7f77e1c118a5d4877f7 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 25 Nov 2020 16:18:57 +0100 Subject: [PATCH 021/324] Fixed user permission test --- thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala index dd68b7d3a9..e7ac8f762c 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala @@ -109,7 +109,6 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { Permissions.managePage, Permissions.manageObservable, Permissions.manageAlert, - Permissions.manageTaxonomy, Permissions.manageAction, Permissions.manageConfig ), From 471b8947abbc8b1d0486f250a1ba16e86b9adc62 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 8 Dec 2020 11:37:36 +0100 Subject: [PATCH 022/324] Review changes --- .../thp/thehive/controllers/v1/DescribeCtrl.scala | 4 +++- .../thehive/models/TheHiveSchemaDefinition.scala | 1 - .../org/thp/thehive/services/TaxonomySrv.scala | 15 +++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index 8db4d8e430..8afb150c0d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -39,6 +39,7 @@ class DescribeCtrl @Inject() ( // pageCtrl: PageCtrl, profileCtrl: ProfileCtrl, taskCtrl: TaskCtrl, + taxonomyCtrl: TaxonomyCtrl, userCtrl: UserCtrl, customFieldSrv: CustomFieldSrv, impactStatusSrv: ImpactStatusSrv, @@ -100,7 +101,8 @@ class DescribeCtrl @Inject() ( EntityDescription("customField", customFieldCtrl.publicProperties.list.flatMap(propertyToJson("customField", _))), EntityDescription("observableType", observableTypeCtrl.publicProperties.list.flatMap(propertyToJson("observableType", _))), EntityDescription("organisation", organisationCtrl.publicProperties.list.flatMap(propertyToJson("organisation", _))), - EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))) + EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))), + EntityDescription("taxonomy", taxonomyCtrl.publicProperties.list.flatMap(propertyToJson("taxonomy", _))) // EntityDescription("dashboard", dashboardCtrl.publicProperties.list.flatMap(propertyToJson("dashboard", _))), // EntityDescription("page", pageCtrl.publicProperties.list.flatMap(propertyToJson("page", _))) ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 5a6ffb2a02..8f03bb4186 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -93,7 +93,6 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("namespace", "_freetags") taxoVertex.property("description", "Custom taxonomy") taxoVertex.property("version", 1) - taxoVertex.property("enabled", true) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) case _ => Success(()) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 2051c64930..aab26143cf 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -27,9 +27,8 @@ class TaxonomySrv @Inject() ( def existsInOrganisation(namespace: String)(implicit graph: Graph, authContext: AuthContext): Boolean = { startTraversal - .has(_.namespace, namespace) - .in[OrganisationTaxonomy] - .v[Organisation] + .getByNamespace(namespace) + .organisations .current .exists } @@ -39,7 +38,6 @@ class TaxonomySrv @Inject() ( taxonomy <- createEntity(taxo) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) - _ <- activate(richTaxonomy._id) } yield richTaxonomy def createFreetag(organisation: Organisation with Entity)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { @@ -59,7 +57,8 @@ class TaxonomySrv @Inject() ( taxo <- get(taxonomyId).getOrFail("Taxonomy") organisations <- Try(organisationSrv.startTraversal.filterNot(_ .out[OrganisationTaxonomy] - .filter(_.unsafeHas("namespace", taxo.namespace)) + .v[Taxonomy] + .has(_.namespace, taxo.namespace) ).toSeq) _ <- organisations.toTry(o => organisationTaxonomySrv.create(OrganisationTaxonomy(), o, taxo)) } yield Success(()) @@ -68,9 +67,9 @@ class TaxonomySrv @Inject() ( for { taxo <- get(taxonomyId).getOrFail("Taxonomy") _ <- Try(organisationSrv.startTraversal - .filterNot(_.unsafeHas("name", "admin")) + .hasNot(_.name, "admin") .outE[OrganisationTaxonomy] - .filter(_.otherV().unsafeHas("namespace", taxo.namespace)) + .filter(_.otherV.v[Taxonomy].has(_.namespace, taxo.namespace)) .remove()) } yield Success(()) @@ -80,7 +79,7 @@ object TaxonomyOps { implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { def get(idOrName: EntityId): Traversal.V[Taxonomy] = - traversal.getByIds(idOrName) + idOrName.fold(traversal.getByIds(_), getByNamespace) def getByNamespace(namespace: String): Traversal.V[Taxonomy] = traversal.has(_.namespace, namespace) From 673ded1d7039cb58861af421a5674a3f2017107f Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 11 Nov 2020 10:50:39 +0100 Subject: [PATCH 023/324] Added taxonomy to database's schema --- .../org/thp/thehive/models/Organisation.scala | 3 ++ .../models/TheHiveSchemaDefinition.scala | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/thehive/app/org/thp/thehive/models/Organisation.scala b/thehive/app/org/thp/thehive/models/Organisation.scala index 41ca8dd5c2..b7ad03b1fc 100644 --- a/thehive/app/org/thp/thehive/models/Organisation.scala +++ b/thehive/app/org/thp/thehive/models/Organisation.scala @@ -20,6 +20,9 @@ case class OrganisationShare() @BuildEdgeEntity[Organisation, Organisation] case class OrganisationOrganisation() +@BuildEdgeEntity[Organisation, Taxonomy] +case class OrganisationTaxonomy() + case class RichOrganisation(organisation: Organisation with Entity, links: Seq[Organisation with Entity]) { def name: String = organisation.name def description: String = organisation.description diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index eeab7f15fd..c9ece8d6fc 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -11,9 +11,11 @@ import org.janusgraph.graphdb.types.TypeDefinitionCategory import org.reflections.Reflections import org.reflections.scanners.SubTypesScanner import org.reflections.util.ConfigurationBuilder +import org.thp.scalligraph.{EntityId, RichSeq} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.janus.JanusDatabase import org.thp.scalligraph.models._ +import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import play.api.Logger @@ -84,6 +86,34 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .iterate() Success(()) } + // Taxonomies + .addVertexModel[String]("Taxonomy", Seq("namespace")) + .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => + db.tryTransaction { g => + db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => + val taxoVertex = g.addVertex("Taxonomy") + taxoVertex.property("namespace", "Custom") + o.addEdge("OrganisationTaxonomy", taxoVertex) + Success(()) + } + }.map(_ => ()) + } + .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => + db.tryTransaction { implicit g => + db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => + val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "Custom").head + Traversal.V(EntityId(o.id())).unionFlat( + _.out("OrganisationShare").out("ShareCase").out("CaseTag"), + _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), + _.in("AlertOrganisation").out("AlertTag"), + _.in("CaseTemplateOrganisation").out("CaseTemplateTag") + ).toSeq.foreach(tag => + customTaxo.addEdge("TaxonomyTag", tag) + ) + Success(()) + } + }.map(_ => ()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() From fb2c911020c0fec86dc9ce1b2212b8e453e60a24 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 12 Nov 2020 11:13:26 +0100 Subject: [PATCH 024/324] WIP adding taxonomy routes & mapping --- .../org/thp/thehive/dto/v0/Taxonomy.scala | 43 +++++++++++ .../org/thp/thehive/dto/v1/Taxonomy.scala | 43 +++++++++++ .../thp/thehive/controllers/v0/CaseCtrl.scala | 23 ++++-- .../thehive/controllers/v0/Conversion.scala | 21 ++++++ .../thehive/controllers/v0/TaxonomyCtrl.scala | 74 +++++++++++++++++++ .../thehive/controllers/v1/Conversion.scala | 22 ++++++ .../org/thp/thehive/models/Permissions.scala | 2 + .../app/org/thp/thehive/models/Taxonomy.scala | 36 +++++++++ .../thehive/services/OrganisationSrv.scala | 2 + .../thp/thehive/services/TaxonomySrv.scala | 66 +++++++++++++++++ 10 files changed, 324 insertions(+), 8 deletions(-) create mode 100644 dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala create mode 100644 dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala create mode 100644 thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala create mode 100644 thehive/app/org/thp/thehive/models/Taxonomy.scala create mode 100644 thehive/app/org/thp/thehive/services/TaxonomySrv.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala new file mode 100644 index 0000000000..a5af5ff61e --- /dev/null +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala @@ -0,0 +1,43 @@ +package org.thp.thehive.dto.v0 + +import java.util.Date + +import play.api.libs.json.{Json, OFormat, OWrites} + +case class InputTaxonomy ( + namespace: String, + description: String, + version: Int, + predicates: Seq[InputPredicate], + values: Option[Seq[InputValue]] +) + +case class InputPredicate(value: String, expanded: String) + +case class InputValue(predicate: String, entry: Seq[InputPredicate]) + +object InputTaxonomy { + implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] +} + +case class OutputTaxonomy( + _id: String, + _type: String, + _createdBy: String, + _updatedBy: Option[String] = None, + _createdAt: Date, + _updatedAt: Option[Date] = None, + namespace: String, + description: String, + version: Int, + predicates: Seq[OutputPredicate], + values: Option[Seq[OutputValue]] +) + +case class OutputPredicate(value: String, expanded: String) + +case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) + +object OutputTaxonomy { + implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] +} \ No newline at end of file diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala new file mode 100644 index 0000000000..1c6a1b2bc9 --- /dev/null +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -0,0 +1,43 @@ +package org.thp.thehive.dto.v1 + +import java.util.Date + +import play.api.libs.json.{Json, OFormat, OWrites} + +case class InputTaxonomy ( + namespace: String, + description: String, + version: Int, + predicates: Seq[InputPredicate], + values: Option[Seq[InputValue]] +) + +case class InputPredicate(value: String, expanded: String) + +case class InputValue(predicate: String, entry: Seq[InputPredicate]) + +object InputTaxonomy { + implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] +} + +case class OutputTaxonomy( + _id: String, + _type: String, + _createdBy: String, + _updatedBy: Option[String] = None, + _createdAt: Date, + _updatedAt: Option[Date] = None, + namespace: String, + description: String, + version: Int, + predicates: Seq[OutputPredicate], + values: Option[Seq[OutputValue]] +) + +case class OutputPredicate(value: String, expanded: String) + +case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) + +object OutputTaxonomy { + implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] +} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index f8fcd8aeed..71dff80ff5 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -195,13 +195,17 @@ class PublicCase @Inject() ( with CaseRenderer { override val entityName: String = "case" override val initialQuery: Query = - Query.init[Traversal.V[Case]]("listCase", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).cases) - override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Case]]( - "getCase", - FieldsParser[EntityIdOrName], - (idOrName, graph, authContext) => caseSrv.get(idOrName)(graph).visible(authContext) - ) - override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Case], IteratorOutput]( + Query.init[Traversal.V[Case]]("listCase", (graph, authContext) => + organisationSrv.get(authContext.organisation)(graph).cases + ) + override val getQuery: ParamQuery[EntityIdOrName] = + Query.initWithParam[EntityIdOrName, Traversal.V[Case]]( + "getCase", + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => caseSrv.get(idOrName)(graph).visible(authContext) + ) + override val pageQuery: ParamQuery[OutputParam] = + Query.withParam[OutputParam, Traversal.V[Case], IteratorOutput]( "page", FieldsParser[OutputParam], { @@ -215,7 +219,10 @@ class PublicCase @Inject() ( } } ) - override val outputQuery: Query = Query.outputWithContext[RichCase, Traversal.V[Case]]((caseSteps, authContext) => caseSteps.richCase(authContext)) + override val outputQuery: + Query = Query.outputWithContext[RichCase, Traversal.V[Case]]((caseSteps, authContext) => + caseSteps.richCase(authContext) + ) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Case], Traversal.V[Observable]]("observables", (caseSteps, authContext) => caseSteps.observables(authContext)), Query[Traversal.V[Case], Traversal.V[Task]]("tasks", (caseSteps, authContext) => caseSteps.tasks(authContext)) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index f972afd972..345a2a643e 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -571,6 +571,27 @@ object Conversion { .transform } + implicit class InputTaxonomyOps(inputTaxonomy: InputTaxonomy) { + + def toTaxonomy: Taxonomy = + inputTaxonomy + .into[Taxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .transform + } + + implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( + _.into[OutputTaxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .withFieldComputed(_.predicates, _.predicates) + .withFieldComputed(_.values, _.values) + .transform + ) + implicit class InputUserOps(inputUser: InputUser) { def toUser: User = diff --git a/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala new file mode 100644 index 0000000000..ac6622e6d2 --- /dev/null +++ b/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala @@ -0,0 +1,74 @@ +package org.thp.thehive.controllers.v0 + +import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName +import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} +import org.thp.scalligraph.models.{Database, UMapping} +import org.thp.scalligraph.query.{ParamQuery, PublicProperties, PublicPropertyListBuilder, Query, QueryExecutor} +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.thehive.controllers.v0.Conversion.taxonomyOutput +import org.thp.thehive.dto.v1.InputTaxonomy +import org.thp.thehive.models.{Permissions, RichTaxonomy, Taxonomy} +import org.thp.thehive.services.{OrganisationSrv, TaxonomySrv} +import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.TaxonomyOps._ +import play.api.mvc.{Action, AnyContent} + +class TaxonomyCtrl @Inject() ( + override val entrypoint: Entrypoint, + @Named("with-thehive-schema") override val db: Database, + @Named("v0") override val queryExecutor: QueryExecutor, + override val publicData: PublicTaxonomy +) extends QueryCtrl { + def importTaxonomy: Action[AnyContent] = + entrypoint("import taxonomy") + .extract("file", FieldsParser.file.optional.on("file")) + .extract("taxonomy", FieldsParser[InputTaxonomy]) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val file: Option[FFile] = request.body("file") + val taxonomy: InputTaxonomy = request.body("taxonomy") + + // Create Taxonomy vertex + // Create Tags associated + // Add edge orgaTaxo + + ??? + } + +} + +@Singleton +class PublicTaxonomy @Inject() ( + taxonomySrv: TaxonomySrv, + organisationSrv: OrganisationSrv +) extends PublicData { + override val entityName: String = "taxonomy" + override val initialQuery: Query = + Query.init[Traversal.V[Taxonomy]]("listTaxonomy", (graph, authContext) => + organisationSrv.get(authContext.organisation)(graph).taxonomies + ) + override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( + "page", + FieldsParser[OutputParam], + (range, taxoSteps, _) => taxoSteps.page(range.from, range.to, withTotal = true)(???) + ) + override val outputQuery: Query = Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((taxonomySteps, authContext) => + taxonomySteps.richTaxonomy(authContext) + ) + override val getQuery: ParamQuery[EntityIdOrName] = + Query.initWithParam[EntityIdOrName, Traversal.V[Taxonomy]]( + "getTaxonomy", + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => taxonomySrv.get(idOrName)(graph).visible(authContext) + ) + override val publicProperties: PublicProperties = + PublicPropertyListBuilder[Taxonomy] + .property("namespace", UMapping.string)(_.field.readonly) + .property("description", UMapping.string)(_.field.readonly) + .property("version", UMapping.int)(_.field.readonly) + // Predicates ? + // Values ? + .build + +} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index ac556fca70..eb3e8f4d2e 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -5,6 +5,7 @@ import java.util.Date import io.scalaland.chimney.dsl._ import org.thp.scalligraph.controllers.Renderer import org.thp.scalligraph.models.Entity +import org.thp.thehive.dto.v0.{InputTaxonomy, OutputTaxonomy} import org.thp.thehive.dto.v1._ import org.thp.thehive.models._ import play.api.libs.json.{JsObject, JsValue, Json} @@ -251,6 +252,27 @@ object Conversion { .transform } + implicit class InputTaxonomyOps(inputTaxonomy: InputTaxonomy) { + + def toTaxonomy: Taxonomy = + inputTaxonomy + .into[Taxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .transform + } + + implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( + _.into[OutputTaxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .withFieldComputed(_.predicates, _.predicates) + .withFieldComputed(_.values, _.values) + .transform + ) + implicit class InputUserOps(inputUser: InputUser) { def toUser: User = diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index 14b45cf5fc..de57993ad3 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -14,6 +14,7 @@ object Permissions extends Perms { lazy val manageAction: PermissionDesc = PermissionDesc("manageAction", "Run Cortex responders ", "organisation") lazy val manageConfig: PermissionDesc = PermissionDesc("manageConfig", "Manage configurations", "organisation", "admin") lazy val manageProfile: PermissionDesc = PermissionDesc("manageProfile", "Manage user profiles", "admin") + lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "admin") lazy val manageTag: PermissionDesc = PermissionDesc("manageTag", "Manage tags", "admin") lazy val manageCustomField: PermissionDesc = PermissionDesc("manageCustomField", "Manage custom fields", "admin") lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") @@ -35,6 +36,7 @@ object Permissions extends Perms { manageAction, manageConfig, manageProfile, + manageTaxonomy, manageTag, manageCustomField, manageShare, diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala new file mode 100644 index 0000000000..3ed3c6f0cb --- /dev/null +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -0,0 +1,36 @@ +package org.thp.thehive.models + +import java.util.Date + +import org.thp.scalligraph.models.Entity +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} + +@BuildVertexEntity +case class Taxonomy( + namespace: String, + description: String, + version: Int +) + +case class Predicate(value: String) + +case class Value(predicate: String, entry: Seq[String]) + +@BuildEdgeEntity[Taxonomy, Tag] +case class TaxonomyTag() + +case class RichTaxonomy( + taxonomy: Taxonomy with Entity, + predicates: Seq[Predicate], + values: Seq[Value] +) { + def _id: EntityId = taxonomy._id + def _createdBy: String = taxonomy._createdBy + def _updatedBy: Option[String] = taxonomy._updatedBy + def _createdAt: Date = taxonomy._createdAt + def _updatedAt: Option[Date] = taxonomy._updatedAt + def namespace: String = taxonomy.namespace + def description: String = taxonomy.description + def version: Int = taxonomy.version + +} diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 69af5f84de..e74b249a54 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -138,6 +138,8 @@ object OrganisationOps { def shares: Traversal.V[Share] = traversal.out[OrganisationShare].v[Share] + def taxonomies: Traversal.V[Taxonomy] = traversal.out[OrganisationTaxonomy].v[Taxonomy] + def caseTemplates: Traversal.V[CaseTemplate] = traversal.in[CaseTemplateOrganisation].v[CaseTemplate] def users(requiredPermission: Permission): Traversal.V[User] = diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala new file mode 100644 index 0000000000..7bd16db5c6 --- /dev/null +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -0,0 +1,66 @@ +package org.thp.thehive.services + +import java.util.{Map => JMap} + +import javax.inject.{Inject, Named} +import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.auth.AuthContext +import org.thp.scalligraph.models.{Database, Entity} +import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} +import org.thp.scalligraph.traversal.{Converter, Traversal} +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.thehive.models.{Organisation, OrganisationTaxonomy, Predicate, RichTaxonomy, Tag, Taxonomy, TaxonomyTag, Value} +import org.thp.thehive.services.OrganisationOps._ + +import scala.util.Try + +@Singleton +class TaxonomySrv @Inject() ( +)(implicit + @Named("with-thehive-schema") db: Database +) extends VertexSrv[Taxonomy] { + + val taxonomyTagSrv = new EdgeSrv[TaxonomyTag, Taxonomy, Tag] + + def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = + for { + taxonomy <- createEntity(taxo) + _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) + richTaxonomy <- RichTaxonomy(taxonomy, ???, ???) + } yield richTaxonomy +} + +object TaxonomyOps { + implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { + + def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = visible(authContext.organisation) + + def visible(organisationIdOrName: EntityIdOrName): Traversal.V[Taxonomy] = + traversal.filter(_.organisations.get(organisationIdOrName)) + + def organisations: Traversal.V[Organisation] = traversal.in[OrganisationTaxonomy].v[Organisation] + + def tags: Traversal.V[Tag] = traversal.out[TaxonomyTag].v[Tag] + + def richTaxonomy(implicit authContext: AuthContext): Traversal[RichTaxonomy, JMap[String, Any], Converter[RichTaxonomy, JMap[String, Any]]] = + traversal + .project( + _.by + .by(_.tags.fold) + ) + .domainMap { + case (taxonomy, tags) => + val predicates = tags.map(t => Predicate(t.predicate)).distinct + val values = predicates.map { p => + val tagValues = tags + .filter(_.predicate == p.value) + .filter(_.value.isDefined) + .map(_.value.get) + Value(p.value, tagValues) + } + RichTaxonomy(taxonomy, predicates, values) + } + + } +} From 2327ffa2b0cdef7cf01af926d4bd86908ebb39a0 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 12 Nov 2020 18:52:40 +0100 Subject: [PATCH 025/324] WIP Continued mapping for taxonomies --- .../org/thp/thehive/dto/v0/Taxonomy.scala | 43 -------- .../org/thp/thehive/dto/v1/Taxonomy.scala | 17 +-- .../thehive/controllers/v0/Conversion.scala | 21 ---- .../thehive/controllers/v0/TaxonomyCtrl.scala | 74 ------------- .../thehive/controllers/v1/Conversion.scala | 26 +++-- .../thehive/controllers/v1/Properties.scala | 10 ++ .../thehive/controllers/v1/TaxonomyCtrl.scala | 103 ++++++++++++++++++ .../app/org/thp/thehive/models/Taxonomy.scala | 8 +- .../app/org/thp/thehive/services/TagSrv.scala | 12 +- .../thp/thehive/services/TaxonomySrv.scala | 42 +++---- 10 files changed, 168 insertions(+), 188 deletions(-) delete mode 100644 dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala delete mode 100644 thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala create mode 100644 thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala deleted file mode 100644 index a5af5ff61e..0000000000 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Taxonomy.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.thp.thehive.dto.v0 - -import java.util.Date - -import play.api.libs.json.{Json, OFormat, OWrites} - -case class InputTaxonomy ( - namespace: String, - description: String, - version: Int, - predicates: Seq[InputPredicate], - values: Option[Seq[InputValue]] -) - -case class InputPredicate(value: String, expanded: String) - -case class InputValue(predicate: String, entry: Seq[InputPredicate]) - -object InputTaxonomy { - implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] -} - -case class OutputTaxonomy( - _id: String, - _type: String, - _createdBy: String, - _updatedBy: Option[String] = None, - _createdAt: Date, - _updatedAt: Option[Date] = None, - namespace: String, - description: String, - version: Int, - predicates: Seq[OutputPredicate], - values: Option[Seq[OutputValue]] -) - -case class OutputPredicate(value: String, expanded: String) - -case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) - -object OutputTaxonomy { - implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] -} \ No newline at end of file diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 1c6a1b2bc9..a2d05e879c 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -4,17 +4,18 @@ import java.util.Date import play.api.libs.json.{Json, OFormat, OWrites} +// TODO make sure of input format case class InputTaxonomy ( namespace: String, description: String, version: Int, - predicates: Seq[InputPredicate], - values: Option[Seq[InputValue]] + predicates: Seq[String], + values: Option[Seq[InputEntry]] ) -case class InputPredicate(value: String, expanded: String) +case class InputEntry(predicate: String, entry: Seq[InputValue]) -case class InputValue(predicate: String, entry: Seq[InputPredicate]) +case class InputValue(value: String, expanded: String, colour: Option[String]) object InputTaxonomy { implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] @@ -30,13 +31,13 @@ case class OutputTaxonomy( namespace: String, description: String, version: Int, - predicates: Seq[OutputPredicate], - values: Option[Seq[OutputValue]] + predicates: Seq[String], + values: Option[Seq[OutputEntry]] ) -case class OutputPredicate(value: String, expanded: String) +case class OutputEntry(predicate: String, entry: Seq[OutputValue]) -case class OutputValue(predicate: String, entry: Seq[OutputPredicate]) +case class OutputValue(value: String, expanded: String) object OutputTaxonomy { implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index 345a2a643e..f972afd972 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -571,27 +571,6 @@ object Conversion { .transform } - implicit class InputTaxonomyOps(inputTaxonomy: InputTaxonomy) { - - def toTaxonomy: Taxonomy = - inputTaxonomy - .into[Taxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .transform - } - - implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( - _.into[OutputTaxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldComputed(_.predicates, _.predicates) - .withFieldComputed(_.values, _.values) - .transform - ) - implicit class InputUserOps(inputUser: InputUser) { def toUser: User = diff --git a/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala deleted file mode 100644 index ac6622e6d2..0000000000 --- a/thehive/app/org/thp/thehive/controllers/v0/TaxonomyCtrl.scala +++ /dev/null @@ -1,74 +0,0 @@ -package org.thp.thehive.controllers.v0 - -import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.EntityIdOrName -import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} -import org.thp.scalligraph.models.{Database, UMapping} -import org.thp.scalligraph.query.{ParamQuery, PublicProperties, PublicPropertyListBuilder, Query, QueryExecutor} -import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs -import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.thehive.controllers.v0.Conversion.taxonomyOutput -import org.thp.thehive.dto.v1.InputTaxonomy -import org.thp.thehive.models.{Permissions, RichTaxonomy, Taxonomy} -import org.thp.thehive.services.{OrganisationSrv, TaxonomySrv} -import org.thp.thehive.services.OrganisationOps._ -import org.thp.thehive.services.TaxonomyOps._ -import play.api.mvc.{Action, AnyContent} - -class TaxonomyCtrl @Inject() ( - override val entrypoint: Entrypoint, - @Named("with-thehive-schema") override val db: Database, - @Named("v0") override val queryExecutor: QueryExecutor, - override val publicData: PublicTaxonomy -) extends QueryCtrl { - def importTaxonomy: Action[AnyContent] = - entrypoint("import taxonomy") - .extract("file", FieldsParser.file.optional.on("file")) - .extract("taxonomy", FieldsParser[InputTaxonomy]) - .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - val file: Option[FFile] = request.body("file") - val taxonomy: InputTaxonomy = request.body("taxonomy") - - // Create Taxonomy vertex - // Create Tags associated - // Add edge orgaTaxo - - ??? - } - -} - -@Singleton -class PublicTaxonomy @Inject() ( - taxonomySrv: TaxonomySrv, - organisationSrv: OrganisationSrv -) extends PublicData { - override val entityName: String = "taxonomy" - override val initialQuery: Query = - Query.init[Traversal.V[Taxonomy]]("listTaxonomy", (graph, authContext) => - organisationSrv.get(authContext.organisation)(graph).taxonomies - ) - override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( - "page", - FieldsParser[OutputParam], - (range, taxoSteps, _) => taxoSteps.page(range.from, range.to, withTotal = true)(???) - ) - override val outputQuery: Query = Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((taxonomySteps, authContext) => - taxonomySteps.richTaxonomy(authContext) - ) - override val getQuery: ParamQuery[EntityIdOrName] = - Query.initWithParam[EntityIdOrName, Traversal.V[Taxonomy]]( - "getTaxonomy", - FieldsParser[EntityIdOrName], - (idOrName, graph, authContext) => taxonomySrv.get(idOrName)(graph).visible(authContext) - ) - override val publicProperties: PublicProperties = - PublicPropertyListBuilder[Taxonomy] - .property("namespace", UMapping.string)(_.field.readonly) - .property("description", UMapping.string)(_.field.readonly) - .property("version", UMapping.int)(_.field.readonly) - // Predicates ? - // Values ? - .build - -} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index eb3e8f4d2e..ae205ddd65 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -5,7 +5,7 @@ import java.util.Date import io.scalaland.chimney.dsl._ import org.thp.scalligraph.controllers.Renderer import org.thp.scalligraph.models.Entity -import org.thp.thehive.dto.v0.{InputTaxonomy, OutputTaxonomy} +import org.thp.thehive.dto.v1.{InputTaxonomy, OutputTaxonomy} import org.thp.thehive.dto.v1._ import org.thp.thehive.models._ import play.api.libs.json.{JsObject, JsValue, Json} @@ -263,15 +263,20 @@ object Conversion { .transform } - implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( - _.into[OutputTaxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldComputed(_.predicates, _.predicates) - .withFieldComputed(_.values, _.values) - .transform - ) + implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = + Renderer.toJson[RichTaxonomy, OutputTaxonomy]( + _.into[OutputTaxonomy] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.version, _.version) + .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) + .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { + val outputValues = entryMap.getOrElse(tag.predicate, Seq()) + val value = OutputValue(tag.value.getOrElse(""), tag.description.getOrElse("")) + entryMap + (tag.predicate -> (outputValues :+ value)) + }).map(e => OutputEntry(e._1, e._2))) + .transform + ) implicit class InputUserOps(inputUser: InputUser) { @@ -357,6 +362,7 @@ object Conversion { .withFieldComputed(_.tlp, _.tlp.getOrElse(2)) .transform } + implicit val observableOutput: Renderer.Aux[RichObservable, OutputObservable] = Renderer.toJson[RichObservable, OutputObservable](richObservable => richObservable .into[OutputObservable] diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index a41f6a537d..9832b6b6f3 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -491,4 +491,14 @@ class Properties @Inject() ( .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build + + lazy val taxonomy: PublicProperties = + PublicPropertyListBuilder[Taxonomy] + .property("namespace", UMapping.string)(_.field.readonly) + .property("description", UMapping.string)(_.field.readonly) + .property("version", UMapping.int)(_.field.readonly) + // Predicates ? + // Values ? + .build + } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala new file mode 100644 index 0000000000..d60db4d141 --- /dev/null +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -0,0 +1,103 @@ +package org.thp.thehive.controllers.v1 + +import javax.inject.{Inject, Named} +import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} +import org.thp.scalligraph.models.Database +import org.thp.scalligraph.query._ +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.thehive.controllers.v1.Conversion._ +import org.thp.thehive.dto.v1.InputTaxonomy +import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} +import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.TaxonomyOps._ +import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} +import play.api.mvc.{Action, AnyContent, Results} + +import scala.+: + +class TaxonomyCtrl @Inject() ( + entrypoint: Entrypoint, + properties: Properties, + taxonomySrv: TaxonomySrv, + organisationSrv: OrganisationSrv, + tagSrv: TagSrv, + @Named("with-thehive-schema") implicit val db: Database +) extends QueryableCtrl { + + override val entityName: String = "taxonomy" + override val publicProperties: PublicProperties = properties.taxonomy + override val initialQuery: Query = + Query.init[Traversal.V[Taxonomy]]("listTaxonomy", (graph, authContext) => + organisationSrv.get(authContext.organisation)(graph).taxonomies + ) + override val getQuery: ParamQuery[EntityIdOrName] = + Query.initWithParam[EntityIdOrName, Traversal.V[Taxonomy]]( + "getTaxonomy", + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => taxonomySrv.get(idOrName)(graph).visible(authContext) + ) + override val pageQuery: ParamQuery[OutputParam] = + Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( + "page", + FieldsParser[OutputParam], + (range, traversal, authContext) => + traversal.richPage(range.from, range.to, range.extraData.contains("total"))(_.richTaxonomy(authContext)) + ) + override val outputQuery: Query = + Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((traversal, authContext) => + traversal.richTaxonomy(authContext) + ) + override val extraQueries: Seq[ParamQuery[_]] = Seq( + Query[Traversal.V[Taxonomy], Traversal.V[Tag]]("tags", (traversal, _) => traversal.tags) + ) + + def importTaxonomy: Action[AnyContent] = + entrypoint("import taxonomy") + .extract("file", FieldsParser.file.optional.on("file")) + .extract("taxonomy", FieldsParser[InputTaxonomy]) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val file: Option[FFile] = request.body("file") + val inputTaxo: InputTaxonomy = request.body("taxonomy") + + // TODO Parse file & combine with body + + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) + + // Create tags + val tagValues = inputTaxo.values.getOrElse(Seq()) + val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { + all ++ value.entry.map(e => + Tag(inputTaxo.namespace, + value.predicate, + Some(e.value), + Some(e.expanded), + e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour)) + + ) + } + ) + + // Create a tag for predicates with no tags associated + val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + tags ++ predicateWithNoTags.map(p => + Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) + ) + + for { + tagsEntities <- tags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield Results.Created(richTaxonomy.toJson) + } + + def delete(namespace: String): Action[AnyContent] = + entrypoint("delete taxonomy") + .authTransaction(db) { implicit request => implicit graph => + for { + t <- taxonomySrv.getByNamespace(namespace) + + } yield Results.Nocontent + } + +} diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index 3ed3c6f0cb..925956d9cd 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -12,17 +12,12 @@ case class Taxonomy( version: Int ) -case class Predicate(value: String) - -case class Value(predicate: String, entry: Seq[String]) - @BuildEdgeEntity[Taxonomy, Tag] case class TaxonomyTag() case class RichTaxonomy( taxonomy: Taxonomy with Entity, - predicates: Seq[Predicate], - values: Seq[Value] + tags: Seq[Tag with Entity] ) { def _id: EntityId = taxonomy._id def _createdBy: String = taxonomy._createdBy @@ -32,5 +27,4 @@ case class RichTaxonomy( def namespace: String = taxonomy.namespace def description: String = taxonomy.description def version: Int = taxonomy.version - } diff --git a/thehive/app/org/thp/thehive/services/TagSrv.scala b/thehive/app/org/thp/thehive/services/TagSrv.scala index a035558eaa..d937be0d92 100644 --- a/thehive/app/org/thp/thehive/services/TagSrv.scala +++ b/thehive/app/org/thp/thehive/services/TagSrv.scala @@ -19,27 +19,31 @@ class TagSrv @Inject() (appConfig: ApplicationConfig, @Named("integrity-check-ac @Named("with-thehive-schema") db: Database ) extends VertexSrv[Tag] { - val autoCreateConfig: ConfigItem[Boolean, Boolean] = + private val autoCreateConfig: ConfigItem[Boolean, Boolean] = appConfig.item[Boolean]("tags.autocreate", "If true, create automatically tag if it doesn't exist") def autoCreate: Boolean = autoCreateConfig.get - val defaultNamespaceConfig: ConfigItem[String, String] = + private val defaultNamespaceConfig: ConfigItem[String, String] = appConfig.item[String]("tags.defaultNamespace", "Default namespace of the automatically created tags") def defaultNamespace: String = defaultNamespaceConfig.get - val defaultColourConfig: ConfigItem[String, Int] = + private val defaultColourConfig: ConfigItem[String, Int] = appConfig.mapItem[String, Int]( "tags.defaultColour", "Default colour of the automatically created tags", { - case s if s(0) == '#' => Try(Integer.parseUnsignedInt(s.tail, 16)).getOrElse(defaultColour) + case s if s(0) == '#' => parseTagColour(s.tail) case _ => defaultColour } ) + def defaultColour: Int = defaultColourConfig.get + // TODO Duplication in Tag.scala + def parseTagColour(c: String) = Try(Integer.parseUnsignedInt(c, 16)).getOrElse(defaultColour) + def parseString(tagName: String): Tag = Tag.fromString(tagName, defaultNamespace, defaultColour) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 7bd16db5c6..8a7e906979 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -4,36 +4,48 @@ import java.util.{Map => JMap} import javax.inject.{Inject, Named} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.{EntityIdOrName, RichSeq} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} -import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs -import org.thp.thehive.models.{Organisation, OrganisationTaxonomy, Predicate, RichTaxonomy, Tag, Taxonomy, TaxonomyTag, Value} +import org.thp.scalligraph.traversal.{Converter, Traversal} +import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ import scala.util.Try @Singleton class TaxonomySrv @Inject() ( -)(implicit - @Named("with-thehive-schema") db: Database + organisationSrv: OrganisationSrv, + tagSrv: TagSrv +)(implicit @Named("with-thehive-schema") db: Database ) extends VertexSrv[Taxonomy] { val taxonomyTagSrv = new EdgeSrv[TaxonomyTag, Taxonomy, Tag] + val organisationTaxonomySrv = new EdgeSrv[OrganisationTaxonomy, Organisation, Taxonomy] def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { - taxonomy <- createEntity(taxo) - _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- RichTaxonomy(taxonomy, ???, ???) + taxonomy <- createEntity(taxo) + organisation <- organisationSrv.getOrFail(authContext.organisation) + _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) + _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy +/* + + def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = + Try(startTraversal.getByNamespace(namespace)).getOrElse(startTraversal.limit(0)) +*/ + } object TaxonomyOps { implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { + def getByNamespace(namespace: String): Traversal.V[Taxonomy] = traversal.has(_.namespace, namespace) + def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = visible(authContext.organisation) def visible(organisationIdOrName: EntityIdOrName): Traversal.V[Taxonomy] = @@ -49,18 +61,6 @@ object TaxonomyOps { _.by .by(_.tags.fold) ) - .domainMap { - case (taxonomy, tags) => - val predicates = tags.map(t => Predicate(t.predicate)).distinct - val values = predicates.map { p => - val tagValues = tags - .filter(_.predicate == p.value) - .filter(_.value.isDefined) - .map(_.value.get) - Value(p.value, tagValues) - } - RichTaxonomy(taxonomy, predicates, values) - } - + .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags) } } } From 9960e4dbd6db12da59846fdabce3dcc6b44b4c2f Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 15:18:24 +0100 Subject: [PATCH 026/324] WIP create taxonomiy / get works --- .../thp/thehive/dto/v1/CustomFieldValue.scala | 1 + .../org/thp/thehive/dto/v1/Taxonomy.scala | 30 +++++- .../thehive/controllers/v1/Conversion.scala | 10 +- .../thp/thehive/controllers/v1/Router.scala | 8 ++ .../thehive/controllers/v1/TaxonomyCtrl.scala | 93 +++++++++++-------- .../org/thp/thehive/models/Permissions.scala | 2 +- .../models/TheHiveSchemaDefinition.scala | 17 +++- .../thehive/services/OrganisationSrv.scala | 10 +- .../thp/thehive/services/TaxonomySrv.scala | 8 +- 9 files changed, 125 insertions(+), 54 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index 6e72438d06..06d6fb16e4 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -70,6 +70,7 @@ object InputCustomFieldValue { } case _ => Good(Nil) } + implicit val writes: Writes[Seq[InputCustomFieldValue]] = Writes[Seq[InputCustomFieldValue]] { icfv => val fields = icfv.map { case InputCustomFieldValue(name, Some(s: String), _) => name -> JsString(s) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index a2d05e879c..f0ebfb9659 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -2,7 +2,11 @@ package org.thp.thehive.dto.v1 import java.util.Date -import play.api.libs.json.{Json, OFormat, OWrites} +import org.scalactic.Accumulation.convertGenTraversableOnceToValidatable +import org.scalactic.{Bad, Good, One} +import org.thp.scalligraph.InvalidFormatAttributeError +import org.thp.scalligraph.controllers.{FObject, FSeq, FieldsParser, WithParser} +import play.api.libs.json.{JsArray, JsObject, JsString, Json, OFormat, OWrites, Writes} // TODO make sure of input format case class InputTaxonomy ( @@ -17,6 +21,20 @@ case class InputEntry(predicate: String, entry: Seq[InputValue]) case class InputValue(value: String, expanded: String, colour: Option[String]) +object InputEntry { + implicitly[FieldsParser[Option[Seq[InputEntry]]]] + + implicit val parser: FieldsParser[InputEntry] = FieldsParser[InputEntry] + + implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] +} + +object InputValue { + implicit val parser: FieldsParser[InputValue] = FieldsParser[InputValue] + + implicit val writes: Writes[InputValue] = Json.writes[InputValue] +} + object InputTaxonomy { implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] } @@ -32,7 +50,7 @@ case class OutputTaxonomy( description: String, version: Int, predicates: Seq[String], - values: Option[Seq[OutputEntry]] + values: Seq[OutputEntry] ) case class OutputEntry(predicate: String, entry: Seq[OutputValue]) @@ -41,4 +59,12 @@ case class OutputValue(value: String, expanded: String) object OutputTaxonomy { implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] +} + +object OutputEntry { + implicit val format: OFormat[OutputEntry] = Json.format[OutputEntry] +} + +object OutputValue { + implicit val format: OFormat[OutputValue] = Json.format[OutputValue] } \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index ae205ddd65..c6905ff5cd 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -266,15 +266,19 @@ object Conversion { implicit val taxonomyOutput: Renderer.Aux[RichTaxonomy, OutputTaxonomy] = Renderer.toJson[RichTaxonomy, OutputTaxonomy]( _.into[OutputTaxonomy] + .withFieldComputed(_._id, _._id.toString) + .withFieldConst(_._type, "Taxonomy") .withFieldComputed(_.namespace, _.namespace) .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { val outputValues = entryMap.getOrElse(tag.predicate, Seq()) - val value = OutputValue(tag.value.getOrElse(""), tag.description.getOrElse("")) - entryMap + (tag.predicate -> (outputValues :+ value)) - }).map(e => OutputEntry(e._1, e._2))) + if (tag.value.isDefined) + entryMap + (tag.predicate -> (outputValues :+ OutputValue(tag.value.get, tag.description.getOrElse("")))) + else + entryMap + (tag.predicate -> outputValues) + }).map(e => OutputEntry(e._1, e._2)).toSeq) .transform ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index feffe865bb..1683c010ad 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -14,6 +14,7 @@ class Router @Inject() ( taskCtrl: TaskCtrl, customFieldCtrl: CustomFieldCtrl, alertCtrl: AlertCtrl, + taxonomyCtrl: TaxonomyCtrl, auditCtrl: AuditCtrl, statusCtrl: StatusCtrl, authenticationCtrl: AuthenticationCtrl, @@ -90,6 +91,13 @@ class Router @Inject() ( // DELETE /alert/:alertId controllers.AlertCtrl.delete(alertId) // POST /alert/:alertId/merge/:caseId controllers.AlertCtrl.mergeWithCase(alertId, caseId) + case GET(p"/taxonomy") => taxonomyCtrl.list + case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) + case POST(p"/taxonomy") => taxonomyCtrl.create + // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip + // case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.activate + // case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.deactivate + case GET(p"/audit") => auditCtrl.flow // GET /flow controllers.AuditCtrl.flow(rootId: Option[String], count: Option[Int]) // GET /audit controllers.AuditCtrl.find() diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index d60db4d141..6a8220514c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -2,7 +2,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} import org.thp.scalligraph.{EntityIdOrName, RichSeq} -import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} +import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs @@ -15,7 +15,7 @@ import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} import play.api.mvc.{Action, AnyContent, Results} -import scala.+: +import scala.util.Success class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, @@ -42,55 +42,73 @@ class TaxonomyCtrl @Inject() ( Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( "page", FieldsParser[OutputParam], - (range, traversal, authContext) => - traversal.richPage(range.from, range.to, range.extraData.contains("total"))(_.richTaxonomy(authContext)) + (range, traversal, _) => + traversal.richPage(range.from, range.to, range.extraData.contains("total"))(_.richTaxonomy) ) override val outputQuery: Query = - Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((traversal, authContext) => - traversal.richTaxonomy(authContext) + Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((traversal, _) => + traversal.richTaxonomy ) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Taxonomy], Traversal.V[Tag]]("tags", (traversal, _) => traversal.tags) ) - def importTaxonomy: Action[AnyContent] = - entrypoint("import taxonomy") - .extract("file", FieldsParser.file.optional.on("file")) - .extract("taxonomy", FieldsParser[InputTaxonomy]) - .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - val file: Option[FFile] = request.body("file") - val inputTaxo: InputTaxonomy = request.body("taxonomy") - - // TODO Parse file & combine with body + def list: Action[AnyContent] = + entrypoint("list taxonomies") + .authRoTransaction(db) { implicit request => implicit graph => + val taxos = taxonomySrv + .startTraversal + .visible + .richTaxonomy + .toSeq + Success(Results.Ok(taxos.toJson)) + } - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) + def create: Action[AnyContent] = + entrypoint("import taxonomy") + .extract("taxonomy", FieldsParser[InputTaxonomy]) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val inputTaxo: InputTaxonomy = request.body("taxonomy") - // Create tags - val tagValues = inputTaxo.values.getOrElse(Seq()) - val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { - all ++ value.entry.map(e => - Tag(inputTaxo.namespace, - value.predicate, - Some(e.value), - Some(e.expanded), - e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour)) + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) - ) - } - ) + // Create tags + val tagValues = inputTaxo.values.getOrElse(Seq()) + val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { + all ++ value.entry.map(e => + Tag(inputTaxo.namespace, + value.predicate, + Some(e.value), + Some(e.expanded), + e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour) + ) + ) + }) - // Create a tag for predicates with no tags associated - val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) - tags ++ predicateWithNoTags.map(p => - Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) - ) + // Create a tag for predicates with no tags associated + val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + val allTags = tags ++ predicateWithNoTags.map(p => + Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) + ) - for { - tagsEntities <- tags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) - } yield Results.Created(richTaxonomy.toJson) + for { + tagsEntities <- allTags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield Results.Created(richTaxonomy.toJson) } + def get(taxonomyId: String): Action[AnyContent] = + entrypoint("get taxonomy") + .authRoTransaction(db) { implicit request => implicit graph => + taxonomySrv + .get(EntityIdOrName(taxonomyId)) + .visible + .richTaxonomy + .getOrFail("Taxonomy") + .map(taxonomy => Results.Ok(taxonomy.toJson)) + } + +/* def delete(namespace: String): Action[AnyContent] = entrypoint("delete taxonomy") .authTransaction(db) { implicit request => implicit graph => @@ -99,5 +117,6 @@ class TaxonomyCtrl @Inject() ( } yield Results.Nocontent } +*/ } diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index de57993ad3..af57429268 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -14,7 +14,7 @@ object Permissions extends Perms { lazy val manageAction: PermissionDesc = PermissionDesc("manageAction", "Run Cortex responders ", "organisation") lazy val manageConfig: PermissionDesc = PermissionDesc("manageConfig", "Manage configurations", "organisation", "admin") lazy val manageProfile: PermissionDesc = PermissionDesc("manageProfile", "Manage user profiles", "admin") - lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "admin") + lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "organisation", "admin") lazy val manageTag: PermissionDesc = PermissionDesc("manageTag", "Manage tags", "admin") lazy val manageCustomField: PermissionDesc = PermissionDesc("manageCustomField", "Manage custom fields", "admin") lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index c9ece8d6fc..5353be8aaf 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -1,6 +1,7 @@ package org.thp.thehive.models import java.lang.reflect.Modifier +import java.util.Date import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.P @@ -92,7 +93,12 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { db.tryTransaction { g => db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => val taxoVertex = g.addVertex("Taxonomy") - taxoVertex.property("namespace", "Custom") + taxoVertex.property("_label", "Taxonomy") + taxoVertex.property("_createdBy", "???") // TODO What user should be used ? + taxoVertex.property("_createdAt", new Date()) + taxoVertex.property("namespace", "custom") + taxoVertex.property("description", "Custom taxonomy") + taxoVertex.property("version", 1) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) } @@ -107,13 +113,18 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), _.in("AlertOrganisation").out("AlertTag"), _.in("CaseTemplateOrganisation").out("CaseTemplateTag") - ).toSeq.foreach(tag => + ).toSeq.foreach { tag => + tag.property("namespace", "custom") customTaxo.addEdge("TaxonomyTag", tag) - ) + } Success(()) } }.map(_ => ()) } + .updateGraph("Add manageTaxonomy to org-admin profile", "Profile") { traversal => + Try(traversal.unsafeHas("name", "org-admin").raw.property("permissions", "manageTaxonomy").iterate()) + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index e74b249a54..026f8a9554 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -3,7 +3,7 @@ package org.thp.thehive.services import java.util.{Map => JMap} import akka.actor.ActorRef -import javax.inject.{Inject, Named, Singleton} +import javax.inject.{Inject, Named, Provider, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.models._ @@ -23,6 +23,7 @@ import scala.util.{Failure, Success, Try} @Singleton class OrganisationSrv @Inject() ( + taxonomySrvProvider: Provider[TaxonomySrv], roleSrv: RoleSrv, profileSrv: ProfileSrv, auditSrv: AuditSrv, @@ -31,9 +32,9 @@ class OrganisationSrv @Inject() ( )(implicit @Named("with-thehive-schema") db: Database ) extends VertexSrv[Organisation] { - - val organisationOrganisationSrv = new EdgeSrv[OrganisationOrganisation, Organisation, Organisation] - val organisationShareSrv = new EdgeSrv[OrganisationShare, Organisation, Share] + lazy val taxonomySrv: TaxonomySrv = taxonomySrvProvider.get + val organisationOrganisationSrv = new EdgeSrv[OrganisationOrganisation, Organisation, Organisation] + val organisationShareSrv = new EdgeSrv[OrganisationShare, Organisation, Share] override def createEntity(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("Organisation") @@ -51,6 +52,7 @@ class OrganisationSrv @Inject() ( def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = for { createdOrganisation <- createEntity(e) + _ <- taxonomySrv.create(Taxonomy("custom", "Custom taxonomy", 1), Seq()) _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) } yield createdOrganisation diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 8a7e906979..55117defea 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -2,7 +2,7 @@ package org.thp.thehive.services import java.util.{Map => JMap} -import javax.inject.{Inject, Named} +import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} @@ -17,8 +17,7 @@ import scala.util.Try @Singleton class TaxonomySrv @Inject() ( - organisationSrv: OrganisationSrv, - tagSrv: TagSrv + organisationSrv: OrganisationSrv )(implicit @Named("with-thehive-schema") db: Database ) extends VertexSrv[Taxonomy] { @@ -33,6 +32,7 @@ class TaxonomySrv @Inject() ( _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy + /* def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = @@ -55,7 +55,7 @@ object TaxonomyOps { def tags: Traversal.V[Tag] = traversal.out[TaxonomyTag].v[Tag] - def richTaxonomy(implicit authContext: AuthContext): Traversal[RichTaxonomy, JMap[String, Any], Converter[RichTaxonomy, JMap[String, Any]]] = + def richTaxonomy: Traversal[RichTaxonomy, JMap[String, Any], Converter[RichTaxonomy, JMap[String, Any]]] = traversal .project( _.by From f285f2be966affd898b1a9ed925cc07b625a2354 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 16:47:17 +0100 Subject: [PATCH 027/324] WIP Custom taxonomy when new organisation is created --- .../main/scala/org/thp/thehive/dto/v1/Taxonomy.scala | 2 -- .../thp/thehive/models/TheHiveSchemaDefinition.scala | 2 +- .../app/org/thp/thehive/services/OrganisationSrv.scala | 2 +- thehive/app/org/thp/thehive/services/TaxonomySrv.scala | 10 +++++++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index f0ebfb9659..a73243a59e 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -22,8 +22,6 @@ case class InputEntry(predicate: String, entry: Seq[InputValue]) case class InputValue(value: String, expanded: String, colour: Option[String]) object InputEntry { - implicitly[FieldsParser[Option[Seq[InputEntry]]]] - implicit val parser: FieldsParser[InputEntry] = FieldsParser[InputEntry] implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 5353be8aaf..1a436fba6f 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -94,7 +94,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => val taxoVertex = g.addVertex("Taxonomy") taxoVertex.property("_label", "Taxonomy") - taxoVertex.property("_createdBy", "???") // TODO What user should be used ? + taxoVertex.property("_createdBy", "system@thehive.local") taxoVertex.property("_createdAt", new Date()) taxoVertex.property("namespace", "custom") taxoVertex.property("description", "Custom taxonomy") diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 026f8a9554..e2a8c61068 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -52,7 +52,7 @@ class OrganisationSrv @Inject() ( def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = for { createdOrganisation <- createEntity(e) - _ <- taxonomySrv.create(Taxonomy("custom", "Custom taxonomy", 1), Seq()) + _ <- taxonomySrv.createWithOrg(Taxonomy("custom", "Custom taxonomy", 1), Seq(), createdOrganisation) _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) } yield createdOrganisation diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 55117defea..28734aefb9 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -26,8 +26,16 @@ class TaxonomySrv @Inject() ( def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { - taxonomy <- createEntity(taxo) organisation <- organisationSrv.getOrFail(authContext.organisation) + richTaxonomy <- createWithOrg(taxo, tags, organisation) + } yield richTaxonomy + + def createWithOrg(taxo: Taxonomy, + tags: Seq[Tag with Entity], + organisation: Organisation with Entity) + (implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = + for { + taxonomy <- createEntity(taxo) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) From 8a4d73eb5680a792dfaa3cba21ecbe09bca98271 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 19:02:21 +0100 Subject: [PATCH 028/324] WIP Added taxonomy activate / deactivate --- .../main/scala/org/thp/thehive/dto/v1/Taxonomy.scala | 2 +- .../org/thp/thehive/controllers/v1/Conversion.scala | 2 ++ .../app/org/thp/thehive/controllers/v1/Router.scala | 6 +++--- .../org/thp/thehive/controllers/v1/TaxonomyCtrl.scala | 10 +++++++++- thehive/app/org/thp/thehive/models/Taxonomy.scala | 4 +++- .../thp/thehive/models/TheHiveSchemaDefinition.scala | 3 ++- .../app/org/thp/thehive/services/OrganisationSrv.scala | 6 ++++-- thehive/app/org/thp/thehive/services/TaxonomySrv.scala | 10 +++++++++- 8 files changed, 33 insertions(+), 10 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index a73243a59e..fe8eaa467c 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -8,7 +8,6 @@ import org.thp.scalligraph.InvalidFormatAttributeError import org.thp.scalligraph.controllers.{FObject, FSeq, FieldsParser, WithParser} import play.api.libs.json.{JsArray, JsObject, JsString, Json, OFormat, OWrites, Writes} -// TODO make sure of input format case class InputTaxonomy ( namespace: String, description: String, @@ -47,6 +46,7 @@ case class OutputTaxonomy( namespace: String, description: String, version: Int, + enabled: Boolean, predicates: Seq[String], values: Seq[OutputEntry] ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index c6905ff5cd..a3291bc422 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -260,6 +260,7 @@ object Conversion { .withFieldComputed(_.namespace, _.namespace) .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) + .withFieldConst(_.enabled, false) // TODO always false when importing a taxonomy ? .transform } @@ -271,6 +272,7 @@ object Conversion { .withFieldComputed(_.namespace, _.namespace) .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) + .withFieldComputed(_.enabled, _.enabled) .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { val outputValues = entryMap.getOrElse(tag.predicate, Seq()) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 1683c010ad..7fd69f6291 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -94,9 +94,9 @@ class Router @Inject() ( case GET(p"/taxonomy") => taxonomyCtrl.list case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case POST(p"/taxonomy") => taxonomyCtrl.create - // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip - // case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.activate - // case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.deactivate + // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip< + case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) + case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) case GET(p"/audit") => auditCtrl.flow // GET /flow controllers.AuditCtrl.flow(rootId: Option[String], count: Option[Int]) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 6a8220514c..b2b3ff7136 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -70,7 +70,7 @@ class TaxonomyCtrl @Inject() ( .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => val inputTaxo: InputTaxonomy = request.body("taxonomy") - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version) + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) // Create tags val tagValues = inputTaxo.values.getOrElse(Seq()) @@ -108,6 +108,14 @@ class TaxonomyCtrl @Inject() ( .map(taxonomy => Results.Ok(taxonomy.toJson)) } + def setEnabled(taxonomyId: String, isEnabled: Boolean): Action[AnyContent] = + entrypoint("toggle taxonomy") + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + taxonomySrv + .setEnabled(EntityIdOrName(taxonomyId), isEnabled) + .map(_ => Results.NoContent) + } + /* def delete(namespace: String): Action[AnyContent] = entrypoint("delete taxonomy") diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index 925956d9cd..7a8f9a46c2 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -9,7 +9,8 @@ import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} case class Taxonomy( namespace: String, description: String, - version: Int + version: Int, + enabled: Boolean ) @BuildEdgeEntity[Taxonomy, Tag] @@ -27,4 +28,5 @@ case class RichTaxonomy( def namespace: String = taxonomy.namespace def description: String = taxonomy.description def version: Int = taxonomy.version + def enabled: Boolean = taxonomy.enabled } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 1a436fba6f..78c439feb8 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -99,6 +99,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("namespace", "custom") taxoVertex.property("description", "Custom taxonomy") taxoVertex.property("version", 1) + taxoVertex.property("enabled", true) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) } @@ -107,7 +108,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => db.tryTransaction { implicit g => db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => - val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "Custom").head + val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "custom").head Traversal.V(EntityId(o.id())).unionFlat( _.out("OrganisationShare").out("ShareCase").out("CaseTag"), _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index e2a8c61068..05f3889499 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -49,12 +49,14 @@ class OrganisationSrv @Inject() ( _ <- roleSrv.create(user, createdOrganisation, profileSrv.orgAdmin) } yield createdOrganisation - def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = + def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { + val customTaxo = Taxonomy("custom", "Custom taxonomy", 1, enabled = true) for { createdOrganisation <- createEntity(e) - _ <- taxonomySrv.createWithOrg(Taxonomy("custom", "Custom taxonomy", 1), Seq(), createdOrganisation) + _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), createdOrganisation) _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) } yield createdOrganisation + } def current(implicit graph: Graph, authContext: AuthContext): Traversal.V[Organisation] = get(authContext.organisation) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 28734aefb9..42003e4f54 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -9,7 +9,7 @@ import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs import org.thp.scalligraph.traversal.{Converter, Traversal} -import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.{EntityId, EntityIdOrName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ @@ -41,6 +41,11 @@ class TaxonomySrv @Inject() ( richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy + def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = + for { + _ <- get(taxonomyId).update(_.enabled, isEnabled).getOrFail("Taxonomy") + } yield () + /* def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = @@ -52,6 +57,9 @@ class TaxonomySrv @Inject() ( object TaxonomyOps { implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { + def get(idOrName: EntityId): Traversal.V[Taxonomy] = + traversal.getByIds(idOrName) + def getByNamespace(namespace: String): Traversal.V[Taxonomy] = traversal.has(_.namespace, namespace) def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = visible(authContext.organisation) From a092f15b515bfe7f2e376e7f6d28feabfdf8fa55 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 16 Nov 2020 19:08:08 +0100 Subject: [PATCH 029/324] Fixed taxonomy values parsing --- .../main/scala/org/thp/thehive/dto/v1/Taxonomy.scala | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index fe8eaa467c..20890915a5 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -2,11 +2,7 @@ package org.thp.thehive.dto.v1 import java.util.Date -import org.scalactic.Accumulation.convertGenTraversableOnceToValidatable -import org.scalactic.{Bad, Good, One} -import org.thp.scalligraph.InvalidFormatAttributeError -import org.thp.scalligraph.controllers.{FObject, FSeq, FieldsParser, WithParser} -import play.api.libs.json.{JsArray, JsObject, JsString, Json, OFormat, OWrites, Writes} +import play.api.libs.json.{Json, OFormat, OWrites, Writes} case class InputTaxonomy ( namespace: String, @@ -21,14 +17,10 @@ case class InputEntry(predicate: String, entry: Seq[InputValue]) case class InputValue(value: String, expanded: String, colour: Option[String]) object InputEntry { - implicit val parser: FieldsParser[InputEntry] = FieldsParser[InputEntry] - implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] } object InputValue { - implicit val parser: FieldsParser[InputValue] = FieldsParser[InputValue] - implicit val writes: Writes[InputValue] = Json.writes[InputValue] } From 8863acb1ef31c8602ebbe2bdde8fc0a54b85f13c Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 11:42:50 +0100 Subject: [PATCH 030/324] Idempotent TheHive schema --- .../thp/thehive/controllers/v1/Router.scala | 2 +- .../models/TheHiveSchemaDefinition.scala | 27 +++++++++++-------- .../thehive/controllers/v1/UserCtrlTest.scala | 1 + 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 7fd69f6291..f3184dffdb 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -94,7 +94,7 @@ class Router @Inject() ( case GET(p"/taxonomy") => taxonomyCtrl.list case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case POST(p"/taxonomy") => taxonomyCtrl.create - // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip< + // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 78c439feb8..297d6f35bb 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -91,17 +91,22 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .addVertexModel[String]("Taxonomy", Seq("namespace")) .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => db.tryTransaction { g => - db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => - val taxoVertex = g.addVertex("Taxonomy") - taxoVertex.property("_label", "Taxonomy") - taxoVertex.property("_createdBy", "system@thehive.local") - taxoVertex.property("_createdAt", new Date()) - taxoVertex.property("namespace", "custom") - taxoVertex.property("description", "Custom taxonomy") - taxoVertex.property("version", 1) - taxoVertex.property("enabled", true) - o.addEdge("OrganisationTaxonomy", taxoVertex) - Success(()) + // If there are no taxonomies in database, add a custom one for each organisation + db.labelFilter("Taxonomy")(Traversal.V()(g)).headOption match { + case None => + db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => + val taxoVertex = g.addVertex("Taxonomy") + taxoVertex.property("_label", "Taxonomy") + taxoVertex.property("_createdBy", "system@thehive.local") + taxoVertex.property("_createdAt", new Date()) + taxoVertex.property("namespace", "custom") + taxoVertex.property("description", "Custom taxonomy") + taxoVertex.property("version", 1) + taxoVertex.property("enabled", true) + o.addEdge("OrganisationTaxonomy", taxoVertex) + Success(()) + } + case _ => Success(()) } }.map(_ => ()) } diff --git a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala index 8a5773b794..e1831040c5 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala @@ -109,6 +109,7 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { Permissions.managePage, Permissions.manageObservable, Permissions.manageAlert, + Permissions.manageTaxonomy, Permissions.manageAction, Permissions.manageConfig, Permissions.accessTheHiveFS From 2be85d8cd8246563424c6282ab59d175a1396830 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 14:23:35 +0100 Subject: [PATCH 031/324] Checked if taxonomy namespace is present before creating --- .../thp/thehive/controllers/v1/TaxonomyCtrl.scala | 15 +++++++++------ .../org/thp/thehive/services/TaxonomySrv.scala | 9 +++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index b2b3ff7136..844dbc311b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -1,7 +1,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} -import org.thp.scalligraph.{EntityIdOrName, RichSeq} +import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ @@ -15,7 +15,7 @@ import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} import play.api.mvc.{Action, AnyContent, Results} -import scala.util.Success +import scala.util.{Failure, Success} class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, @@ -91,10 +91,13 @@ class TaxonomyCtrl @Inject() ( Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) ) - for { - tagsEntities <- allTags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) - } yield Results.Created(richTaxonomy.toJson) + if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) + Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) + else + for { + tagsEntities <- allTags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield Results.Created(richTaxonomy.toJson) } def get(taxonomyId: String): Action[AnyContent] = diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 42003e4f54..49ee30d3d3 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -24,6 +24,15 @@ class TaxonomySrv @Inject() ( val taxonomyTagSrv = new EdgeSrv[TaxonomyTag, Taxonomy, Tag] val organisationTaxonomySrv = new EdgeSrv[OrganisationTaxonomy, Organisation, Taxonomy] + def existsInOrganisation(namespace: String)(implicit graph: Graph, authContext: AuthContext): Boolean = { + startTraversal + .has(_.namespace, namespace) + .in[OrganisationTaxonomy] + .v[Organisation] + .has(_.name, authContext.organisation.toString) // TODO not great + .exists + } + def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { organisation <- organisationSrv.getOrFail(authContext.organisation) From 54218aae48cedf1eeb84b6bf0beca92f07d8845c Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 15:09:28 +0100 Subject: [PATCH 032/324] Correct output format for taxonomies --- .../scala/org/thp/thehive/dto/v1/Tag.scala | 15 ++++++++++++++ .../org/thp/thehive/dto/v1/Taxonomy.scala | 15 +------------- .../thehive/controllers/v1/Conversion.scala | 20 +++++++++++-------- thehive/app/org/thp/thehive/models/Tag.scala | 19 ++++++++++++++++-- .../app/org/thp/thehive/models/Taxonomy.scala | 2 +- .../thp/thehive/services/TaxonomySrv.scala | 10 ++-------- 6 files changed, 48 insertions(+), 33 deletions(-) create mode 100644 dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala new file mode 100644 index 0000000000..3b536c867c --- /dev/null +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala @@ -0,0 +1,15 @@ +package org.thp.thehive.dto.v1 + +import play.api.libs.json.{Json, OFormat} + +case class OutputTag( + namespace: String, + predicate: String, + value: Option[String], + description: Option[String], + colour: Int +) + +object OutputTag { + implicit val format: OFormat[OutputTag] = Json.format[OutputTag] +} diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 20890915a5..70f0b23208 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -39,22 +39,9 @@ case class OutputTaxonomy( description: String, version: Int, enabled: Boolean, - predicates: Seq[String], - values: Seq[OutputEntry] + tags: Seq[OutputTag] ) -case class OutputEntry(predicate: String, entry: Seq[OutputValue]) - -case class OutputValue(value: String, expanded: String) - object OutputTaxonomy { implicit val format: OFormat[OutputTaxonomy] = Json.format[OutputTaxonomy] } - -object OutputEntry { - implicit val format: OFormat[OutputEntry] = Json.format[OutputEntry] -} - -object OutputValue { - implicit val format: OFormat[OutputValue] = Json.format[OutputValue] -} \ No newline at end of file diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index a3291bc422..db5d575096 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -273,14 +273,18 @@ object Conversion { .withFieldComputed(_.description, _.description) .withFieldComputed(_.version, _.version) .withFieldComputed(_.enabled, _.enabled) - .withFieldComputed(_.predicates, _.tags.map(_.predicate).distinct) - .withFieldComputed(_.values, _.tags.foldLeft(Map[String, Seq[OutputValue]]())((entryMap, tag) => { - val outputValues = entryMap.getOrElse(tag.predicate, Seq()) - if (tag.value.isDefined) - entryMap + (tag.predicate -> (outputValues :+ OutputValue(tag.value.get, tag.description.getOrElse("")))) - else - entryMap + (tag.predicate -> outputValues) - }).map(e => OutputEntry(e._1, e._2)).toSeq) + .withFieldComputed(_.tags, _.tags.map(_.toOutput)) + .transform + ) + + implicit val tagOutput: Renderer.Aux[RichTag, OutputTag] = + Renderer.toJson[RichTag, OutputTag]( + _.into[OutputTag] + .withFieldComputed(_.namespace, _.namespace) + .withFieldComputed(_.predicate, _.predicate) + .withFieldComputed(_.value, _.value) + .withFieldComputed(_.description, _.description) + .withFieldComputed(_.colour, _.colour) .transform ) diff --git a/thehive/app/org/thp/thehive/models/Tag.scala b/thehive/app/org/thp/thehive/models/Tag.scala index e188ee45c2..cc97dc317e 100644 --- a/thehive/app/org/thp/thehive/models/Tag.scala +++ b/thehive/app/org/thp/thehive/models/Tag.scala @@ -1,7 +1,9 @@ package org.thp.thehive.models -import org.thp.scalligraph.BuildVertexEntity -import org.thp.scalligraph.models.{DefineIndex, IndexType} +import java.util.Date + +import org.thp.scalligraph.{BuildVertexEntity, EntityId} +import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType} import play.api.Logger import scala.util.Try @@ -54,3 +56,16 @@ object Tag { } } } + +case class RichTag(tag: Tag with Entity) { + def _id: EntityId = tag._id + def _createdBy: String = tag._createdBy + def _updatedBy: Option[String] = tag._updatedBy + def _createdAt: Date = tag._createdAt + def _updatedAt: Option[Date] = tag._updatedAt + def namespace: String = tag.namespace + def predicate: String = tag.predicate + def value: Option[String] = tag.value + def description: Option[String] = tag.description + def colour: Int = tag.colour +} diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index 7a8f9a46c2..a7815963e6 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -18,7 +18,7 @@ case class TaxonomyTag() case class RichTaxonomy( taxonomy: Taxonomy with Entity, - tags: Seq[Tag with Entity] + tags: Seq[RichTag] ) { def _id: EntityId = taxonomy._id def _createdBy: String = taxonomy._createdBy diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 49ee30d3d3..1b29f20081 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -47,7 +47,7 @@ class TaxonomySrv @Inject() ( taxonomy <- createEntity(taxo) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags.map(RichTag))) } yield richTaxonomy def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = @@ -55,12 +55,6 @@ class TaxonomySrv @Inject() ( _ <- get(taxonomyId).update(_.enabled, isEnabled).getOrFail("Taxonomy") } yield () -/* - - def getByNamespace(namespace: String)(implicit graph: Graph): Traversal.V[Taxonomy] = - Try(startTraversal.getByNamespace(namespace)).getOrElse(startTraversal.limit(0)) -*/ - } object TaxonomyOps { @@ -86,6 +80,6 @@ object TaxonomyOps { _.by .by(_.tags.fold) ) - .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags) } + .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags.map(RichTag)) } } } From db3a94456150cc7f0a85cf1f03fb769d97f1eb91 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 17 Nov 2020 15:31:29 +0100 Subject: [PATCH 033/324] Query for taxonomies --- thehive/app/org/thp/thehive/controllers/v1/Router.scala | 3 +-- .../org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index f3184dffdb..1d49363c75 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -91,10 +91,9 @@ class Router @Inject() ( // DELETE /alert/:alertId controllers.AlertCtrl.delete(alertId) // POST /alert/:alertId/merge/:caseId controllers.AlertCtrl.mergeWithCase(alertId, caseId) - case GET(p"/taxonomy") => taxonomyCtrl.list - case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case POST(p"/taxonomy") => taxonomyCtrl.create // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip + case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index bbc3b86b81..27fed93b12 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -32,6 +32,7 @@ class TheHiveQueryExecutor @Inject() ( profileCtrl: ProfileCtrl, taskCtrl: TaskCtrl, userCtrl: UserCtrl, + taxonomyCtrl: TaxonomyCtrl, // dashboardCtrl: DashboardCtrl, properties: Properties, @Named("with-thehive-schema") implicit val db: Database @@ -53,7 +54,8 @@ class TheHiveQueryExecutor @Inject() ( profileCtrl, // tagCtrl, taskCtrl, - userCtrl + userCtrl, + taxonomyCtrl ) override val version: (Int, Int) = 1 -> 1 From c5e138538e70fac085fbb1eb6e74be9ad0e6b9d8 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 18 Nov 2020 12:13:01 +0100 Subject: [PATCH 034/324] Basic zip import --- .../org/thp/thehive/dto/v1/Taxonomy.scala | 49 +++++++--- .../thp/thehive/controllers/v1/Router.scala | 2 +- .../thehive/controllers/v1/TaxonomyCtrl.scala | 91 +++++++++++-------- 3 files changed, 90 insertions(+), 52 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 70f0b23208..576683127a 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -2,30 +2,57 @@ package org.thp.thehive.dto.v1 import java.util.Date -import play.api.libs.json.{Json, OFormat, OWrites, Writes} +import play.api.libs.json.{Json, OFormat} -case class InputTaxonomy ( +/* +Format based on : +https://tools.ietf.org/id/draft-dulaunoy-misp-taxonomy-format-04.html +*/ + +case class InputTaxonomy( namespace: String, description: String, version: Int, - predicates: Seq[String], - values: Option[Seq[InputEntry]] + `type`: Option[Seq[String]], + exclusive: Option[Boolean], + predicates: Seq[InputPredicate], + values: Option[Seq[InputValue]] +) + +case class InputPredicate( + value: String, + expanded: Option[String], + exclusive: Option[Boolean], + description: Option[String] ) -case class InputEntry(predicate: String, entry: Seq[InputValue]) +case class InputValue( + predicate: String, + entry: Seq[InputEntry] +) -case class InputValue(value: String, expanded: String, colour: Option[String]) +case class InputEntry( + value: String, + expanded: Option[String], + colour: Option[String], + description: Option[String], + numerical_value: Option[Int] +) -object InputEntry { - implicit val writes: Writes[InputEntry] = Json.writes[InputEntry] +object InputTaxonomy { + implicit val format: OFormat[InputTaxonomy] = Json.format[InputTaxonomy] +} + +object InputPredicate { + implicit val format: OFormat[InputPredicate] = Json.format[InputPredicate] } object InputValue { - implicit val writes: Writes[InputValue] = Json.writes[InputValue] + implicit val format: OFormat[InputValue] = Json.format[InputValue] } -object InputTaxonomy { - implicit val writes: OWrites[InputTaxonomy] = Json.writes[InputTaxonomy] +object InputEntry { + implicit val format: OFormat[InputEntry] = Json.format[InputEntry] } case class OutputTaxonomy( diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 1d49363c75..324df8eb8d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -92,7 +92,7 @@ class Router @Inject() ( // POST /alert/:alertId/merge/:caseId controllers.AlertCtrl.mergeWithCase(alertId, caseId) case POST(p"/taxonomy") => taxonomyCtrl.create - // case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip + case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 844dbc311b..5946240540 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -1,21 +1,25 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} -import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} -import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} +import net.lingala.zip4j.ZipFile +import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.auth.AuthContext +import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ -import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs +import org.thp.scalligraph.traversal.TraversalOps.{TraversalOpsDefs, logger} import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputTaxonomy import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} +import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, Results} -import scala.util.{Failure, Success} +import scala.util.{Failure, Success, Try} class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, @@ -68,36 +72,54 @@ class TaxonomyCtrl @Inject() ( entrypoint("import taxonomy") .extract("taxonomy", FieldsParser[InputTaxonomy]) .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - val inputTaxo: InputTaxonomy = request.body("taxonomy") + for { + richTaxonomy <- createFromInput(request.body("taxonomy")) + } yield Results.Created(richTaxonomy.toJson) + } - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) + def importZip: Action[AnyContent] = + entrypoint("import taxonomies zip") + .extract("file", FieldsParser.file.on("file")) + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + val file: FFile = request.body("file") + val zipFile = new ZipFile(file.filepath.toString) + zipFile.getFileHeaders.stream.forEach { fileHeader => + val json = Json.parse(zipFile.getInputStream(fileHeader)) + createFromInput(json.as[InputTaxonomy]) + } + + Success(Results.NoContent) + } - // Create tags - val tagValues = inputTaxo.values.getOrElse(Seq()) - val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { - all ++ value.entry.map(e => - Tag(inputTaxo.namespace, - value.predicate, - Some(e.value), - Some(e.expanded), - e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour) - ) - ) - }) + private def createFromInput(inputTaxo: InputTaxonomy)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { + val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) - // Create a tag for predicates with no tags associated - val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) - val allTags = tags ++ predicateWithNoTags.map(p => - Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) + // Create tags + val tagValues = inputTaxo.values.getOrElse(Seq()) + val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { + all ++ value.entry.map(e => + Tag(inputTaxo.namespace, + value.predicate, + Some(e.value), + e.expanded, + e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour) ) + ) + }) - if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) - Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) - else - for { - tagsEntities <- allTags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) - } yield Results.Created(richTaxonomy.toJson) + // Create a tag for predicates with no tags associated + val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + val allTags = tags ++ predicateWithNoTags.map(p => + Tag(inputTaxo.namespace, p.value, None, None, tagSrv.defaultColour) + ) + + if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) + Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) + else + for { + tagsEntities <- allTags.toTry(t => tagSrv.create(t)) + richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + } yield richTaxonomy } def get(taxonomyId: String): Action[AnyContent] = @@ -119,15 +141,4 @@ class TaxonomyCtrl @Inject() ( .map(_ => Results.NoContent) } -/* - def delete(namespace: String): Action[AnyContent] = - entrypoint("delete taxonomy") - .authTransaction(db) { implicit request => implicit graph => - for { - t <- taxonomySrv.getByNamespace(namespace) - - } yield Results.Nocontent - } -*/ - } From 120823564392bf32c09f8aa30068f731eaab4552 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 18 Nov 2020 17:32:51 +0100 Subject: [PATCH 035/324] Handled zip import errors --- .../thehive/controllers/v1/Conversion.scala | 14 +-------- .../thehive/controllers/v1/Properties.scala | 3 +- .../thehive/controllers/v1/TaxonomyCtrl.scala | 31 +++++++++++++------ .../thp/thehive/services/TaxonomySrv.scala | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index db5d575096..aef3fc1eea 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -257,10 +257,7 @@ object Conversion { def toTaxonomy: Taxonomy = inputTaxonomy .into[Taxonomy] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldConst(_.enabled, false) // TODO always false when importing a taxonomy ? + .withFieldConst(_.enabled, false) .transform } @@ -269,10 +266,6 @@ object Conversion { _.into[OutputTaxonomy] .withFieldComputed(_._id, _._id.toString) .withFieldConst(_._type, "Taxonomy") - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.version, _.version) - .withFieldComputed(_.enabled, _.enabled) .withFieldComputed(_.tags, _.tags.map(_.toOutput)) .transform ) @@ -280,11 +273,6 @@ object Conversion { implicit val tagOutput: Renderer.Aux[RichTag, OutputTag] = Renderer.toJson[RichTag, OutputTag]( _.into[OutputTag] - .withFieldComputed(_.namespace, _.namespace) - .withFieldComputed(_.predicate, _.predicate) - .withFieldComputed(_.value, _.value) - .withFieldComputed(_.description, _.description) - .withFieldComputed(_.colour, _.colour) .transform ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 9832b6b6f3..e8095fa781 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -497,8 +497,7 @@ class Properties @Inject() ( .property("namespace", UMapping.string)(_.field.readonly) .property("description", UMapping.string)(_.field.readonly) .property("version", UMapping.int)(_.field.readonly) - // Predicates ? - // Values ? + .property("enabled", UMapping.boolean)(_.field.readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 5946240540..5a48e28d8a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -2,14 +2,15 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named} import net.lingala.zip4j.ZipFile +import net.lingala.zip4j.model.FileHeader import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ -import org.thp.scalligraph.traversal.TraversalOps.{TraversalOpsDefs, logger} +import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq} +import org.thp.scalligraph.{BadRequestError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputTaxonomy import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} @@ -19,6 +20,7 @@ import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, Results} +import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} class TaxonomyCtrl @Inject() ( @@ -80,17 +82,28 @@ class TaxonomyCtrl @Inject() ( def importZip: Action[AnyContent] = entrypoint("import taxonomies zip") .extract("file", FieldsParser.file.on("file")) - .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + .authPermitted(Permissions.manageTaxonomy) { implicit request => val file: FFile = request.body("file") val zipFile = new ZipFile(file.filepath.toString) - zipFile.getFileHeaders.stream.forEach { fileHeader => - val json = Json.parse(zipFile.getInputStream(fileHeader)) - createFromInput(json.as[InputTaxonomy]) - } + val headers = zipFile + .getFileHeaders + .iterator() + .asScala - Success(Results.NoContent) + for { + inputTaxos <- headers.toTry(h => parseJsonFile(zipFile, h)) + richTaxos <- db.tryTransaction { implicit graph => + inputTaxos.toTry(inputTaxo => createFromInput(inputTaxo)).map(_.toJson) + } + } yield Results.Created(richTaxos) } + private def parseJsonFile(zipFile: ZipFile, h: FileHeader): Try[InputTaxonomy] = { + Try(Json.parse(zipFile.getInputStream(h)).as[InputTaxonomy]).recoverWith { + case _ => Failure(BadRequestError(s"File '${h.getFileName}' does not comply with the MISP taxonomy formatting")) + } + } + private def createFromInput(inputTaxo: InputTaxonomy)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) @@ -114,7 +127,7 @@ class TaxonomyCtrl @Inject() ( ) if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) - Failure(CreateError("A taxonomy with this namespace already exists in this organisation")) + Failure(BadRequestError(s"A taxonomy with namespace '${inputTaxo.namespace}' already exists in this organisation")) else for { tagsEntities <- allTags.toTry(t => tagSrv.create(t)) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 1b29f20081..062d96b4e4 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -29,7 +29,7 @@ class TaxonomySrv @Inject() ( .has(_.namespace, namespace) .in[OrganisationTaxonomy] .v[Organisation] - .has(_.name, authContext.organisation.toString) // TODO not great + .current .exists } From de6f2df5f875ca5bbe8d04f5ce723d416627356f Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 19 Nov 2020 11:18:59 +0100 Subject: [PATCH 036/324] Review changes --- .../thp/thehive/controllers/v1/Conversion.scala | 4 ++-- .../thp/thehive/controllers/v1/TaxonomyCtrl.scala | 8 +++----- thehive/app/org/thp/thehive/models/Tag.scala | 13 ------------- thehive/app/org/thp/thehive/models/Taxonomy.scala | 2 +- .../thehive/models/TheHiveSchemaDefinition.scala | 14 +++++++------- .../app/org/thp/thehive/services/TaxonomySrv.scala | 4 ++-- 6 files changed, 15 insertions(+), 30 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index aef3fc1eea..6ba56b3938 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -270,8 +270,8 @@ object Conversion { .transform ) - implicit val tagOutput: Renderer.Aux[RichTag, OutputTag] = - Renderer.toJson[RichTag, OutputTag]( + implicit val tagOutput: Renderer.Aux[Tag, OutputTag] = + Renderer.toJson[Tag, OutputTag]( _.into[OutputTag] .transform ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 5a48e28d8a..10c91d6974 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -105,12 +105,10 @@ class TaxonomyCtrl @Inject() ( } private def createFromInput(inputTaxo: InputTaxonomy)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { - val taxonomy = Taxonomy(inputTaxo.namespace, inputTaxo.description, inputTaxo.version, enabled = false) - // Create tags val tagValues = inputTaxo.values.getOrElse(Seq()) - val tags = tagValues.foldLeft(Seq[Tag]())((all, value) => { - all ++ value.entry.map(e => + val tags = tagValues.flatMap(value => { + value.entry.map(e => Tag(inputTaxo.namespace, value.predicate, Some(e.value), @@ -131,7 +129,7 @@ class TaxonomyCtrl @Inject() ( else for { tagsEntities <- allTags.toTry(t => tagSrv.create(t)) - richTaxonomy <- taxonomySrv.create(taxonomy, tagsEntities) + richTaxonomy <- taxonomySrv.create(inputTaxo.toTaxonomy, tagsEntities) } yield richTaxonomy } diff --git a/thehive/app/org/thp/thehive/models/Tag.scala b/thehive/app/org/thp/thehive/models/Tag.scala index cc97dc317e..3ad58979a5 100644 --- a/thehive/app/org/thp/thehive/models/Tag.scala +++ b/thehive/app/org/thp/thehive/models/Tag.scala @@ -56,16 +56,3 @@ object Tag { } } } - -case class RichTag(tag: Tag with Entity) { - def _id: EntityId = tag._id - def _createdBy: String = tag._createdBy - def _updatedBy: Option[String] = tag._updatedBy - def _createdAt: Date = tag._createdAt - def _updatedAt: Option[Date] = tag._updatedAt - def namespace: String = tag.namespace - def predicate: String = tag.predicate - def value: Option[String] = tag.value - def description: Option[String] = tag.description - def colour: Int = tag.colour -} diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index a7815963e6..bc4fb1a6d4 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -18,7 +18,7 @@ case class TaxonomyTag() case class RichTaxonomy( taxonomy: Taxonomy with Entity, - tags: Seq[RichTag] + tags: Seq[Tag] ) { def _id: EntityId = taxonomy._id def _createdBy: String = taxonomy._createdBy diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 297d6f35bb..6a206703e9 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -90,11 +90,11 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { // Taxonomies .addVertexModel[String]("Taxonomy", Seq("namespace")) .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => - db.tryTransaction { g => - // If there are no taxonomies in database, add a custom one for each organisation - db.labelFilter("Taxonomy")(Traversal.V()(g)).headOption match { - case None => - db.labelFilter("Organisation")(Traversal.V()(g)).toIterator.toTry { o => + db.tryTransaction { implicit g => + // For each organisation, if there is no custom taxonomy, create it + db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => + Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", "custom").headOption match { + case None => val taxoVertex = g.addVertex("Taxonomy") taxoVertex.property("_label", "Taxonomy") taxoVertex.property("_createdBy", "system@thehive.local") @@ -105,8 +105,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("enabled", true) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) - } - case _ => Success(()) + case _ => Success(()) + } } }.map(_ => ()) } diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 062d96b4e4..4b4b7c28a2 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -47,7 +47,7 @@ class TaxonomySrv @Inject() ( taxonomy <- createEntity(taxo) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- Try(RichTaxonomy(taxonomy, tags.map(RichTag))) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = @@ -80,6 +80,6 @@ object TaxonomyOps { _.by .by(_.tags.fold) ) - .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags.map(RichTag)) } + .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags) } } } From a0a4ca4291664b2d3be6080085a6ad0709d0f8cc Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 09:46:17 +0100 Subject: [PATCH 037/324] Added (de)activation & deletion --- .../org/thp/thehive/dto/v1/Taxonomy.scala | 1 - .../thehive/controllers/v1/Conversion.scala | 1 - .../thehive/controllers/v1/Properties.scala | 1 - .../thp/thehive/controllers/v1/Router.scala | 5 +-- .../thehive/controllers/v1/TaxonomyCtrl.scala | 31 ++++++++++++++----- .../org/thp/thehive/models/Permissions.scala | 2 +- .../app/org/thp/thehive/models/Taxonomy.scala | 4 +-- .../models/TheHiveSchemaDefinition.scala | 19 ++++++++++-- .../thehive/services/OrganisationSrv.scala | 13 +++++--- .../thp/thehive/services/TaxonomySrv.scala | 27 +++++++++++++--- 10 files changed, 76 insertions(+), 28 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 576683127a..7081347184 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -65,7 +65,6 @@ case class OutputTaxonomy( namespace: String, description: String, version: Int, - enabled: Boolean, tags: Seq[OutputTag] ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 6ba56b3938..d2518f459f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -257,7 +257,6 @@ object Conversion { def toTaxonomy: Taxonomy = inputTaxonomy .into[Taxonomy] - .withFieldConst(_.enabled, false) .transform } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index e8095fa781..2799630705 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -497,7 +497,6 @@ class Properties @Inject() ( .property("namespace", UMapping.string)(_.field.readonly) .property("description", UMapping.string)(_.field.readonly) .property("version", UMapping.int)(_.field.readonly) - .property("enabled", UMapping.boolean)(_.field.readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 324df8eb8d..f3bd9e882b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -94,8 +94,9 @@ class Router @Inject() ( case POST(p"/taxonomy") => taxonomyCtrl.create case POST(p"/taxonomy/import-zip") => taxonomyCtrl.importZip case GET(p"/taxonomy/$taxoId") => taxonomyCtrl.get(taxoId) - case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = true) - case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.setEnabled(taxoId, isEnabled = false) + case PUT(p"/taxonomy/$taxoId/activate") => taxonomyCtrl.toggleActivation(taxoId, isActive = true) + case PUT(p"/taxonomy/$taxoId/deactivate") => taxonomyCtrl.toggleActivation(taxoId, isActive = false) + case DELETE(p"/taxonomy/$taxoId") => taxonomyCtrl.delete(taxoId) case GET(p"/audit") => auditCtrl.flow // GET /flow controllers.AuditCtrl.flow(rootId: Option[String], count: Option[Int]) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 10c91d6974..e81c47a098 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -119,12 +119,16 @@ class TaxonomyCtrl @Inject() ( }) // Create a tag for predicates with no tags associated - val predicateWithNoTags = inputTaxo.predicates.diff(tagValues.map(_.predicate)) + val predicateWithNoTags = inputTaxo.predicates.map(_.value).diff(tagValues.map(_.predicate)) val allTags = tags ++ predicateWithNoTags.map(p => - Tag(inputTaxo.namespace, p.value, None, None, tagSrv.defaultColour) + Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour) ) - if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) + if (inputTaxo.namespace.isEmpty) + Failure(BadRequestError(s"A taxonomy with no namespace cannot be imported")) + else if (inputTaxo.namespace == "_freetags") + Failure(BadRequestError(s"Namespace _freetags is restricted for TheHive")) + else if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) Failure(BadRequestError(s"A taxonomy with namespace '${inputTaxo.namespace}' already exists in this organisation")) else for { @@ -144,12 +148,25 @@ class TaxonomyCtrl @Inject() ( .map(taxonomy => Results.Ok(taxonomy.toJson)) } - def setEnabled(taxonomyId: String, isEnabled: Boolean): Action[AnyContent] = + def toggleActivation(taxonomyId: String, isActive: Boolean): Action[AnyContent] = entrypoint("toggle taxonomy") .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => - taxonomySrv - .setEnabled(EntityIdOrName(taxonomyId), isEnabled) - .map(_ => Results.NoContent) + val toggleF = if (isActive) taxonomySrv.activate _ else taxonomySrv.deactivate _ + toggleF(EntityIdOrName(taxonomyId)).map(_ => Results.NoContent) + } + + def delete(taxoId: String): Action[AnyContent] = + entrypoint("delete taxonomy") + .authPermittedTransaction(db, Permissions.manageTaxonomy) { implicit request => implicit graph => + for { + taxo <- taxonomySrv + .get(EntityIdOrName(taxoId)) + .visible + .getOrFail("Taxonomy") + tags <- Try(taxonomySrv.get(taxo).tags.toSeq) + _ <- tags.toTry(t => tagSrv.delete(t)) + _ <- taxonomySrv.delete(taxo) + } yield Results.NoContent } } diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index af57429268..de57993ad3 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -14,7 +14,7 @@ object Permissions extends Perms { lazy val manageAction: PermissionDesc = PermissionDesc("manageAction", "Run Cortex responders ", "organisation") lazy val manageConfig: PermissionDesc = PermissionDesc("manageConfig", "Manage configurations", "organisation", "admin") lazy val manageProfile: PermissionDesc = PermissionDesc("manageProfile", "Manage user profiles", "admin") - lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "organisation", "admin") + lazy val manageTaxonomy: PermissionDesc = PermissionDesc("manageTaxonomy", "Manage taxonomies", "admin") lazy val manageTag: PermissionDesc = PermissionDesc("manageTag", "Manage tags", "admin") lazy val manageCustomField: PermissionDesc = PermissionDesc("manageCustomField", "Manage custom fields", "admin") lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") diff --git a/thehive/app/org/thp/thehive/models/Taxonomy.scala b/thehive/app/org/thp/thehive/models/Taxonomy.scala index bc4fb1a6d4..e5fcdb0c03 100644 --- a/thehive/app/org/thp/thehive/models/Taxonomy.scala +++ b/thehive/app/org/thp/thehive/models/Taxonomy.scala @@ -9,8 +9,7 @@ import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} case class Taxonomy( namespace: String, description: String, - version: Int, - enabled: Boolean + version: Int ) @BuildEdgeEntity[Taxonomy, Tag] @@ -28,5 +27,4 @@ case class RichTaxonomy( def namespace: String = taxonomy.namespace def description: String = taxonomy.description def version: Int = taxonomy.version - def enabled: Boolean = taxonomy.enabled } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 6a206703e9..9abe0a84de 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -99,7 +99,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("_label", "Taxonomy") taxoVertex.property("_createdBy", "system@thehive.local") taxoVertex.property("_createdAt", new Date()) - taxoVertex.property("namespace", "custom") + taxoVertex.property("namespace", "_freetags") taxoVertex.property("description", "Custom taxonomy") taxoVertex.property("version", 1) taxoVertex.property("enabled", true) @@ -113,14 +113,22 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => db.tryTransaction { implicit g => db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => - val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "custom").head + val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "_freetags").head Traversal.V(EntityId(o.id())).unionFlat( _.out("OrganisationShare").out("ShareCase").out("CaseTag"), _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), _.in("AlertOrganisation").out("AlertTag"), _.in("CaseTemplateOrganisation").out("CaseTemplateTag") ).toSeq.foreach { tag => - tag.property("namespace", "custom") + // Create a freetext tag and store it into predicate + val tagStr = tagString( + tag.property("namespace").value().toString, + tag.property("predicate").value().toString, + tag.property("value").value().toString + ) + tag.property("namespace", "_freetags") + tag.property("predicate", tagStr) + tag.property("value").remove() customTaxo.addEdge("TaxonomyTag", tag) } Success(()) @@ -158,5 +166,10 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { case vertexModel: VertexModel => vertexModel.getInitialValues }.flatten + private def tagString(namespace: String, predicate: String, value: String): String = + (if (namespace.headOption.getOrElse('_') == '_') "" else namespace + ':') + + (if (predicate.headOption.getOrElse('_') == '_') "" else predicate) + + (if (value.isEmpty) "" else f"""="$value"""") + override def init(db: Database)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = Success(()) } diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 05f3889499..0a30d74c74 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -35,6 +35,7 @@ class OrganisationSrv @Inject() ( lazy val taxonomySrv: TaxonomySrv = taxonomySrvProvider.get val organisationOrganisationSrv = new EdgeSrv[OrganisationOrganisation, Organisation, Organisation] val organisationShareSrv = new EdgeSrv[OrganisationShare, Organisation, Share] + val organisationTaxonomySrv = new EdgeSrv[OrganisationTaxonomy, Organisation, Taxonomy] override def createEntity(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("Organisation") @@ -50,12 +51,14 @@ class OrganisationSrv @Inject() ( } yield createdOrganisation def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { - val customTaxo = Taxonomy("custom", "Custom taxonomy", 1, enabled = true) + val customTaxo = Taxonomy("_freetags", "Custom taxonomy", 1) + val activeTaxos = getByName("admin").taxonomies.toSeq for { - createdOrganisation <- createEntity(e) - _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), createdOrganisation) - _ <- auditSrv.organisation.create(createdOrganisation, createdOrganisation.toJson) - } yield createdOrganisation + newOrga <- createEntity(e) + _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), newOrga) + _ <- activeTaxos.toTry(t => organisationTaxonomySrv.create(OrganisationTaxonomy(), newOrga, t)) + _ <- auditSrv.organisation.create(newOrga, newOrga.toJson) + } yield newOrga } def current(implicit graph: Graph, authContext: AuthContext): Traversal.V[Organisation] = get(authContext.organisation) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 4b4b7c28a2..b172b44c6b 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -12,8 +12,9 @@ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.scalligraph.{EntityId, EntityIdOrName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.TaxonomyOps._ -import scala.util.Try +import scala.util.{Success, Try} @Singleton class TaxonomySrv @Inject() ( @@ -50,10 +51,28 @@ class TaxonomySrv @Inject() ( richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy - def setEnabled(taxonomyId: EntityIdOrName, isEnabled: Boolean)(implicit graph: Graph): Try[Unit] = + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Taxonomy] = + Try(startTraversal.getByNamespace(name)).getOrElse(startTraversal.limit(0)) + + def activate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + for { + taxo <- get(taxonomyId).getOrFail("Taxonomy") + organisations <- Try(organisationSrv.startTraversal.filter(_ + .out[OrganisationTaxonomy] + .filter(_.unsafeHas("namespace", taxo.namespace)) + ).toSeq) + _ <- organisations.toTry(o => organisationTaxonomySrv.create(OrganisationTaxonomy(), o, taxo)) + } yield Success(()) + + def deactivate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { - _ <- get(taxonomyId).update(_.enabled, isEnabled).getOrFail("Taxonomy") - } yield () + taxo <- get(taxonomyId).getOrFail("Taxonomy") + _ <- Try(organisationSrv + .get(authContext.organisation) + .outE[OrganisationTaxonomy] + .filter(_.otherV().unsafeHas("namespace", taxo.namespace)) + .remove()) + } yield Success(()) } From c53c619c59e09ca51ad119cffb4b7d1c9993d0ba Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 10:26:59 +0100 Subject: [PATCH 038/324] Used correct Scalligraph commit --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index f6a4d2165c..1a55a0db73 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit f6a4d2165c26826c5b28db1a513ade15dfb060f2 +Subproject commit 1a55a0db730460c6f548695251248934196b6ecc From c87f090bcbb060e62e5527c45b7b25acdc3670dd Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 10:39:38 +0100 Subject: [PATCH 039/324] Edit drone.yml --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 16354f6ca6..2b2ebfffeb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,7 +8,7 @@ steps: - name: submodules image: alpine/git commands: - - git submodule update --recursive --init --remote + - git submodule update --recursive --init # Restore cache of downloaded dependencies - name: restore-cache From a934a82484725a30f6c3746c31baa9237b6a8101 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 17:07:18 +0100 Subject: [PATCH 040/324] Fixed schema erros & (de)activation --- .../models/TheHiveSchemaDefinition.scala | 10 +- .../thehive/services/OrganisationSrv.scala | 3 +- .../thp/thehive/services/TaxonomySrv.scala | 24 +-- .../org/thp/thehive/DatabaseBuilder.scala | 5 + .../controllers/v1/TaxonomyCtrlTest.scala | 204 ++++++++++++++++++ .../resources/data/OrganisationTaxonomy.json | 5 + thehive/test/resources/data/Tag.json | 7 + thehive/test/resources/data/Taxonomy.json | 8 + thehive/test/resources/data/TaxonomyTag.json | 3 + .../test/resources/machinetag-badformat.zip | Bin 0 -> 4274 bytes .../test/resources/machinetag-otherfiles.zip | Bin 0 -> 3841 bytes thehive/test/resources/machinetag-present.zip | Bin 0 -> 3941 bytes thehive/test/resources/machinetag.zip | Bin 0 -> 4076 bytes 13 files changed, 250 insertions(+), 19 deletions(-) create mode 100644 thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala create mode 100644 thehive/test/resources/data/OrganisationTaxonomy.json create mode 100644 thehive/test/resources/data/Taxonomy.json create mode 100644 thehive/test/resources/data/TaxonomyTag.json create mode 100644 thehive/test/resources/machinetag-badformat.zip create mode 100644 thehive/test/resources/machinetag-otherfiles.zip create mode 100644 thehive/test/resources/machinetag-present.zip create mode 100644 thehive/test/resources/machinetag.zip diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 9abe0a84de..c0542d306f 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -92,8 +92,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db => db.tryTransaction { implicit g => // For each organisation, if there is no custom taxonomy, create it - db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => - Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", "custom").headOption match { + db.labelFilter("Organisation")(Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o => + Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", "_freetags").headOption match { case None => val taxoVertex = g.addVertex("Taxonomy") taxoVertex.property("_label", "Taxonomy") @@ -124,7 +124,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { val tagStr = tagString( tag.property("namespace").value().toString, tag.property("predicate").value().toString, - tag.property("value").value().toString + tag.property ("value").orElse("") ) tag.property("namespace", "_freetags") tag.property("predicate", tagStr) @@ -135,8 +135,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { } }.map(_ => ()) } - .updateGraph("Add manageTaxonomy to org-admin profile", "Profile") { traversal => - Try(traversal.unsafeHas("name", "org-admin").raw.property("permissions", "manageTaxonomy").iterate()) + .updateGraph("Add manageTaxonomy to admin profile", "Profile") { traversal => + Try(traversal.unsafeHas("name", "admin").raw.property("permissions", "manageTaxonomy").iterate()) Success(()) } diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 0a30d74c74..c567696848 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -51,11 +51,10 @@ class OrganisationSrv @Inject() ( } yield createdOrganisation def create(e: Organisation)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = { - val customTaxo = Taxonomy("_freetags", "Custom taxonomy", 1) val activeTaxos = getByName("admin").taxonomies.toSeq for { newOrga <- createEntity(e) - _ <- taxonomySrv.createWithOrg(customTaxo, Seq(), newOrga) + _ <- taxonomySrv.createFreetag(newOrga) _ <- activeTaxos.toTry(t => organisationTaxonomySrv.create(OrganisationTaxonomy(), newOrga, t)) _ <- auditSrv.organisation.create(newOrga, newOrga.toJson) } yield newOrga diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index b172b44c6b..2051c64930 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -36,20 +36,20 @@ class TaxonomySrv @Inject() ( def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { - organisation <- organisationSrv.getOrFail(authContext.organisation) - richTaxonomy <- createWithOrg(taxo, tags, organisation) + taxonomy <- createEntity(taxo) + _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) + richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) + _ <- activate(richTaxonomy._id) } yield richTaxonomy - def createWithOrg(taxo: Taxonomy, - tags: Seq[Tag with Entity], - organisation: Organisation with Entity) - (implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = + def createFreetag(organisation: Organisation with Entity)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { + val customTaxo = Taxonomy("_freetags", "Custom taxonomy", 1) for { - taxonomy <- createEntity(taxo) + taxonomy <- createEntity(customTaxo) + richTaxonomy <- Try(RichTaxonomy(taxonomy, Seq())) _ <- organisationTaxonomySrv.create(OrganisationTaxonomy(), organisation, taxonomy) - _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) - richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) } yield richTaxonomy + } override def getByName(name: String)(implicit graph: Graph): Traversal.V[Taxonomy] = Try(startTraversal.getByNamespace(name)).getOrElse(startTraversal.limit(0)) @@ -57,7 +57,7 @@ class TaxonomySrv @Inject() ( def activate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { taxo <- get(taxonomyId).getOrFail("Taxonomy") - organisations <- Try(organisationSrv.startTraversal.filter(_ + organisations <- Try(organisationSrv.startTraversal.filterNot(_ .out[OrganisationTaxonomy] .filter(_.unsafeHas("namespace", taxo.namespace)) ).toSeq) @@ -67,8 +67,8 @@ class TaxonomySrv @Inject() ( def deactivate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { taxo <- get(taxonomyId).getOrFail("Taxonomy") - _ <- Try(organisationSrv - .get(authContext.organisation) + _ <- Try(organisationSrv.startTraversal + .filterNot(_.unsafeHas("name", "admin")) .outE[OrganisationTaxonomy] .filter(_.otherV().unsafeHas("namespace", taxo.namespace)) .remove()) diff --git a/thehive/test/org/thp/thehive/DatabaseBuilder.scala b/thehive/test/org/thp/thehive/DatabaseBuilder.scala index 51767a822f..52094c2147 100644 --- a/thehive/test/org/thp/thehive/DatabaseBuilder.scala +++ b/thehive/test/org/thp/thehive/DatabaseBuilder.scala @@ -35,6 +35,7 @@ class DatabaseBuilder @Inject() ( observableSrv: ObservableSrv, observableTypeSrv: ObservableTypeSrv, taskSrv: TaskSrv, + taxonomySrv: TaxonomySrv, tagSrv: TagSrv, keyValueSrv: KeyValueSrv, dataSrv: DataSrv, @@ -82,11 +83,15 @@ class DatabaseBuilder @Inject() ( createVertex(impactStatusSrv, FieldsParser[ImpactStatus]) ++ createVertex(attachmentSrv, FieldsParser[Attachment]) ++ createVertex(tagSrv, FieldsParser[Tag]) ++ + createVertex(taxonomySrv, FieldsParser[Taxonomy]) ++ createVertex(pageSrv, FieldsParser[Page]) ++ createVertex(dashboardSrv, FieldsParser[Dashboard]) createEdge(organisationSrv.organisationOrganisationSrv, organisationSrv, organisationSrv, FieldsParser[OrganisationOrganisation], idMap) createEdge(organisationSrv.organisationShareSrv, organisationSrv, shareSrv, FieldsParser[OrganisationShare], idMap) + createEdge(organisationSrv.organisationTaxonomySrv, organisationSrv, taxonomySrv, FieldsParser[OrganisationTaxonomy], idMap) + + createEdge(taxonomySrv.taxonomyTagSrv, taxonomySrv, tagSrv, FieldsParser[TaxonomyTag], idMap) createEdge(roleSrv.userRoleSrv, userSrv, roleSrv, FieldsParser[UserRole], idMap) diff --git a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala new file mode 100644 index 0000000000..d08034f2c9 --- /dev/null +++ b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala @@ -0,0 +1,204 @@ +package org.thp.thehive.controllers.v1 + +import org.thp.scalligraph.controllers.FakeTemporaryFile +import org.thp.thehive.TestAppBuilder +import org.thp.thehive.dto.v1.{InputEntry, InputPredicate, InputTaxonomy, InputValue, OutputTag, OutputTaxonomy} +import play.api.libs.Files +import play.api.libs.json.Json +import play.api.mvc.{AnyContentAsMultipartFormData, MultipartFormData} +import play.api.mvc.MultipartFormData.FilePart +import play.api.test.{FakeRequest, PlaySpecification} + +case class TestTaxonomy( + namespace: String, + description: String, + version: Int, + tags: List[OutputTag] +) + +object TestTaxonomy { + def apply(outputTaxonomy: OutputTaxonomy): TestTaxonomy = + TestTaxonomy( + outputTaxonomy.namespace, + outputTaxonomy.description, + outputTaxonomy.version, + outputTaxonomy.tags.toList, + ) +} + +class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { + "taxonomy controller" should { + + val inputTaxo = InputTaxonomy( + "test-taxo", + "A test taxonomy", + 1, + None, + None, + List( + InputPredicate("pred1", None, None, None), + InputPredicate("pred2", None, None, None) + ), + Some(List( + InputValue("pred1", List( + InputEntry("entry1", None, None, None, None)) + ), + InputValue("pred2", List( + InputEntry("entry2", None, None, None, None), + InputEntry("entry21", None, None, None, None) + )) + )) + ) + + "create a valid taxonomy" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(inputTaxo)) + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + + val resultCase = contentAsJson(result).as[OutputTaxonomy] + + TestTaxonomy(resultCase) must_=== TestTaxonomy( + "test-taxo", + "A test taxonomy", + 1, + List( + OutputTag("test-taxo", "pred1", Some("entry1"), None, 0), + OutputTag("test-taxo", "pred2", Some("entry2"), None, 0), + OutputTag("test-taxo", "pred2", Some("entry21"), None, 0) + ) + ) + } + + "return error if not admin" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(inputTaxo)) + .withHeaders("user" -> "certuser@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(403).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("AuthorizationError") + } + + "return error if namespace is present in database" in testApp { app => + val alreadyInDatabase = inputTaxo.copy(namespace = "taxonomy1") + + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(alreadyInDatabase)) + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("already exists") + } + + "return error if namespace is empty" in testApp { app => + val emptyNamespace = inputTaxo.copy(namespace = "") + + val request = FakeRequest("POST", "/api/v1/taxonomy") + .withJsonBody(Json.toJson(emptyNamespace)) + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].create(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + + } + + "get a taxonomy present" in testApp { app => + val request = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + + val result = app[TaxonomyCtrl].get("taxonomy1")(request) + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + val resultCase = contentAsJson(result).as[OutputTaxonomy] + + TestTaxonomy(resultCase) must_=== TestTaxonomy( + "taxonomy1", + "The taxonomy 1", + 1, + List(OutputTag("taxonomy1", "pred1", Some("value1"), None, 0)) + ) + } + + "return error if taxonomy is not present in database" in testApp { app => + val request = FakeRequest("GET", "/api/v1/taxonomy/taxonomy404") + .withHeaders("user" -> "admin@thehive.local") + + val result = app[TaxonomyCtrl].get("taxonomy404")(request) + status(result) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("NotFoundError") + } + + "import zip file correctly" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + + val zipTaxos = contentAsJson(result).as[Seq[OutputTaxonomy]] + zipTaxos.size must beEqualTo(2) + } + + "return error if zip file contains other files than taxonomies" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-otherfiles.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("formatting") + } + + "return error if zip file contains an already present taxonomy" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-present.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("already exists") + } + + "return error if zip file contains a bad formatted taxonomy" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-badformat.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") + (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") + (contentAsJson(result) \ "message").as[String] must contain("formatting") + } + + /* + "activate a taxonomy" in testApp { app => + + } + + "deactivate a taxonomy" in testApp { app => + + } + + "delete a taxonomy" in testApp { app => + + } + + */ + } + + def multipartZipFile(name: String): MultipartFormData[Files.TemporaryFile] = MultipartFormData( + // file must be place in test/resources/ + dataParts = Map.empty, + files = Seq(FilePart("file", name, Option("application/zip"), FakeTemporaryFile.fromResource(s"/$name"))), + badParts = Seq() + ) + +} diff --git a/thehive/test/resources/data/OrganisationTaxonomy.json b/thehive/test/resources/data/OrganisationTaxonomy.json new file mode 100644 index 0000000000..df6a1338b2 --- /dev/null +++ b/thehive/test/resources/data/OrganisationTaxonomy.json @@ -0,0 +1,5 @@ +[ + {"from": "admin", "to": "taxonomy1"}, + {"from": "cert", "to": "taxonomy1"}, + {"from": "soc", "to": "taxonomy1"} +] \ No newline at end of file diff --git a/thehive/test/resources/data/Tag.json b/thehive/test/resources/data/Tag.json index c6136decb4..094be1895a 100644 --- a/thehive/test/resources/data/Tag.json +++ b/thehive/test/resources/data/Tag.json @@ -68,5 +68,12 @@ "predicate": "testPredicate", "value": "world", "colour": 0 + }, + { + "id": "taxonomy-tag1", + "namespace": "taxonomy1", + "predicate": "pred1", + "value": "value1", + "colour": 0 } ] \ No newline at end of file diff --git a/thehive/test/resources/data/Taxonomy.json b/thehive/test/resources/data/Taxonomy.json new file mode 100644 index 0000000000..500c39c010 --- /dev/null +++ b/thehive/test/resources/data/Taxonomy.json @@ -0,0 +1,8 @@ +[ + { + "id": "taxonomy1", + "namespace": "taxonomy1", + "description": "The taxonomy 1", + "version": "1" + } +] \ No newline at end of file diff --git a/thehive/test/resources/data/TaxonomyTag.json b/thehive/test/resources/data/TaxonomyTag.json new file mode 100644 index 0000000000..80806c707c --- /dev/null +++ b/thehive/test/resources/data/TaxonomyTag.json @@ -0,0 +1,3 @@ +[ + {"from": "taxonomy1", "to": "taxonomy-tag1"} +] \ No newline at end of file diff --git a/thehive/test/resources/machinetag-badformat.zip b/thehive/test/resources/machinetag-badformat.zip new file mode 100644 index 0000000000000000000000000000000000000000..aae10498e3ed41fec6c29969e841a5ab9785e8d3 GIT binary patch literal 4274 zcmb7{Rag}4wt$BQ3F(&Zly3YbL^_56B&EA%1{jA>lys2pkWMLqA!q0wKuUfXb%GfK{x95#000UA2f)GJ$;B7! zZ4VXmfcW}=OvwR6WZzMCe~R&U1;9uFIQTnw0Koq)d8R!E8xJYbKZKtwkUk{03FKdO_?wmGEPo5+V;MLg7(%#>ptKH=Y_WvtWL6d()Gc+M>E95{ea@7IOv z*k_V3MLmpVOB+O)ja9#TiyO0vq8ITkqOt#OEVJ6br9nU>$VsArf3*g*dEh$a7exme zjBP*2+O%l5UI<=))A%fKUi#SKJe#odI-s=P&!ggCu4WkrcJ4b=S{mf=!e3!kve)sl zu7TPD^_7xi)465=fq8EFF6^bsig5)xLMHiXp*N8|i7WI~eFDCA!|tjsukgU65%M0X z0zI_;!mHx%#+sDVtccK6nMTM69%ON>9%9iuZ?Lk_?D$2~?o+ZeisVbB0${r#6$xHz zvAd0x-GSf4E?#lP&-fq#sM0_%7_OMLUnrY*7ss&dfz$e#C97$6rStH0bb(C5+6bF` z!lfaq^(yZo&Z&>lrymr!lJ_aR4_JyumL9Z6S* zU7mpSp_7GqSk>Z7 zQUCzuZ&<#)S?j4q}=~bFPZ? z^Ehq0)L}jt@XYeu;P&T6fNM^yL#EqW_^5-LPToNyrAd#CRV^G5#T6J@L+yc|0*5|F z<__QXS4~X_kUvJR3QqDbe@s`JrkJtW{i0hXKIXpDZIG;6;7bz?=smrL1G?L>%IiEjN}e-LS| z`;Tn=jH4j}BEr^jNE#;@?AW;hL@+z1>42D?<2&EqkCo{ZgH4#^=)F)yS_tIkfF0^e zEw-okO?f@u4A0ND8LuGBJ(5HL>wA(daDd;YNBjlL^DOOI=ke8e60)XFSuKuS<22}yvx!o<9@x4n z+DX`d2b}k#*OAjlKlyJ99L8M}m?n?eiA8UaY1+-b&l~*@Fr&6BJ>&^@bJvufySw;t z#Gicok=Uc}q&B=}c->ud@){x89YYblU2jKBfGkdhIZ}mN7MGF@^d6XB1 zZlQb<8ZR}LfN<`IFNJG%zg|2H7vtXOTS9*pa}G^#C^yw!Btqk(z%|X1tA0XC7WPcJ z+Vm@AtMec0D{`f6`Qi#8O{=33X06q?8IHJpC94B@Km|Wm=AQ0USU^flw)HFD@CeCh znEHe)hf5kyv>o-6Kw{D*Xz)92h-@(sKW;_3ji>CgL|gzWc6f9#mHTbHRDq$znr51^ zo16Pk#(s-BiyT(+D_!K+qB5NhNi>mo##ch9=sn062eY%(3d>R?FOO{D8->voaJFRe zx$T+eIr9KH|9ZY{xg`M=qgxU)@Ol9qb_IpJ&cVp;i9YU6&#+i&(v9gj<|X92iBi5Q zKzq#GkaUJRe{2(goj>7}RLG;JZcYlWghXAZ^HL^-b22xS+lyQ%%ncw!i3y6r;$xq~ z9i{j&2NLP-d4?R*Se|=WXsDR-{Mp|q5N?(zuwkXRUl?sfdfC0Q6@0>4q$F3G^v_;FPC_VvoiZBBwp%JGw=?sW8nAFUJF^Jt!*tj2!d zD4`y8f91l)iLGS`GGB$!zH)gaM+A>-aAI*(@uqdttYQ%QZzl#_4`>+o53Twv2QnraNN7O`=4J1C8X*#zbb}Uu6~GeX|8Ir?ACM7GmU9u zX1mY6;85ZHgln=H92`wIU&_rlyphYGeaaLiWmkKa14^{!cQjIT&R@OdIHO9Y zYQm+kWZ9HxcQ(TPa7a@un){3!J1Zt?qHmKH5Q;5U(M9&ptw9$DSW5FXaeQHFi3|*z zM3!A?N-|D;r6R82Mm_qykOqerazYYDTMq(qf~^sXq(e*|oA84YCEL06J&ABmCNfEA zJXsJR35o1bfJR8!k=4Vq!ym4?ShLWfDPJk$yft3dd^3mU#o4yh(E|C3)Y=e!M^01f=TfMVNJ0MMd1^Wu&JW6la) z$3|Oj2-e&2PzVqvvQu_Bd5eG;AW)IBH`xW8LEC+ZgN%G<2MUpJV6%xD!}A*QTPUFjbCm=)2B{p2qDIoFDeQ zF$;9=lwnzx?n|sBMg%X*%7aUjn^&KX;r~e#A^}+x`?Z zB`DM)QU2_bJb`j|9ys!&b)06XJgs!&o72t687F6V_M`<{1p~Z{sWLv(=0YgUkIwqD z-oK-O^57(P#oY)awa0nwf*O9&;1_DfOcc80s`4L>O+;%kQ@m-%qgIsCFK($g)hRY9 zpzgaWgNl3f)jkHs;u)@FaDG9 z%x=vH4)L{HT^KO3UgB)pU}uYA-EmLRxNfa1KKd_gHOri$4Yx=HEExMyqukQI=1UY` z{t+rCj)HG-CpNDrn5K1b_F#a-Cfm<4j4@xf{)>5YVAN0T0ZCVTzbFXF7L*Y8h5V7V!!XPq)R!h}%%ks8!=Pa8*TPQy z(5pu2GhH`Fe|RL#j4+Fm;_ADUfVYlK_!068V+-v({KR(S>tKas#T#M5%_=9Q0j%Z7 zHPr~`hF4^9+0+rDqC*m^SloD6PDNHqA(ycx0vt~w>9Gf0TBLi~`0KY82F4mkUok$` zALRGG;w4e(i(_}{P51oiM%D80F}^0On-$$YvFBW)8iybY@F*SDkvHC={H}NMxh&o7 zKwhg$tv#=*Tb7y*kINwe8&YJcqAgCcc=)O0(~2%JiF4MoFR0g=hA{ajZzrVf=CeB3JW{gnL(okAEi0gZ&ad@=Czy0yok7JX;(8=&dKd1cqNR z72*5_3_SFYk2OroBcGi3$aT)}HZwtXiOn!G#W~}HZbM)CW1DA^o*WLGSwB!R>v}(9 zymTP@*<~R>)4AG`oP@;k4>dgTB2wy`g2()UcXeNMLK=LwUtV4P{)HUHip#J@(CAgk zQtPFw+Ga>DR6V-QUk*LDTYu062o6Kao=opR5He0@PdWq*rbeN9GS-v<_R=;KNJuZj zj%n2$^H%WjGLvA7kM1Xl%Z_i1)y*unz5 z4Z1RH-D9<~D-xJHdHRPJtVf-0M$c$)T1wY;4_R6_$;2m9$1uIy^?WR$c~(ZbGc3A# z*tOtPo1L`0?vNbE_ap2=MoHy?FKGH*yidK2R;laCUKrKCU~a(NmvIoHwq;T~EGt`A z!x}f@E5B)utvvfV)zc3386qM}Vjk=t3c|{t4|?-69tMDZ>|LRH449V9j?C7@6XO&? zzpQOy1Gql1z_Z|7_2wc^{+@Sr=zBUWN_7h?WQH4ll_+=Vve0$hLK0mz5b}znUiEsM zs716Mx2`3sqpi^N%h^d53Ids|j%JLJY(Wtf?7Qq&N`1^h7F$^?Qv9@cFj>VA_2jBi zgUIv&pbu1tR^a_o?kZ8CB&5-JMC7Kh+KpfNq9f@zJpM5cfU}D9*_dbHjksjh(cMr? zISrQd=h+P6?zf~75x-NnYBnba0RE_%kuDA{E#7|||DQtl$MOFaP2qn||DEywv-ck* k{O8<1O8Bp63Kzu%{MUkLq)R~frvvWqZ2!Hl-~V&_A2=xanE(I) literal 0 HcmV?d00001 diff --git a/thehive/test/resources/machinetag-otherfiles.zip b/thehive/test/resources/machinetag-otherfiles.zip new file mode 100644 index 0000000000000000000000000000000000000000..cac42ffef199dfb9148db6dd29e41cdd970023df GIT binary patch literal 3841 zcmb7{XE+?%x5h^&N_1gF?=@dz%?dMo9!gM)VRbdW{w_dM{DNh#(H4 z1yM!|Mi*{!o_qfHJonpwt!MAG_Luj4KklEg9uYAkfc$qeI3i2{|64r2Cl!DT;OPT{ zJG*&0Nx%_suqicw%zUrS_MhXApac*QZx8_he_w}>swPc-ntlckVkAw3B*cXVB@GQs zgoex?>4FCziR=DYCIlGkk&#oUkdyNL*8k&S{2zBWB=kS-xiSC%?QeH4I|o-cZzs5& zi-LqZ%*Xrh1TNxe+uOZuw%mPa+y9pUYlV$lk*DwY45Eci>k}ol-|ympZs9MBK5oWH zsSL3*D~!>L!^9JB2+!s4oKo3te7oF>CQFM_jo(I zjv06`c6BaBtrTd-vl|3@@(vK4_zigLKA&BgFRC65_AJt?z3vO7%yv7;Bu?KOAR!}d z!opIysSx{4bzqXo5iNVv_>{oW_I8Y1hXi8uahCowRkVdrb{5pWrr2VAeA|@YJ$!I_ zve{$-W$vCRuDL0u0;M)uDTPn~*Xrxk1$1mLi|jlNLLtkj;mrsLn%ucBfn{N>JJ8EI zMq6|fJ?ih@X4$qKa0S9g_=Dr#!lv{skFosuVc;MrVRFBji?460r_mKt%$=wa!XQ&K z#LFAfjYxp+s(aiq@M(t5q!V^Aj)JPbLtdLJ+awh};PhCjR38edh;kJ5+kobr^f+*P z8zlW@L2zsHF+z*a?7*TY&@}b(#@Cg;+n8aSg>LHj>!}M`kIhZuSn|(4eQ4a?PfBZk zGor4}se78inXLm6J)N(I9)p06`B^ggA1Ae-p2thp#mbLq4oa=DCI;tTmDmVN2STFM zj>Uor9Y3cVh&w`m_z8us4dHu zwGoKTht)3*N0>ELUZpz__7*Mn=YSM^+1a|gQV{;h(V37JK4B5kQ3#Duc`oNvz9?Jz zhXLf2bMT-yIxzV{5OM5+Y%^cUX_2H5Tw-_c_)GQ=Y_S4MBZOg`wu_f{SI%yYK7$%j z^eatl-=aK?2~9SdfEOqtRrLBK5R0(2)DF#1q%MtY6c|P@<#9J=2)J&U<~Z?zxF0-S zw_KBgOEAqz7h+>q_xVW$E?+61<+Zy{TvBv|IDKdq zLL5EhmR87Nrf*0LDu=zgNaLqX4C7|2E433lR+#EXiIbBQgvP}@MLNg`Vs@m`+;WV# z#&LW%aPSZb%X!DQYZZO;7yYc9QB65E}*~$5Ry8P_t@yW2(6G?0uPE=jyj|e?_St1GwzG0YmnR_<>)>BCi|JSVl0ok`-csP{7M%^I1hih>TNv- zn1_iYfzC;w#W-wR*ZRf6!BtkgO7i~0#ICgV+b2z7wGrkx|#5t?XU@Tz}8n zT5V^&{o}x}^ESh+?Om&03sRXcHR57`nAb^iIi7l zfsxe0)nfX4RIJiboIgfB&JKW64#kot{kRvb*g0xPQIpG+u~nIcrq+HL7JTz9gHQxK&l3Vru8J$sqK4w7x2YGZ{%T z%fzfc^f0p;jFJ*gMfW{D4UA9GZ+KA%vs`@t%DJJU-m**IEypyvnT_Kn=bZhm7%P^; zXsiur&=CA`@r|xBsuYc)%+T?A>R$viDn=~HX3wn)1sT`%;V~_tYz$3h6_Vf{yzoH& z&G-wd2?7EwS0CEdaQ?`7@HTC-lCp~(aEG>C>t{P!e9BLwaX-C6u42?Vzi8f+Y;!Wg zZFfLRB8nH!i<^`Xe{5i#>K}qDRMA8CO)bG^`+>!|S_D1_^#m4{RWi%YR3$mb-eNHq z=tq6#t>8NQXLm)UjMr}aX9Yn}ij)Ja?yJb1A|;!tMP|E|d%(g7}CRSO1z4lbE`sNln2&!@Z$9RYQMJ%B=+6adJg@a^$67si|&zKn~qToJCd<)aZI zP2i;Mbo3Gfv%uhD`0&g;?!fh4mv%XixX7y@^5NmxHa9xhiI^(-Gsh9@nkxeDo9ocl zz|i-zn(o|QNEVLe666}#0ekMAWez&O2a&dRa2?ivU0cPE_fa!t8vkhhdV!D=kuxr# zJ0O&4S)*`~7(KG^BJ~k7w!g2@d^eEqu8|7U#PW63tEHPrk@)I|pUBv{Ve$~_a5l54 zFuW~A@%l0Oe zlcR#>B&e%&) zv<{ZvHn8p&1984q^k}|wwyNOn$Y_){8%=l%5xt^}L1AOrp>E;hJo?_V61b#$Z?ziO zru^rxN=zN3|BaH7_3`(_6Nkqd-tVby;2>jlUMZRdifLG9n|$S`1&Leg=5w6Qgaiv7 zXkvrm76n)}U1d#44c#YVQsiA}ukfz!yXp;HQEAOuJhv#L4Khc!LB-`qepyI?>RH=w z7pI}7+m{AJOVLoeLGlNr_uafNWZ*fOku6?w4kIkii}Ft2Jp~W2Z@aW2xFnaZ^bnxP zS}FXv;l>)vvcs05NzGD8T+{r7wE&|A6kwcpGe0$!c#^h+xX!Z$UNX#!djOFTF#F$$if3eD$(KD; zS*tWovGC(=DIHZY&E51RF%UM9qT3F9YLVt??Pt)G9}r^_b#_b5;FF-&89#+eZ!D){ zPnyTMD_!FqHDWD97khil=$2EJYAlM%-@SNH_nyfb?N9xKuO(@&JNL9Z)mw5Zy5#AZ zhBzQ#fTj`PTMQ%P2YzDqualY zx$_}Mw#&CW0YRZ?`GfHd7)s6&|FBiq@XIhY*`oGFt3Hx=2?Xs zrFX1EAl9oe_d?Aya9t|M$KnG(6zT&*+PW)-keuZ;-(=X3;|TTlGVDyZaf{K;EQ@FB z;OiTlfebrd4&xaJ8jF`UcY#f-RFY#UBbc7`S^;24j+Jrt1W->Ow-l6Oy^)&J6`bYp zZirLFII%R~8AG45*P)m3B7IHCGvnH4Y;~BMa`qzhHmvHqB_+%1IFpaW%Ha($<@ob2 z-7RqM0WzvYwoiQnfw+4|pS%Qb?fQdHw$9qR4O!>S_RN+g6Jiy?zaZ8z{yd+7$P6S; zt-094zouR6dmjyo(_Mk`*^oxxrAnPUE%Y4L&}64|r2LX?FM8Y$)FYbqo0bwZ+O67| z=Q9&66olTgJDA;yHIgcZ4I|U~LEdl?Mxl3e z*^6ZP(y)&vLt>Ztm9B!y$E}I`VR34F0PYIPCnFyDmy*&Id)EWer3^UAbNmGA`fq4^ zqhZxxP7MJ3K{I1L0zyWj|AWSV1Kl5J{HN-}{%rpZLjGI%4-o#_^$!sKQ}tn?zt{g3 W4~_LmNdG7h{_dvVQ-wf^0JP`2! literal 0 HcmV?d00001 diff --git a/thehive/test/resources/machinetag-present.zip b/thehive/test/resources/machinetag-present.zip new file mode 100644 index 0000000000000000000000000000000000000000..07a6812f6540fd60c1a27c077326f18742418161 GIT binary patch literal 3941 zcmb7{Wmpv4w#NsgOBxC3PDv$ILSA6VVE{?#9AajWaR@=$K~fr|Q(A`ZhG9@ac?Br} ziJ=6BM!0gHd(L^E`|bSKvvzz~d++uCw0}l=w}==4WPidP2{i`%pYmh@09pVy0Alax z=I-qbw|7;P^nm$zgH5Rcq^=TZySx1zc02=UJAVWvfPiS@769Lru6U{XA?w9h0(cuBBs!X=qEO zW~Lkmwqtq0;%31|VB^CTN*-Ibr^0q)nw^){9l(2GFf4oV#aCrn`x13pYK|)L8f*_Fjd08@9^xR zclzOEdE&x%&BO59ZLFmwBFbA^ug^1m+2}8Ao%0(6tGmKUx3Udkk~*r z?^B<7jX~vX1xk0WnZD24#qE>6Cu2|dV)RO(b^?cffEVBJtrNdtZ{3%(EAz!Q6G2|Z zdUe+W0hBrJCs{=4d&9(}BrRB2Di0N6-?;%y{AFCr0W~=-c(lD8E8iuFm@vr(zEDG3 z2zI|ymUO<_9BuQv)imO7Y4Ohyn$=TN$8`K4L>@JJ#z4Sw@ zmr-Mz5!Prjmw`lpE!6;tfJ=yAe8Q!~UX3Ylgkz<1Rn{+Rt)rp^~<>|#7QRb!WeHg}G3Dty@4 zM7azIwXTeI67$=D=AQI9@_6ef|7}5VXVV0s#cy_C(HCHvdP(thW#BGm%yyxdI^lZy zg4T0$lPHetqt5^uxA%+EM!@V=Pxtf#&7iFI;fTKO*P|vNcBg`D*@6#Ww4h$cOEx7c zCNu|SHdtf*b4V37!qSoOR%%yy!Y_dEaz1e)rkJt2$^Vv#)Ja5ODcb~9sQH%WR!F8z&AQDg(479+=5&yf9YwllrUU!cklRf&QENK zB1^M1!z67FAK$LL{Th8HHKO=;n)tp&MH&;DbRrQiSWKb>84`>`*jZ|aWGYdYMK%kL zA(--cnllC6woG%K`9VC7o~>K1Ny8U{9gD43y z;=+*l*k?#bSs~1hbeemvA@?MXp8^LDmQ?wM|GNWBm?;5jSST70>olS~?O9k0Jm4tQ z*;#uq=I;Lxb&U^5_VRx>WTe*Ih8$nc`!-FDWvuN>xZg=!Qwn0kVB~p=1Da}A4Jf8d zy3gL1`-RMIxU##8ZYJcBN9YTGHXzzso8M8lF2Z{h4a!dX;^~^Cc1Sq;kH5=(WUc&^ zPu=^|7DRTXiz1wdzgqRSnFGwjB#`Vb$)Lq}Y)8-f#lpc=c7kfk{*$Dhw9dOHEfZPa zI{D8zjD3I7!aZsqOrZA82+mHdX@ zodL}9FUmP|AKC%yQv8xAuSf$TsYPnUfqPV}GSOT=$3M&tgWn#CCx7wdS+L^ZteZf6 zS+0t!&MGps@yoQ}e~{@{-}TsEH1@d9+;D3{aIZu`jK45CjP6@PKBt0;K}69j(z&TS z)s1gW?R_>GgkMiIRtIyXAc<$0nAJz0WYvIC(juwofoG=y32%W-p+zvu#rIJzO_hz7 zJwW$d)0kE^&f8pb4tHX$SPo;bwxAIM@TS4)2lC9n7-m?4Sdz<`TNe&A zY8b#{+CtbETFNV>zUXrIp z;Db<4WMNq)wd_t+mUrqe5qE`t05Wd{H8{MuFDh-ccGo{U&>E#gIn3&@irgtyww+$y zk`CixrILZiQw0E$(CBtWc!aDSRUI-b?B1e_H9J!$ZKw*tYvY+m63UXfT+O2ut|O{y z@uI*VnjEpsGRBQu#%}A|9gb12p1G`AFbH;W%QixVp8S3}4KZ*6Gzj$pifrEkKo&(A z$Jg8#a}M}2F4}TMq|T0?Mwld#i?-VdA`WJO!Nu|6S@}Ex>;0~s@}BXLSAk?>W3wIZ zbg&a~HS|Z$Bi1!H1l~8#v7?D$;8zXZxr4AA9Lp`lJ-h?<+C9q}ad{6SY474bZ2Y>m zik}>yX38@9+5YtcAulR#R7!V1DBHG1?kYKPWC0=Z5jS#psL^^qfd9UsD$~^Rb#>Iz zZKP;I&66QgwqBTmHFY?f*>o6S&Mc27t$+L}xw~%4pNI8#Wa!ZbYqGmZi1N}g=h#zj zdMPEvnyDDvJ{{YA#~srH!<|k=#0(OwaOM7t^5ADiJyl8EcwRHzstQ#6T!wMzzetas zB~agwFtu)o%$q2Oy1b7+KrGI72 zfzg=n;RiC`y`h2g6C`%WU5cS~umW}g4Zj(P3bbM-3S4s3gm%X#qP5v*!rN}qE6M5? zHJ2ai7MbMJ_n(!*r9Ap;9)oQwe*Lb(G*}P5Q8u(W{*iRzXrkf$o{9oz{iy-+R-j{)d_w?RZZ|aFoYt`bt zLm8u=HL(pUsW|e>Mhe!<+I_z`4Kdxm)W5Y91Em`wdqnck-TOioo|_ff29b9hXK`6n zaQ@*Xbb!t1(Td=fTDsChfFkRp@skD{Yb?u-TS~^YOQrGAk8oA&(@Hjc;t_~IT)k$Q zrG54HD8anFj_f!Z!P$-2+`>SHmZ2{@gXA_@zLp_(@)YX6n>Rg=I@cMLQ4xY~YNXIS zZ%Zom$jo%+Wh#!!PheR=MQgG$d3r%Bb6-e(0JJ1ZE5F!;m@d(SNFG zkiYbMW}|L6^n)B;&&|;f8Obmu#;&Zi_$I~wwPPbu#DkgfnKphQGCS-tR54lUQjBD^ z(n)y`XE}O7H_Ef(6|KYd9YgiRvvwFjSCq9-U- zk2Q`yyYpCoNCDrR_P~p|pK2Tuj#Onx;l#6dlyf)~wCfiiYnYl#{b{0}_lV_nMuNf|r(s5lbNYL|hW@n2Hcw?dxgB^i zPden6f#)$^y0Go6(jbuObWL$~LSotbYW{d}S&dbZeWB;qwcmAv8obwEo}K;qh3v&j z$#X<704o*ffoW>C=`u5w53lm(gOBW%?{)(ML(mEblN&IUyc7OOyNJQ(F*s1(n%3W5 z&V~jJ>qFVGF1ll0i#(oZ6={~)u@bemS%rBNX{Lc2Qb9fz^#C!b4-9GNp%hGfmfw1t zVMCrX#2;eNoo?$MtDRMzz}Ce-FfwC3=5#rRXCP=US=!uXZ&{_1`t)`j)3;tH$R3<) zWt20;t_Q>|1-`Y}NX_jD%65D=$|Y))RQCJ@!+;Cq5Ms1QUt9XZsO|+@1Ln58gDAZ% ztNLzf>9RV`_ydtjcvEZz{`_-q8{B)Clq!jBXka)1_uyyLy`;i@)iR7&_7k~2iH&=)LzeXhJu0REBNW&k}WiH(odX8&o($fYK z0jZAAK92+Sh?f19r9_QRt4`+mtRxFX;S3H(vpbQT0TJa~n_Oqg{cHgiYnkk_LX6i4 z1*IU3$rn~=(Jd(wVb{9}FqPbKBkanFKF zDVfT>>*1I(1{~!%ehPIBFw!F>d!u31WKInL+@P7!zoBj23Gr`eqa^?UsQy5kFVq?4 z><#}9u&w;+u)9gzpZtHo28pky>F1`>4Cv}R$caQRey$!1jVa-k4-GYaQ+OFz&yZ|J z5l(!v8zfEv=wubqb|m)+9SnMPd!?%0QNgHbba45VEAl119O-u2{gMuQiM9)^@W#P|9R@NOng4Nmub%%;nG(<}3l01v>| z(b>b(&lTb5t|;XNf8+--rvZ>bCD9Hy``R6N`_T@8NGbpU@g@-f@SjVxX`5f)E$aDS zEKwUSn{zb}GC*4sMmnU{ML=|u$Y$~&`Q`qnVt=2Z`)rXbHxq*<(Et$^?@wNX@4{ZB z1SQyvG`k2?!k7Xg(}9(CKE#76ZAnEG;?1fLMgr`!NEvUIqW!&&T2A#j zb_cw(yjS@H_))N$LulA!OS2GTP~+h%MD%T{15mp3}9? z9gb67;u^;~DqPZgUM?_(w;f()s2+k5_nxOhq^kmgWG_J^MsoQRy_U5`Rdban{rP6b zUOzXF2}5t@?yjYn)gs+QPQxG{{vo2D0YiTJPv%w^N@~Z$d`b-Juls|jay@@$6KCuX zk&uyoz{1mbsgVb+4G@y)F&!t=x~VXLV6CDC5}FZ5zrOX=vEXIP43p8#J0HJ z6Xa_brz^gN9uEv`w`t#bdLDcPnO$ zFx1=v`HWJwIRxOp=9O>)e3+#-?TTGWprCH-RM6$gHBCbdxtghzgJ96A7#Hz?O<3N~ zUT0oE!<4_Q2ySedA$0^S4y}5F%+oF@Kd<)R#EjZ6_Ru6=&s@-ZZ*39BlYe~FkH+o) zpt2LRAnNX#QPd90ZX1f~?Rq(41_rtm=ExPkpVoo-oGjaws+!RrmfK-X4bOe6u~9b8 zghXkb6^TCpA}a+XNz5ZU5dQ#xZfbY`;H>7_d5leY`L}hSqS+IHJo*!3k8mLw?I+sv z;1K>>PsD4szMb5Pl;U6QoA3B6EIgqzlp~NAhwslsihCD9sF@B*{sNA`;>rhB(U!V|HcI zJ@brtCUF9kI7GOV>KFW9?GVB&NpQnrald$n3Ds%$;(F*IXOZ6Sy5gv3pa$w1AC%$~ z_-4>Vy}31HY$gB83=NjKt~c>c2T5%im;-~6=PL*nxuD0x%y7N-u}p5HBms*^W7dyex;8hTtGZq z^Rt@=EWjm0fNm+^r37qy_r}HI;Z;teTI#|5hK0?xyC)kn78Uojuz=D^iw9WeiKwu}CX1--jvzW513dR{7uxPJImrPk5(~H#RE?sFpOMW^-Kc4NZSMGJi%I0=cwksOf1j+-B+R3%91KMw@PsXP$X%3kMe+_q@}M zI9s-(SgbvG*a-4$>6N}JsvM1?%F^?F7+3-~E=4Zi&Yj;72{mcx$75O}IG8?ER7pd6 z@uEY8bcq+#Qv?J$9*^kPo(o1VKz8U;RaD&_fxGk_IzKwlk~0BX%?BA(@>Sz*g(VB- zWLwiwo_j+&QZf8^e%!Q_q?w^zT3|S?Sj_<4KeLRO8vvH(>kvFbY9_Im#dD*FD5ed{mfMhhf zO$iYt=RjQ_k{x+#$qfo*>7akAO7PNj_O7IgR33NpNTvI*ntFm5=$kfYT(hicBe$u? z#!kEQi)Rnr)~uL>I(g(8VZ!%+J(=+}asf05_X3LTUjx8a#hEA9JQzz(#0oCPW>vJ_ zL4a0-G>MzO%f(j$!Uji3;Gbs~@CI%4xp&BWCq!R`l8=tgwRTte~^r$2d)66 zdCp-m6A75N$mdP(8?&bH)KC3ASYLal0e!eO&_hy;kAXGUk!s6FIXTWk)rjtNbngvM zY&RTtIu#W=K(NZ4_x**J04vH`HEHv4?M#arNa<5K#;NZjBW9AYQ}oNQ_oc;S*G_rh z!tK7KN^(@_f`THfG^J_j0hTb#moP`yPd0qWTRr@`sCnSvqVpYQV#EGi)SRSHmqPWk zTgrI5r|ZDJKeO{BTjf!uhtP~cMh=#S+tr&HW*Z9dF=fd3%$y6SwcN+|XT5nvix414 z>WaS zg^*y;8%=CD+NuPvWvH$zt78}>CJnhG>-)UB=ZQVmbP3pqqyc66(+yz>~F+meFoHy`0cY-YDk6p!@s zl>rhQT`z;5G}>HeTXEi2HmzGOONhCPs|L;}+wn_8AwzNRw99QAYreh^%HMC#iKi8s z+l4W*Ba+x}r~INNrJhwm(TN41Ylo4TZNwv{tTC^?Q*EPy zsP)BjyY!}epL;Mg-?~SvgY4$)XdT~ntx=ChQ3raJ4(ltLuG9Yj9eyrL_t;g`?b2+` ztLj!@WFg{nO2mZ~*{JDBQ_SssEcv)-Ku+PBW&O4NnT~O60Zhnlbl24jFYbA&bwoH^ zohglz$k|I+NJQo8TvPCGsR;>Bz3}2b4Hin^*=Tto$0~`Wyz{ z3rL7FPRpa27=Ooi%=R)fQDL6TI5X8XIo0g{K(2Rh!iqea7eQ4Z`&o|u=S z_ZHYin`L)x#h`X;aIa$RbVx%Q_>t8+fH>?CJjB6EIh^FIpoNZUQ=Tg#(ATId!`?Ga zH@hN{qf?-NcosV9ayg1;B4{pM-r56xSfiGncs+*c-KZA=hUeLu5c^yr5+CVBO-Tt)K>rgZ5!@-B; zB&`nH4%UV2WGf|+OipKu8_`@rQ5D=<+-E9%96?s=SwJ~q=4+&aa+p?1)u>T)`T*Du zA;v87W-`+l~Jzbbs9VzpF9w&*}eb%KuFNBZdEb`G2Ro d{+7tU)BkUZXktJ@`e!P_-`)Ou=b-y@`w!^JiR1tP literal 0 HcmV?d00001 From f2248d3a05e5e411ccd19b29b25b7fdc7e22454b Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 24 Nov 2020 17:19:23 +0100 Subject: [PATCH 041/324] Fixed schema --- .../app/org/thp/thehive/models/TheHiveSchemaDefinition.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index c0542d306f..ad5c3bb2d9 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -112,7 +112,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { } .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => db.tryTransaction { implicit g => - db.labelFilter("Organisation")(Traversal.V()).toIterator.toTry { o => + db.labelFilter("Organisation")(Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o => val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "_freetags").head Traversal.V(EntityId(o.id())).unionFlat( _.out("OrganisationShare").out("ShareCase").out("CaseTag"), From 6234bc0626882725ff846fa70fb6ab1048a8ad6c Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 25 Nov 2020 16:10:54 +0100 Subject: [PATCH 042/324] Fixed unit test for taxonomy --- .../controllers/v1/TaxonomyCtrlTest.scala | 93 +++++++++++++------ thehive/test/resources/data/Taxonomy.json | 8 +- 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala index d08034f2c9..6c320635ac 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala @@ -2,11 +2,11 @@ package org.thp.thehive.controllers.v1 import org.thp.scalligraph.controllers.FakeTemporaryFile import org.thp.thehive.TestAppBuilder -import org.thp.thehive.dto.v1.{InputEntry, InputPredicate, InputTaxonomy, InputValue, OutputTag, OutputTaxonomy} +import org.thp.thehive.dto.v1._ import play.api.libs.Files import play.api.libs.json.Json -import play.api.mvc.{AnyContentAsMultipartFormData, MultipartFormData} import play.api.mvc.MultipartFormData.FilePart +import play.api.mvc.{AnyContentAsMultipartFormData, MultipartFormData} import play.api.test.{FakeRequest, PlaySpecification} case class TestTaxonomy( @@ -22,7 +22,7 @@ object TestTaxonomy { outputTaxonomy.namespace, outputTaxonomy.description, outputTaxonomy.version, - outputTaxonomy.tags.toList, + outputTaxonomy.tags.toList ) } @@ -39,15 +39,18 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { InputPredicate("pred1", None, None, None), InputPredicate("pred2", None, None, None) ), - Some(List( - InputValue("pred1", List( - InputEntry("entry1", None, None, None, None)) - ), - InputValue("pred2", List( - InputEntry("entry2", None, None, None, None), - InputEntry("entry21", None, None, None, None) - )) - )) + Some( + List( + InputValue("pred1", List(InputEntry("entry1", None, None, None, None))), + InputValue( + "pred2", + List( + InputEntry("entry2", None, None, None, None), + InputEntry("entry21", None, None, None, None) + ) + ) + ) + ) ) "create a valid taxonomy" in testApp { app => @@ -113,7 +116,7 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { .withHeaders("user" -> "certuser@thehive.local") val result = app[TaxonomyCtrl].get("taxonomy1")(request) - status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + status(result) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") val resultCase = contentAsJson(result).as[OutputTaxonomy] TestTaxonomy(resultCase) must_=== TestTaxonomy( @@ -178,27 +181,65 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { (contentAsJson(result) \ "message").as[String] must contain("formatting") } - /* - "activate a taxonomy" in testApp { app => + "activate a taxonomy" in testApp { app => + val request1 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy2") + .withHeaders("user" -> "certuser@thehive.local") + val result1 = app[TaxonomyCtrl].get("taxonomy2")(request1) + status(result1) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result1)}") + + val request2 = FakeRequest("PUT", "/api/v1/taxonomy/taxonomy2") + .withHeaders("user" -> "admin@thehive.local") + val result2 = app[TaxonomyCtrl].toggleActivation("taxonomy2", isActive = true)(request2) + status(result2) must beEqualTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") + + val request3 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy2") + .withHeaders("user" -> "certuser@thehive.local") + val result3 = app[TaxonomyCtrl].get("taxonomy2")(request3) + status(result3) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result3)}") + } + + "deactivate a taxonomy" in testApp { app => + val request1 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result1 = app[TaxonomyCtrl].get("taxonomy1")(request1) + status(result1) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result1)}") - } + val request2 = FakeRequest("PUT", "/api/v1/taxonomy/taxonomy1/deactivate") + .withHeaders("user" -> "admin@thehive.local") + val result2 = app[TaxonomyCtrl].toggleActivation("taxonomy1", isActive = false)(request2) + status(result2) must beEqualTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") - "deactivate a taxonomy" in testApp { app => + val request3 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result3 = app[TaxonomyCtrl].get("taxonomy1")(request3) + status(result3) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result3)}") + } - } + "delete a taxonomy" in testApp { app => + val request1 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result1 = app[TaxonomyCtrl].get("taxonomy1")(request1) + status(result1) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result1)}") - "delete a taxonomy" in testApp { app => + val request2 = FakeRequest("DELETE", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "admin@thehive.local") + val result2 = app[TaxonomyCtrl].delete("taxonomy1")(request2) + status(result2) must beEqualTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") - } + val request3 = FakeRequest("GET", "/api/v1/taxonomy/taxonomy1") + .withHeaders("user" -> "certuser@thehive.local") + val result3 = app[TaxonomyCtrl].get("taxonomy1")(request3) + status(result3) must beEqualTo(404).updateMessage(s => s"$s\n${contentAsString(result3)}") + } - */ } - def multipartZipFile(name: String): MultipartFormData[Files.TemporaryFile] = MultipartFormData( + def multipartZipFile(name: String): MultipartFormData[Files.TemporaryFile] = // file must be place in test/resources/ - dataParts = Map.empty, - files = Seq(FilePart("file", name, Option("application/zip"), FakeTemporaryFile.fromResource(s"/$name"))), - badParts = Seq() - ) + MultipartFormData( + dataParts = Map.empty, + files = Seq(FilePart("file", name, Option("application/zip"), FakeTemporaryFile.fromResource(s"/$name"))), + badParts = Seq() + ) } diff --git a/thehive/test/resources/data/Taxonomy.json b/thehive/test/resources/data/Taxonomy.json index 500c39c010..5c661448dc 100644 --- a/thehive/test/resources/data/Taxonomy.json +++ b/thehive/test/resources/data/Taxonomy.json @@ -3,6 +3,12 @@ "id": "taxonomy1", "namespace": "taxonomy1", "description": "The taxonomy 1", - "version": "1" + "version": 1 + }, + { + "id": "taxonomy2", + "namespace": "taxonomy2", + "description": "The taxonomy 2", + "version": 1 } ] \ No newline at end of file From 872a7d5b6bce0b3ee65d230834ea007a4f7120a2 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 25 Nov 2020 16:18:57 +0100 Subject: [PATCH 043/324] Fixed user permission test --- thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala index e1831040c5..8a5773b794 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala @@ -109,7 +109,6 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { Permissions.managePage, Permissions.manageObservable, Permissions.manageAlert, - Permissions.manageTaxonomy, Permissions.manageAction, Permissions.manageConfig, Permissions.accessTheHiveFS From 91861c56afeb7ac7c6ca72f90cca9295b37e0e09 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 8 Dec 2020 11:37:36 +0100 Subject: [PATCH 044/324] Review changes --- .../thp/thehive/controllers/v1/DescribeCtrl.scala | 4 +++- .../thehive/models/TheHiveSchemaDefinition.scala | 1 - .../org/thp/thehive/services/TaxonomySrv.scala | 15 +++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index 33e3fae52e..193aa4d5c0 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -40,6 +40,7 @@ class DescribeCtrl @Inject() ( // pageCtrl: PageCtrl, profileCtrl: ProfileCtrl, taskCtrl: TaskCtrl, + taxonomyCtrl: TaxonomyCtrl, userCtrl: UserCtrl, customFieldSrv: CustomFieldSrv, impactStatusSrv: ImpactStatusSrv, @@ -102,7 +103,8 @@ class DescribeCtrl @Inject() ( EntityDescription("customField", customFieldCtrl.publicProperties.list.flatMap(propertyToJson("customField", _))), EntityDescription("observableType", observableTypeCtrl.publicProperties.list.flatMap(propertyToJson("observableType", _))), EntityDescription("organisation", organisationCtrl.publicProperties.list.flatMap(propertyToJson("organisation", _))), - EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))) + EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))), + EntityDescription("taxonomy", taxonomyCtrl.publicProperties.list.flatMap(propertyToJson("taxonomy", _))) // EntityDescription("dashboard", dashboardCtrl.publicProperties.list.flatMap(propertyToJson("dashboard", _))), // EntityDescription("page", pageCtrl.publicProperties.list.flatMap(propertyToJson("page", _))) ) ++ describeCortexEntity("case_artifact_job", "JobCtrl") ++ diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index ad5c3bb2d9..e0b4f9f09a 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -102,7 +102,6 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { taxoVertex.property("namespace", "_freetags") taxoVertex.property("description", "Custom taxonomy") taxoVertex.property("version", 1) - taxoVertex.property("enabled", true) o.addEdge("OrganisationTaxonomy", taxoVertex) Success(()) case _ => Success(()) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index 2051c64930..aab26143cf 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -27,9 +27,8 @@ class TaxonomySrv @Inject() ( def existsInOrganisation(namespace: String)(implicit graph: Graph, authContext: AuthContext): Boolean = { startTraversal - .has(_.namespace, namespace) - .in[OrganisationTaxonomy] - .v[Organisation] + .getByNamespace(namespace) + .organisations .current .exists } @@ -39,7 +38,6 @@ class TaxonomySrv @Inject() ( taxonomy <- createEntity(taxo) _ <- tags.toTry(t => taxonomyTagSrv.create(TaxonomyTag(), taxonomy, t)) richTaxonomy <- Try(RichTaxonomy(taxonomy, tags)) - _ <- activate(richTaxonomy._id) } yield richTaxonomy def createFreetag(organisation: Organisation with Entity)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { @@ -59,7 +57,8 @@ class TaxonomySrv @Inject() ( taxo <- get(taxonomyId).getOrFail("Taxonomy") organisations <- Try(organisationSrv.startTraversal.filterNot(_ .out[OrganisationTaxonomy] - .filter(_.unsafeHas("namespace", taxo.namespace)) + .v[Taxonomy] + .has(_.namespace, taxo.namespace) ).toSeq) _ <- organisations.toTry(o => organisationTaxonomySrv.create(OrganisationTaxonomy(), o, taxo)) } yield Success(()) @@ -68,9 +67,9 @@ class TaxonomySrv @Inject() ( for { taxo <- get(taxonomyId).getOrFail("Taxonomy") _ <- Try(organisationSrv.startTraversal - .filterNot(_.unsafeHas("name", "admin")) + .hasNot(_.name, "admin") .outE[OrganisationTaxonomy] - .filter(_.otherV().unsafeHas("namespace", taxo.namespace)) + .filter(_.otherV.v[Taxonomy].has(_.namespace, taxo.namespace)) .remove()) } yield Success(()) @@ -80,7 +79,7 @@ object TaxonomyOps { implicit class TaxonomyOpsDefs(traversal: Traversal.V[Taxonomy]) { def get(idOrName: EntityId): Traversal.V[Taxonomy] = - traversal.getByIds(idOrName) + idOrName.fold(traversal.getByIds(_), getByNamespace) def getByNamespace(namespace: String): Traversal.V[Taxonomy] = traversal.has(_.namespace, namespace) From 4fe311197d200f9a8a5550a257501726d18a90ab Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 8 Dec 2020 11:50:07 +0100 Subject: [PATCH 045/324] Correct Scalligraph version --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index 1a55a0db73..856e64f3e1 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 1a55a0db730460c6f548695251248934196b6ecc +Subproject commit 856e64f3e1b262821a9d5b8c402ebc13f7562f18 From 65189f5e89d6bb8454523d8f8f72e88aca96f7c2 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 8 Dec 2020 12:12:24 +0100 Subject: [PATCH 046/324] Fixed build --- .../app/org/thp/thehive/controllers/v1/Properties.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 92213ab6b3..b6dc5ef69c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -498,11 +498,4 @@ class Properties @Inject() ( .property("description", UMapping.string)(_.field.readonly) .property("version", UMapping.int)(_.field.readonly) .build - - lazy val taxonomy: PublicProperties = - PublicPropertyListBuilder[Taxonomy] - .property("namespace", UMapping.string)(_.field.readonly) - .property("description", UMapping.string)(_.field.readonly) - .property("version", UMapping.int)(_.field.readonly) - .build } From 0bfdf13c3d0cfa7f5eee29f055f37064d44d6de1 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 8 Dec 2020 15:12:45 +0100 Subject: [PATCH 047/324] Zip import works with folders --- .../thehive/controllers/v1/TaxonomyCtrl.scala | 4 +++- .../controllers/v1/TaxonomyCtrlTest.scala | 21 ++++++++++++++---- .../test/resources/machinetag-badformat.zip | Bin 4274 -> 4952 bytes thehive/test/resources/machinetag-folders.zip | Bin 0 -> 4578 bytes .../test/resources/machinetag-otherfiles.zip | Bin 3841 -> 4111 bytes thehive/test/resources/machinetag-present.zip | Bin 3941 -> 4595 bytes thehive/test/resources/machinetag.zip | Bin 4076 -> 4618 bytes 7 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 thehive/test/resources/machinetag-folders.zip diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index e81c47a098..39bc1c0a49 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -91,7 +91,9 @@ class TaxonomyCtrl @Inject() ( .asScala for { - inputTaxos <- headers.toTry(h => parseJsonFile(zipFile, h)) + inputTaxos <- headers + .filter(h => h.getFileName.endsWith("machinetag.json")) + .toTry(parseJsonFile(zipFile, _)) richTaxos <- db.tryTransaction { implicit graph => inputTaxos.toTry(inputTaxo => createFromInput(inputTaxo)).map(_.toJson) } diff --git a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala index 6c320635ac..68c4627b49 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala @@ -148,15 +148,28 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { zipTaxos.size must beEqualTo(2) } - "return error if zip file contains other files than taxonomies" in testApp { app => + "import zip file with folders correctly" in testApp { app => + val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") + .withHeaders("user" -> "admin@thehive.local") + .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-folders.zip"))) + + val result = app[TaxonomyCtrl].importZip(request) + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + + val zipTaxos = contentAsJson(result).as[Seq[OutputTaxonomy]] + zipTaxos.size must beEqualTo(2) + } + + "return no error if zip file contains other files than taxonomies" in testApp { app => val request = FakeRequest("POST", "/api/v1/taxonomy/import-zip") .withHeaders("user" -> "admin@thehive.local") .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-otherfiles.zip"))) val result = app[TaxonomyCtrl].importZip(request) - status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") - (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") - (contentAsJson(result) \ "message").as[String] must contain("formatting") + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + + val zipTaxos = contentAsJson(result).as[Seq[OutputTaxonomy]] + zipTaxos.size must beEqualTo(1) } "return error if zip file contains an already present taxonomy" in testApp { app => diff --git a/thehive/test/resources/machinetag-badformat.zip b/thehive/test/resources/machinetag-badformat.zip index aae10498e3ed41fec6c29969e841a5ab9785e8d3..f18619f1843e27bb23dd788ef08717b00ec20bd1 100644 GIT binary patch delta 844 zcmdm_ctfo|z?+#xgaHH$N;?9<3@9PWpumuun4FQBms*mTuA7vYl9peTn^>YB8p6xK z-rICOz7L2&bZG@Q10%}|W(Ec@fiO}F%}8+!Bh{g1T0%^11eylJD5g%sZW(!+E&l2xzj*$w=`uxko^PkClOyL6wPtK~@lCGys~u?t%aS delta 285 zcmcbiwn=gF1wJVyJ_ZGbq{Ni8{G!~%61}YA{JhW*UIuoa4}0Q4SmMK;%{u&_8Eu$Z zL>M@LFte&CkO2zj8#6En0QKi4CTC>krIsY7TOv$I*WDT~dw5U0;*qTz|5WmNF*1oT zLyRa%sR%UhPWy9_k%1wToq+-51Q^)T2x8%P(BuL^iOFvTMIu0U!@!nCT_y&CE{QL# e;AUWCdBM!U044&wS=m4)FahB%ApM9R!~+16rA+Yv diff --git a/thehive/test/resources/machinetag-folders.zip b/thehive/test/resources/machinetag-folders.zip new file mode 100644 index 0000000000000000000000000000000000000000..f11bf049dbec9bf6c70ae2539ab0bc0ca5e5a2da GIT binary patch literal 4578 zcmbW5Wl$VSw#Nqx7TgKJf)1KM(BLkEGdO_&VQ>NjcegydR^2^a-PQf&*M0iKIsf{rDWRYe1AaxZudn*=&Hpa=09t^VgPom&J%@{_ zyMw)hod=h;77hSi7rp=W2fxYR^SwJZ010&$1pxTHsQsp405JZA*2n|^(EJ;kov9hb z%HG1o)RNQM*}?w*;J;t<#|3|ZkL!=Z7Mn?yzq6@zq+07le3Wsmb8g^K&_P7-)asP7 z)8Z4yGhtE!X2He9)<5aWf2?>9XGWpZz=o0+I)-}eBTzXVCwYHjEB-(TD?mVl?ZjNV9QNdz%E~Xqv6VYWpXv>&& zFa7~PV@eSOc|ouqGgBnc`Kn$RU!HuQ>#VF?z;}y@B7EXR`J}9x{b{+@xqN|*F>9WSB_bt9Y$`fT6+69M2Tsv&M4C7}C{;OYxtV946#`t6B*{CV zEVsOfb#-v**Z)X4*ofY~Vm&ACHn^eQJ~hmn9jFP_Dk}@(_jDp$RLsL-OckzoObUF}{e33gh@my$19$eADmbhSW(CkLe-d0O!o};^)ovkrg^OF5(JydlM)w5g=^>%m zJsaqg6?80VjT6N5vF8TTOIDnQLtC|0-^wFNPd znAJJPvk1B1exNzU-9g{70hru~XFeJcL9N+#nXy(@Y~rSlMiZ>Nr4?0F&+x=VDq{(? z=J2Xxr9Fhur>BSVtL$`ji*&JF@@8lGv-6;pZZ@FoZ7v5MpLYPI7l4Dq3z)=$IO)7k zIUPxAT}(1>3R2aAx}wkrqAz}JFLreFgk`fpsDkj0*NrgwEZxm)EI6QR4NQ{}EwQBR zx5lAMEFufosrCYr*rOAya%G1e1>HHPR(8LVl5EmM$v!oVYM`Z%YwY4=ChzW_}`PSw@GT150$5H+*ClgWO~L zg0~BDGDSBJx25Mi3+`fiGV4lGZz@{ie*6_QL&~8{y69w!BBw?ZXL}xtM}BwW`0a(- zVXeA=^IP~iZ(&lXG&=GR2)aiP#D=F97$rVMuTW?-MK>c~J&3ib!6Nqtt2d*0zM2&J z`17IprYXq^-SCrb)}!z=nFzDU&C;E`kWMeN3l}zmRRJ^uYc|eg!hxJ1kXPRac=X7G zz>&EvlN_v;_jCb2Til3lj~B%~8G@z()QA;l36ABPIa4ctT+JbnceB8nD%_$gZO;(j z6(T3fTLe|F^xs%I&k4MFClNV}F2B?eAKhiM0+?jIOoiy#^l2gDn2i%>>8ZIoe{_~Ti% z{+!T#rJqLBU-{q$8G>gziWeFt6eRW8hjNazl^w}d$LojhTDaD; zz>yQ}8u3?*TQ6t*A;PPO?&q=kqi(45%PLw%M9VUKTYb&n`gG`nzi2qi>E#IF986Hj z+)MJ1U!Bw~jNXdMMMm9`J|^+qA{*DeL=*9q6ot)mMZXE)e$DMIAyrq9m$J~p-9Gxd z=q@==H#F4fGlJ9Qz8Rr#)vzgFz3!e zqM7s0ZL!W4mZJwJbq9r93R%H)%hMi;22d@+pm&E$2UjLvDqTbcDHUe%f6gRq^DwJq z98r9@p0hspHAV`d^pqPD1|km_XgC`tmPx)zSvAq+6RnpRrHa%r{+PNhiSX?IG%VXx zv;w52^v4EzmS)htS7w57FFf7FYcGhQ25>?@8;rfrLk$VcNiT{=(5{`Wd?5|{X8e{Q zbPgj9{gD3J?qKTaNW;ky36R%no1erJFP}a+8FoG57PoJZ&nVb<}w?@Z72; ziRE4P-mNdiq&K+S^ic3)az`(X@-*{GpL%aNPf8?nC4SX4%SH(ouE8-upbPVK6MiY7 zZhK#F3lD)R)Gl$##^2@?3KOpo`=*H5;u>r@>X z=DTl+N6kDwX!gE(zN!teo`LTwvfEmbq|xa0o);-(?6V|+s=fG+&2x~iybHc#G`Qtq zu2*5RTz&Isj(bI)=fXj;-t)<;KE}p!nZ+E6bot3eWf@}WF5$bww)W)~tDL%OFanY4 zC`ZPIsYB^!H{GOY6Vc!-j56OVzxMjBa);GP$3-lbj;iTIO=H{X)XB!hu9I?B8YN%T z5ZH0H$}!yG$UPoJsMJ|MqbJU3y^wjQNk)15F2FLE+>`s< zgeuCLT7n(OWO1=3D@geK9id-s|NgCDV)9fAMiRoQY5S4vp|~+u$z5nB9UIz9S&<_d zK{Dw9rdgxWfEvT>EW>Y8+ujm_czQ%T#ccgww$J*-b!6HCc9OaC-{&1Wy`$uSAA)^D zj}6yq*yx!FD{(imGT~4L<4vw*S_v>(RALB!URm#kD`{|!x4Ki=rDV+uO{m&a?n*wU zvFj4;>GM4TFIC?8$$BT-)e9_rJMHYS)ex;8R>#p8(K8Q}=3`2-gMEyd&dFMYDVf!+ zj9|gK!A>t@=hnJ9Z%aJvP@^Bieqsw0_dn6ZeRfudSaDZwXl{rSwp!??JYIC&@2r2F z%%hgOHF~_YSL{;RAXYi2s;cT%8`QZE{IPvgXWwR*bWB@EWvoDWq<>?g_` zP!4A^Q(KGwQr^CJ3;(_Aek8`=qVpQZYEZH59bs1^yVFn_|M|}5ckKN5bHhCi8QcQ2`zVt9lRYvo z?}NnB=AMb&gff@#)=bjqFzfeAoV(IlGa4y1*HkRLt_2`a(BR-tlp=bTk^&%CI1~TSO9rLWB{gxRE#A61<)y zm+3Or%2`+fV3YpzL&eyHc}gn6CavaO3CEPdY9mqHqkHNvP5}Db&vl4*YsMzC1mQ`Q z@B6^@KCCq~V(7z*X7VedF#BmwiN>pVvU<8ID`OP~wD28txcs0pxP1bc z5F$iMmFtL-1R!S=3={ltSw>UwDtv!~yex`2w-s;aSyQXY7KRi*X?L@! zy+ww}k^|G|n?UE>%ETCH)O(H-VTZZKv0G-hFm&|}!O;Ej&5~r*AZN>cO~)*%o>Cz# zvn?YliLu301#2JZ-7NkH2pJq<&(e4T z_!B26wDD@><#qfs`AY4@r)Jm7!GZeEYxHa!K!Z%w>7Qd-cV!_$6@_nS^l$CTL=C+r zDnX(=RMau22>@m5NeYki_Xv&XkUTQCavky<&mepJ_zTQ4sqX=5Fa4sYdwG^ z%7BjC(K&TtGwIyN`ML|Z;1vBxyU=uC>^^UJ!9xB?Pus&JIUTgb-#2KQR{NlXxhC|s z(AH^hR1ns7Kr7{lSne;`)2p|e-li=bw=eAJl$eS+vvTzw7^>+y@uTNEYZn@`DPWQ% zBmK;pF)Zshn_5wy-nCTP4ifv((Eh3Aj>?hlB(ac4;2WATPZ>@^uC=Sr^8mRINwbC{~9b5e)IQTgawX*1~NgMrQtCpc&*mv>*|7chm z3O^M;+%Jjcvvw@Yc_}X@#rxXEWzcAQ@|Q|;{E7LuDJkd6{s7!kq>Whi5JRjKfX9jX z#YYH5qlKA)fy3ZN!j`xHPulwOyvN1S*D9HX;;Tw$iFJ9rn;9$5h?)itLUjIAt4i^{QvI3pKijv#IN1o_Y(d=|I3;yqa5r86 literal 0 HcmV?d00001 diff --git a/thehive/test/resources/machinetag-otherfiles.zip b/thehive/test/resources/machinetag-otherfiles.zip index cac42ffef199dfb9148db6dd29e41cdd970023df..81501841780244d906cd544f35dd59b96ee549ea 100644 GIT binary patch delta 447 zcmZpa>sJT~@MdNaVE_T+(vCnd14@W7C@|zECTC>krIsY7>*kkaq!y)R=A;(uhlcPn zuupC}A3qg{L3C*aHv=Qf3uXoeFfl<&N(rat$%%ZT>|sDnKs?bZ&QcYpN`&QlS;hHz zU|Z95x5mpJ-UG6ABHV-3XXxP>;LXYgGL{(#-5D7eHt~UY0GMTfdjJ3c delta 268 zcmeBIXq2-G@MdNaVc-Bl;gpI%1}IPh(mV_b3_1D5C25&Csd^<9C7~g_4D8|W_ryc- z#3gYm0zfsniOCt6d8s9d>6Us~#rb()b?LfW<7E%;iB~+bb+a{NJ+BkY2Hq(gY$`z0 zVK%^Q+|meQ;j?dY8NW^h$U+#{(pU#%60$zNw1S&~k>v$50|S@{@MdKLnZN{uJAkw? HABYD47K=o= diff --git a/thehive/test/resources/machinetag-present.zip b/thehive/test/resources/machinetag-present.zip index 07a6812f6540fd60c1a27c077326f18742418161..bcaa2464e5e1f8097cfab27b37fb032699b6b47c 100644 GIT binary patch delta 905 zcmaDV_gT3N;?9<3@9PYpumuun4FQBms*mTu3J!)TAZ3!q8}Q<%fP;* z>3sY$AO_K;72FJrEH9WD7{CO=Kua_OMX?yD3N+CYV&;6HSul)d<^(AV6)c*;dJ#_4 z%PP*#13NEWcWeB5h;bY3YI*BHKH5`(>OE=f2BLV=wc~ue8xVtNG$%p4$-%(E@I0jg z9&Q4R3=CQr2J3(W4rVeo2S5I>2kc;=MU&t2ZU8$NTYSLc=dUA(g)oVP*kNQ6VTMFF zw)lZq0JQ(FBgj)&Y#f>?y?ocu;WVsfsah#tre7}(Odm6rj(jb{ZV_?Q@&7h> F1OP@pItm6DUuybN` zx5mpJ-UBjXqn$2eJ;>$C_-(g@*j@y*v>1p%G@9iQk8m(>Fl1I0!9!%eF))Ns-DIhc z<|YBKqtc;{0vf$}F=IV1*j3m97Z%um9YHLFSwsXiBa;X-B-XG6Ho{;}AX65}WR$Q6 z2mRy>ewoQ?`~vkL&%wZ!#wkE1iarDfYw%+RBEpH-VgzQ>Uq@CZuuW31bcPv0Kqu;? ipgJ!Aq6wh~oAZz&0@Hb{Y#=AF0^wsu1_nU^5Dx(EGrsWv delta 285 zcmeBDc_VKZ;LXe;!oUH9DOE**3{Y@@iGe|YL4hGRF*zeMFSR5w-9j&`I6p5mgqMLm zMt5tx?BPA}ibu9?yr;`JIgCj}1y!*nLUB4m@#a}f^}KE{tBQ(GE@Nk8V1QW#vwBM- zh(*Za$?}2{lP3v^M1U-Yfh~=xKm!q4u{$Haw1S&~k>v$50|S@{@MdKLnZN{udw{eQ HKZpkaMhi+? From 0d1a8b7638e46720def087992e25bf8524e8910f Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 9 Dec 2020 06:30:48 +0100 Subject: [PATCH 048/324] #1668 WIP: Add taxonomies list admin page --- frontend/app/index.html | 2 + frontend/app/scripts/app.js | 15 +++ .../admin/taxonomy/TaxonomyListCtrl.js | 102 ++++++++++++++++++ .../app/scripts/services/api/ProfileSrv.js | 6 +- .../app/scripts/services/api/TaxonomySrv.js | 78 ++++++++++++++ .../views/components/header.component.html | 6 ++ .../views/partials/admin/taxonomy/import.html | 26 +++++ .../views/partials/admin/taxonomy/list.html | 56 ++++++++++ .../partials/admin/taxonomy/list/toolbar.html | 12 +++ 9 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js create mode 100644 frontend/app/scripts/services/api/TaxonomySrv.js create mode 100644 frontend/app/views/partials/admin/taxonomy/import.html create mode 100644 frontend/app/views/partials/admin/taxonomy/list.html create mode 100644 frontend/app/views/partials/admin/taxonomy/list/toolbar.html diff --git a/frontend/app/index.html b/frontend/app/index.html index 800b08a1ae..9a4760768f 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -166,6 +166,7 @@ + @@ -292,6 +293,7 @@ + diff --git a/frontend/app/scripts/app.js b/frontend/app/scripts/app.js index 800a72397d..80d8e764a0 100644 --- a/frontend/app/scripts/app.js +++ b/frontend/app/scripts/app.js @@ -217,6 +217,21 @@ angular.module('thehive', [ permissions: ['manageProfile'] } }) + .state('app.administration.taxonomies', { + url: '/taxonomies', + templateUrl: 'views/partials/admin/taxonomy/list.html', + controller: 'TaxonomyListCtrl', + controllerAs: '$vm', + title: 'Taxonomies administration', + resolve: { + appConfig: function(VersionSrv) { + return VersionSrv.get(); + } + }, + guard: { + permissions: ['manageTaxonomy'] + } + }) .state('app.administration.organisations', { url: '/organisations', templateUrl: 'views/partials/admin/organisation/list.html', diff --git a/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js b/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js new file mode 100644 index 0000000000..47661db02f --- /dev/null +++ b/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js @@ -0,0 +1,102 @@ +(function() { + 'use strict'; + + angular.module('theHiveControllers') + .controller('TaxonomyListCtrl', TaxonomyListCtrl) + .controller('TaxonomyImportCtrl', TaxonomyImportCtrl); + + function TaxonomyListCtrl($uibModal, TaxonomySrv, NotificationSrv, ModalSrv, appConfig) { + var self = this; + + this.appConfig = appConfig; + + self.load = function() { + TaxonomySrv.list() + .then(function(response) { + self.list = response; + }) + .catch(function(rejection) { + NotificationSrv.error('Taxonomies management', rejection.data, rejection.status); + }); + }; + + self.import = function () { + var modalInstance = $uibModal.open({ + animation: true, + templateUrl: 'views/partials/admin/taxonomy/import.html', + controller: 'TaxonomyImportCtrl', + controllerAs: '$vm', + size: 'lg' + }); + + modalInstance.result + .then(function() { + self.load(); + }) + .catch(function(err){ + if(err && !_.isString(err)) { + NotificationSrv.error('Taxonomies import', err.data, err.status); + } + }); + }; + + this.toggleActive = function(id, active) { + TaxonomySrv.toggleActive(id, active) + .then(function() { + NotificationSrv.log('Taxonomy has been successfully ' + active ? 'activated' : 'deactivated', 'success'); + + self.load(); + }) + .catch(function(err){ + if(err && !_.isString(err)) { + NotificationSrv.error('Taxonomies ' + active ? 'activation' : 'deactivation', err.data, err.status); + } + }); + }; + + self.update = function(id, taxonomy) { + // TODO + // TaxonomySrv.update(id, _.pick(taxonomy, '...')) + TaxonomySrv.update(id, _.pick(taxonomy, '...')) + .then(function(/*response*/) { + self.load(); + NotificationSrv.log('Taxonomy updated successfully', 'success'); + }) + .catch(function(err) { + NotificationSrv.error('Error', 'Taxonomy update failed', err.status); + }); + }; + + self.create = function(taxonomy) { + TaxonomySrv.create(taxonomy) + .then(function(/*response*/) { + self.load(); + NotificationSrv.log('Taxonomy created successfully', 'success'); + }) + .catch(function(err) { + NotificationSrv.error('Error', 'Taxonomy creation failed', err.status); + }); + }; + + self.$onInit = function() { + self.load(); + }; + } + + function TaxonomyImportCtrl($uibModalInstance, TaxonomySrv, NotificationSrv) { + this.formData = {}; + + this.ok = function () { + TaxonomySrv.import(this.formData) + .then(function() { + $uibModalInstance.close(); + }, function(response) { + NotificationSrv.error('TaxonomyImportCtrl', response.data, response.status); + }); + }; + + this.cancel = function () { + $uibModalInstance.dismiss('cancel'); + }; + } +})(); diff --git a/frontend/app/scripts/services/api/ProfileSrv.js b/frontend/app/scripts/services/api/ProfileSrv.js index 40ebe5af40..303809f06d 100644 --- a/frontend/app/scripts/services/api/ProfileSrv.js +++ b/frontend/app/scripts/services/api/ProfileSrv.js @@ -16,6 +16,7 @@ 'manageCustomField', 'manageConfig', 'manageTag', + 'manageTaxonomy', 'manageProfile', 'manageAnalyzerTemplate', 'manageObservableTemplate' @@ -25,6 +26,7 @@ manageOrganisation: 'Manage organisations', manageCustomField: 'Manage custom fields', manageConfig: 'Manage configurations', + manageTaxonomy: 'Manage taxonomies', manageTag: 'Manage tags', manageProfile: 'Manage profiles', manageAnalyzerTemplate: 'Manage analyzer templates', @@ -61,7 +63,9 @@ }; this.list = function() { - return $http.get(baseUrl); + return $http.get(baseUrl, {params: { + range: 'all' + }}); }; this.get = function(name) { diff --git a/frontend/app/scripts/services/api/TaxonomySrv.js b/frontend/app/scripts/services/api/TaxonomySrv.js new file mode 100644 index 0000000000..747b9a4a52 --- /dev/null +++ b/frontend/app/scripts/services/api/TaxonomySrv.js @@ -0,0 +1,78 @@ +(function() { + 'use strict'; + angular.module('theHiveServices') + .service('TaxonomySrv', function($http, QuerySrv) { + // var self = this; + var baseUrl = './api/v1/taxonomy'; + + this.list = function() { + // return $http.get(baseUrl, {params: { + // range: 'all' + // }}); + // + return QuerySrv.call('v1', [ + { _name: 'listTaxonomy' } + ], { + name:'list-taxonomies' + }); + + //listTaxonomies + }; + + this.get = function(name) { + return $http.get(baseUrl + '/' + name); + }; + + this.toggleActive = function(id, active) { + return $http.put([baseUrl, id, !!active ? 'activate' : 'deactivate'].join('/')); + }; + + // this.map = function() { + // return self.list() + // .then(function(response) { + // return _.indexBy(response.data, 'name'); + // }); + // }; + + this.create = function(profile) { + return $http.post(baseUrl, profile); + }; + + this.update = function(id, profile) { + return $http.patch(baseUrl + '/' + id, profile); + }; + + this.remove = function(id) { + return $http.delete(baseUrl + '/' + id); + }; + + this.import = function(post) { + var postData = { + file: post.attachment + }; + + return $http({ + method: 'POST', + url: baseUrl + '/import-zip', + headers: { + 'Content-Type': undefined + }, + transformRequest: function (data) { + var formData = new FormData(), + copy = angular.copy(data, {}); + + angular.forEach(data, function (value, key) { + if (Object.getPrototypeOf(value) instanceof Blob || Object.getPrototypeOf(value) instanceof File) { + formData.append(key, value); + delete copy[key]; + } + }); + + return formData; + }, + data: postData + }); + }; + }); + +})(); diff --git a/frontend/app/views/components/header.component.html b/frontend/app/views/components/header.component.html index 497dcd2a85..ca85b74689 100644 --- a/frontend/app/views/components/header.component.html +++ b/frontend/app/views/components/header.component.html @@ -98,6 +98,12 @@ Case custom fields +
  • + + + Taxonomies + +
  • diff --git a/frontend/app/views/partials/admin/taxonomy/import.html b/frontend/app/views/partials/admin/taxonomy/import.html new file mode 100644 index 0000000000..634dce0d94 --- /dev/null +++ b/frontend/app/views/partials/admin/taxonomy/import.html @@ -0,0 +1,26 @@ +
    +
    +

    Import taxonomies archive

    +
    +
    + +
    +

    Download the official MISP taxonomies archive

    +

    You can download the latest archive of the official MISP Taxonomies from here

    +
    + +
    + + +
    + +

    + The taxonomies archive must be a valid ZIP file containing at least one file names machinetag.json +

    +
    +
    +
    + + +
    +
    diff --git a/frontend/app/views/partials/admin/taxonomy/list.html b/frontend/app/views/partials/admin/taxonomy/list.html new file mode 100644 index 0000000000..e8eb9c5aa0 --- /dev/null +++ b/frontend/app/views/partials/admin/taxonomy/list.html @@ -0,0 +1,56 @@ +
    +
    +
    +
    +

    List of taxonomies

    +
    +
    +
    + +
    +
    +
    No taxnomies found.
    +
    +
    + + +
    +
    + + + + + + + + + + + + + + + + + +
    NameDescription# Tags
    +
    + {{::taxonomy.namespace}} +
    +     		+ {{::taxonomy.description}} + + {{::taxonomy.tags.length}} + + + Edit + + + Delete +
    +
    +
    +
    +
    +
    +
    diff --git a/frontend/app/views/partials/admin/taxonomy/list/toolbar.html b/frontend/app/views/partials/admin/taxonomy/list/toolbar.html new file mode 100644 index 0000000000..e7b922a3d6 --- /dev/null +++ b/frontend/app/views/partials/admin/taxonomy/list/toolbar.html @@ -0,0 +1,12 @@ +
    +
    + +
    +
    From eecd1919b09677fc85b71426771ba857dee423ac Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 10 Dec 2020 11:48:46 +0100 Subject: [PATCH 049/324] Changed freetags namespace & fixed visibily issue for listing taxonomies & added enabled property --- .../org/thp/thehive/dto/v1/Taxonomy.scala | 6 +- .../thehive/controllers/v1/Conversion.scala | 12 +++ .../thehive/controllers/v1/Properties.scala | 2 + .../thehive/controllers/v1/TaxonomyCtrl.scala | 35 +++----- .../controllers/v1/TaxonomyRenderer.scala | 45 +++++++++++ .../models/TheHiveSchemaDefinition.scala | 8 +- .../thp/thehive/services/TaxonomySrv.scala | 79 +++++++++++-------- 7 files changed, 126 insertions(+), 61 deletions(-) create mode 100644 thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala index 7081347184..3835c4c0bc 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Taxonomy.scala @@ -1,8 +1,7 @@ package org.thp.thehive.dto.v1 import java.util.Date - -import play.api.libs.json.{Json, OFormat} +import play.api.libs.json.{JsObject, Json, OFormat} /* Format based on : @@ -65,7 +64,8 @@ case class OutputTaxonomy( namespace: String, description: String, version: Int, - tags: Seq[OutputTag] + tags: Seq[OutputTag], + extraData: JsObject ) object OutputTaxonomy { diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index d2518f459f..e12c8aa200 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -266,9 +266,21 @@ object Conversion { .withFieldComputed(_._id, _._id.toString) .withFieldConst(_._type, "Taxonomy") .withFieldComputed(_.tags, _.tags.map(_.toOutput)) + .withFieldConst(_.extraData, JsObject.empty) .transform ) + implicit val taxonomyWithStatsOutput: Renderer.Aux[(RichTaxonomy, JsObject), OutputTaxonomy] = + Renderer.toJson[(RichTaxonomy, JsObject), OutputTaxonomy] { taxoWithExtraData => + taxoWithExtraData._1 + .into[OutputTaxonomy] + .withFieldComputed(_._id, _._id.toString) + .withFieldConst(_._type, "Taxonomy") + .withFieldComputed(_.tags, _.tags.map(_.toOutput)) + .withFieldConst(_.extraData, taxoWithExtraData._2) + .transform + } + implicit val tagOutput: Renderer.Aux[Tag, OutputTag] = Renderer.toJson[Tag, OutputTag]( _.into[OutputTag] diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index b6dc5ef69c..54ff5f31a4 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -21,6 +21,7 @@ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TagOps._ import org.thp.thehive.services.TaskOps._ +import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.UserOps._ import org.thp.thehive.services._ import play.api.libs.json.{JsObject, JsValue, Json} @@ -497,5 +498,6 @@ class Properties @Inject() ( .property("namespace", UMapping.string)(_.field.readonly) .property("description", UMapping.string)(_.field.readonly) .property("version", UMapping.int)(_.field.readonly) + .property("enabled", UMapping.boolean)(_.select(_.enabled).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 39bc1c0a49..8c1358f137 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -14,29 +14,27 @@ import org.thp.scalligraph.{BadRequestError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputTaxonomy import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} -import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TaxonomyOps._ -import org.thp.thehive.services.{OrganisationSrv, TagSrv, TaxonomySrv} +import org.thp.thehive.services.{TagSrv, TaxonomySrv} import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, Results} import scala.collection.JavaConverters._ -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Try} class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, properties: Properties, taxonomySrv: TaxonomySrv, - organisationSrv: OrganisationSrv, tagSrv: TagSrv, @Named("with-thehive-schema") implicit val db: Database -) extends QueryableCtrl { +) extends QueryableCtrl with TaxonomyRenderer { override val entityName: String = "taxonomy" override val publicProperties: PublicProperties = properties.taxonomy override val initialQuery: Query = Query.init[Traversal.V[Taxonomy]]("listTaxonomy", (graph, authContext) => - organisationSrv.get(authContext.organisation)(graph).taxonomies + taxonomySrv.startTraversal(graph).visible(authContext) ) override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Taxonomy]]( @@ -48,8 +46,12 @@ class TaxonomyCtrl @Inject() ( Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput]( "page", FieldsParser[OutputParam], - (range, traversal, _) => - traversal.richPage(range.from, range.to, range.extraData.contains("total"))(_.richTaxonomy) + { + case (OutputParam(from, to, extraData), taxoSteps, authContext) => + taxoSteps.richPage(from, to, extraData.contains("total")) { + _.richTaxonomyWithCustomRenderer(taxoStatsRenderer(extraData - "total")(authContext)) + } + } ) override val outputQuery: Query = Query.outputWithContext[RichTaxonomy, Traversal.V[Taxonomy]]((traversal, _) => @@ -59,17 +61,6 @@ class TaxonomyCtrl @Inject() ( Query[Traversal.V[Taxonomy], Traversal.V[Tag]]("tags", (traversal, _) => traversal.tags) ) - def list: Action[AnyContent] = - entrypoint("list taxonomies") - .authRoTransaction(db) { implicit request => implicit graph => - val taxos = taxonomySrv - .startTraversal - .visible - .richTaxonomy - .toSeq - Success(Results.Ok(taxos.toJson)) - } - def create: Action[AnyContent] = entrypoint("import taxonomy") .extract("taxonomy", FieldsParser[InputTaxonomy]) @@ -94,7 +85,7 @@ class TaxonomyCtrl @Inject() ( inputTaxos <- headers .filter(h => h.getFileName.endsWith("machinetag.json")) .toTry(parseJsonFile(zipFile, _)) - richTaxos <- db.tryTransaction { implicit graph => + richTaxos <- db.tryTransaction { implicit graph => inputTaxos.toTry(inputTaxo => createFromInput(inputTaxo)).map(_.toJson) } } yield Results.Created(richTaxos) @@ -128,9 +119,9 @@ class TaxonomyCtrl @Inject() ( if (inputTaxo.namespace.isEmpty) Failure(BadRequestError(s"A taxonomy with no namespace cannot be imported")) - else if (inputTaxo.namespace == "_freetags") + else if (inputTaxo.namespace.startsWith("_freetags")) Failure(BadRequestError(s"Namespace _freetags is restricted for TheHive")) - else if (taxonomySrv.existsInOrganisation(inputTaxo.namespace)) + else if (taxonomySrv.startTraversal.alreadyImported(inputTaxo.namespace)) Failure(BadRequestError(s"A taxonomy with namespace '${inputTaxo.namespace}' already exists in this organisation")) else for { diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala new file mode 100644 index 0000000000..07835754be --- /dev/null +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala @@ -0,0 +1,45 @@ +package org.thp.thehive.controllers.v1 + +import org.thp.scalligraph.auth.AuthContext +import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.thehive.services.TaxonomyOps._ +import org.thp.scalligraph.traversal.{Converter, Traversal} +import org.thp.thehive.models.Taxonomy +import play.api.libs.json._ + +import java.util.{Map => JMap} + +trait TaxonomyRenderer { + + def enabledStats: Traversal.V[Taxonomy] => Traversal[JsValue, Boolean, Converter[JsValue, Boolean]] = + _.enabled.domainMap(l => JsBoolean(l)) + + def taxoStatsRenderer(extraData: Set[String])(implicit + authContext: AuthContext + ): Traversal.V[Taxonomy] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { traversal => + def addData[G]( + name: String + )(f: Traversal.V[Taxonomy] => Traversal[JsValue, G, Converter[JsValue, G]]): Traversal[JsObject, JMap[String, Any], Converter[ + JsObject, + JMap[String, Any] + ]] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { t => + val dataTraversal = f(traversal.start) + t.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.by(dataTraversal.raw)) { jmap => + t.converter(jmap) + (name -> dataTraversal.converter(jmap.get(name).asInstanceOf[G])) + } + } + + if (extraData.isEmpty) traversal.constant2[JsObject, JMap[String, Any]](JsObject.empty) + else { + val dataName = extraData.toSeq + dataName.foldLeft[Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]]]( + traversal.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.project(dataName.head, dataName.tail: _*))(_ => + JsObject.empty + ) + ) { + case (f, "enabled") => addData("enabled")(enabledStats)(f) + case (f, _) => f + } + } + } +} diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index e0b4f9f09a..c3af62d20a 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -93,13 +93,13 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { db.tryTransaction { implicit g => // For each organisation, if there is no custom taxonomy, create it db.labelFilter("Organisation")(Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o => - Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", "_freetags").headOption match { + Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", s"_freetags_${o.id()}").headOption match { case None => val taxoVertex = g.addVertex("Taxonomy") taxoVertex.property("_label", "Taxonomy") taxoVertex.property("_createdBy", "system@thehive.local") taxoVertex.property("_createdAt", new Date()) - taxoVertex.property("namespace", "_freetags") + taxoVertex.property("namespace", s"_freetags_${o.id()}") taxoVertex.property("description", "Custom taxonomy") taxoVertex.property("version", 1) o.addEdge("OrganisationTaxonomy", taxoVertex) @@ -112,7 +112,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db => db.tryTransaction { implicit g => db.labelFilter("Organisation")(Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o => - val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", "_freetags").head + val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", s"_freetags_${o.id()}").head Traversal.V(EntityId(o.id())).unionFlat( _.out("OrganisationShare").out("ShareCase").out("CaseTag"), _.out("OrganisationShare").out("ShareObservable").out("ObservableTag"), @@ -125,7 +125,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { tag.property("predicate").value().toString, tag.property ("value").orElse("") ) - tag.property("namespace", "_freetags") + tag.property("namespace", s"_freetags_${o.id()}") tag.property("predicate", tagStr) tag.property("value").remove() customTaxo.addEdge("TaxonomyTag", tag) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index aab26143cf..ec120177d5 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -1,20 +1,20 @@ package org.thp.thehive.services import java.util.{Map => JMap} - import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} +import org.thp.scalligraph.traversal.Converter.Identity import org.thp.scalligraph.traversal.TraversalOps.TraversalOpsDefs import org.thp.scalligraph.traversal.{Converter, Traversal} -import org.thp.scalligraph.{EntityId, EntityIdOrName, RichSeq} +import org.thp.scalligraph.{BadRequestError, EntityId, EntityIdOrName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TaxonomyOps._ -import scala.util.{Success, Try} +import scala.util.{Failure, Success, Try} @Singleton class TaxonomySrv @Inject() ( @@ -25,14 +25,6 @@ class TaxonomySrv @Inject() ( val taxonomyTagSrv = new EdgeSrv[TaxonomyTag, Taxonomy, Tag] val organisationTaxonomySrv = new EdgeSrv[OrganisationTaxonomy, Organisation, Taxonomy] - def existsInOrganisation(namespace: String)(implicit graph: Graph, authContext: AuthContext): Boolean = { - startTraversal - .getByNamespace(namespace) - .organisations - .current - .exists - } - def create(taxo: Taxonomy, tags: Seq[Tag with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = for { taxonomy <- createEntity(taxo) @@ -41,7 +33,7 @@ class TaxonomySrv @Inject() ( } yield richTaxonomy def createFreetag(organisation: Organisation with Entity)(implicit graph: Graph, authContext: AuthContext): Try[RichTaxonomy] = { - val customTaxo = Taxonomy("_freetags", "Custom taxonomy", 1) + val customTaxo = Taxonomy(s"_freetags_${organisation._id}", "Custom taxonomy", 1) for { taxonomy <- createEntity(customTaxo) richTaxonomy <- Try(RichTaxonomy(taxonomy, Seq())) @@ -53,25 +45,23 @@ class TaxonomySrv @Inject() ( Try(startTraversal.getByNamespace(name)).getOrElse(startTraversal.limit(0)) def activate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - for { - taxo <- get(taxonomyId).getOrFail("Taxonomy") - organisations <- Try(organisationSrv.startTraversal.filterNot(_ - .out[OrganisationTaxonomy] - .v[Taxonomy] - .has(_.namespace, taxo.namespace) - ).toSeq) - _ <- organisations.toTry(o => organisationTaxonomySrv.create(OrganisationTaxonomy(), o, taxo)) - } yield Success(()) - - def deactivate(taxonomyId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { taxo <- get(taxonomyId).getOrFail("Taxonomy") - _ <- Try(organisationSrv.startTraversal - .hasNot(_.name, "admin") - .outE[OrganisationTaxonomy] - .filter(_.otherV.v[Taxonomy].has(_.namespace, taxo.namespace)) - .remove()) - } yield Success(()) + _ <- if (taxo.namespace.startsWith("_freetags")) Failure(BadRequestError("Cannot activate a freetags taxonomy")) + else Success(()) + _ <- organisationSrv.startTraversal + .filterNot(_.out[OrganisationTaxonomy].v[Taxonomy].has(_.namespace, taxo.namespace)) + .toSeq + .toTry(o => organisationTaxonomySrv.create(OrganisationTaxonomy(), o, taxo)) + } yield () + + def deactivate(taxonomyId: EntityIdOrName)(implicit graph: Graph): Try[Unit] = { + for { + taxo <- getOrFail(taxonomyId) + _ <- if (taxo.namespace.startsWith("_freetags")) Failure(BadRequestError("Cannot deactivate a freetags taxonomy")) + else Success(()) + } yield get(taxonomyId).inE[OrganisationTaxonomy].remove() + } } @@ -83,13 +73,21 @@ object TaxonomyOps { def getByNamespace(namespace: String): Traversal.V[Taxonomy] = traversal.has(_.namespace, namespace) - def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = visible(authContext.organisation) + def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = { + if (authContext.isPermitted(Permissions.manageTaxonomy)) + traversal + else + traversal.filter(_.organisations.get(authContext.organisation)) + } - def visible(organisationIdOrName: EntityIdOrName): Traversal.V[Taxonomy] = - traversal.filter(_.organisations.get(organisationIdOrName)) + def alreadyImported(namespace: String): Boolean = + traversal.getByNamespace(namespace).exists def organisations: Traversal.V[Organisation] = traversal.in[OrganisationTaxonomy].v[Organisation] + def enabled: Traversal[Boolean, Boolean, Identity[Boolean]] = + traversal.choose(_.organisations, true, false) + def tags: Traversal.V[Tag] = traversal.out[TaxonomyTag].v[Tag] def richTaxonomy: Traversal[RichTaxonomy, JMap[String, Any], Converter[RichTaxonomy, JMap[String, Any]]] = @@ -99,5 +97,22 @@ object TaxonomyOps { .by(_.tags.fold) ) .domainMap { case (taxonomy, tags) => RichTaxonomy(taxonomy, tags) } + + def richTaxonomyWithCustomRenderer[D, G, C <: Converter[D, G]](entityRenderer: Traversal.V[Taxonomy] => Traversal[D, G, C]): + Traversal[(RichTaxonomy, D), JMap[String, Any], Converter[(RichTaxonomy, D), JMap[String, Any]]] = + traversal + .project( + _.by + .by(_.tags.fold) + .by(_.enabled) + .by(entityRenderer) + ) + .domainMap { + case (taxo, tags, _, renderedEntity) => + RichTaxonomy( + taxo, + tags + ) -> renderedEntity + } } } From aac858a949419f46c8afa776a33f334ef1359ca3 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 10 Dec 2020 14:09:14 +0100 Subject: [PATCH 050/324] Removed freetags from admin list taxonomies --- thehive/app/org/thp/thehive/services/TaxonomySrv.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index ec120177d5..a652192711 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -1,5 +1,7 @@ package org.thp.thehive.services +import org.apache.tinkerpop.gremlin.process.traversal.TextP + import java.util.{Map => JMap} import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph @@ -75,11 +77,14 @@ object TaxonomyOps { def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = { if (authContext.isPermitted(Permissions.manageTaxonomy)) - traversal + traversal.noFreetags else traversal.filter(_.organisations.get(authContext.organisation)) } + private def noFreetags: Traversal.V[Taxonomy] = + traversal.filterNot(_.has(_.namespace, TextP.startingWith("_freetags"))) + def alreadyImported(namespace: String): Boolean = traversal.getByNamespace(namespace).exists From c1a6f7dfc54d78be1771dcb7f892f57fb2939fde Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 10 Dec 2020 15:04:04 +0100 Subject: [PATCH 051/324] Zip import can fail and still continue importing --- .../thehive/controllers/v1/TaxonomyCtrl.scala | 18 +++++++++++++----- .../org/thp/thehive/services/TaxonomySrv.scala | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 8c1358f137..778f4e9e1f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -16,11 +16,11 @@ import org.thp.thehive.dto.v1.InputTaxonomy import org.thp.thehive.models.{Permissions, RichTaxonomy, Tag, Taxonomy} import org.thp.thehive.services.TaxonomyOps._ import org.thp.thehive.services.{TagSrv, TaxonomySrv} -import play.api.libs.json.Json +import play.api.libs.json.{JsArray, Json} import play.api.mvc.{Action, AnyContent, Results} import scala.collection.JavaConverters._ -import scala.util.{Failure, Try} +import scala.util.{Failure, Success, Try} class TaxonomyCtrl @Inject() ( entrypoint: Entrypoint, @@ -85,9 +85,17 @@ class TaxonomyCtrl @Inject() ( inputTaxos <- headers .filter(h => h.getFileName.endsWith("machinetag.json")) .toTry(parseJsonFile(zipFile, _)) - richTaxos <- db.tryTransaction { implicit graph => - inputTaxos.toTry(inputTaxo => createFromInput(inputTaxo)).map(_.toJson) - } + richTaxos = inputTaxos.foldLeft[JsArray](JsArray.empty)((array, taxo) => { + val res = db.tryTransaction { implicit graph => + createFromInput(taxo) + } match { + case Failure(e) => + Json.obj("namespace" -> taxo.namespace, "importState" -> s"Failure : ${e.getMessage}") + case Success(t) => + Json.obj("namespace" -> t.namespace, "importState" -> s"Success : ${t.tags.size} tags imported") + } + array :+ res + }) } yield Results.Created(richTaxos) } diff --git a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala index a652192711..2c57ba30e4 100644 --- a/thehive/app/org/thp/thehive/services/TaxonomySrv.scala +++ b/thehive/app/org/thp/thehive/services/TaxonomySrv.scala @@ -77,7 +77,7 @@ object TaxonomyOps { def visible(implicit authContext: AuthContext): Traversal.V[Taxonomy] = { if (authContext.isPermitted(Permissions.manageTaxonomy)) - traversal.noFreetags + noFreetags else traversal.filter(_.organisations.get(authContext.organisation)) } From 213d94c101b4fccd91b7baee1891d926514eb936 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 10 Dec 2020 15:16:51 +0100 Subject: [PATCH 052/324] Changed field names for zip import --- thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index 778f4e9e1f..cd73717175 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -90,9 +90,9 @@ class TaxonomyCtrl @Inject() ( createFromInput(taxo) } match { case Failure(e) => - Json.obj("namespace" -> taxo.namespace, "importState" -> s"Failure : ${e.getMessage}") + Json.obj("namespace" -> taxo.namespace, "status" -> "Failure", "message" -> e.getMessage) case Success(t) => - Json.obj("namespace" -> t.namespace, "importState" -> s"Success : ${t.tags.size} tags imported") + Json.obj("namespace" -> t.namespace, "status" -> "Success", "tagsImported" -> t.tags.size) } array :+ res }) From 457483078b9ede9b3491ca0ee37917b52047e04c Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 10 Dec 2020 16:12:14 +0100 Subject: [PATCH 053/324] Fixed broken tests --- .../controllers/v1/TaxonomyCtrlTest.scala | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala index 68c4627b49..917a6fb6f3 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala @@ -4,7 +4,7 @@ import org.thp.scalligraph.controllers.FakeTemporaryFile import org.thp.thehive.TestAppBuilder import org.thp.thehive.dto.v1._ import play.api.libs.Files -import play.api.libs.json.Json +import play.api.libs.json.{JsArray, Json} import play.api.mvc.MultipartFormData.FilePart import play.api.mvc.{AnyContentAsMultipartFormData, MultipartFormData} import play.api.test.{FakeRequest, PlaySpecification} @@ -144,8 +144,8 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[TaxonomyCtrl].importZip(request) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") - val zipTaxos = contentAsJson(result).as[Seq[OutputTaxonomy]] - zipTaxos.size must beEqualTo(2) + contentAsString(result) must not contain("Failure") + contentAsJson(result).as[JsArray].value.size must beEqualTo(2) } "import zip file with folders correctly" in testApp { app => @@ -156,8 +156,8 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[TaxonomyCtrl].importZip(request) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") - val zipTaxos = contentAsJson(result).as[Seq[OutputTaxonomy]] - zipTaxos.size must beEqualTo(2) + contentAsString(result) must not contain("Failure") + contentAsJson(result).as[JsArray].value.size must beEqualTo(2) } "return no error if zip file contains other files than taxonomies" in testApp { app => @@ -168,8 +168,8 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[TaxonomyCtrl].importZip(request) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") - val zipTaxos = contentAsJson(result).as[Seq[OutputTaxonomy]] - zipTaxos.size must beEqualTo(1) + contentAsString(result) must not contain("Failure") + contentAsJson(result).as[JsArray].value.size must beEqualTo(1) } "return error if zip file contains an already present taxonomy" in testApp { app => @@ -178,9 +178,9 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { .withBody(AnyContentAsMultipartFormData(multipartZipFile("machinetag-present.zip"))) val result = app[TaxonomyCtrl].importZip(request) - status(result) must beEqualTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") - (contentAsJson(result) \ "type").as[String] must beEqualTo("BadRequest") - (contentAsJson(result) \ "message").as[String] must contain("already exists") + status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") + contentAsString(result) must contain("Failure") + contentAsJson(result).as[JsArray].value.size must beEqualTo(2) } "return error if zip file contains a bad formatted taxonomy" in testApp { app => From c9bcf11b933213b4aa5c37d2b509af8cf69c79b0 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 10 Dec 2020 16:17:40 +0100 Subject: [PATCH 054/324] #1705 Improve S3 support --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index f6a4d2165c..9aa06293e3 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit f6a4d2165c26826c5b28db1a513ade15dfb060f2 +Subproject commit 9aa06293e32254466d1c5d7ae089755fdfbbe4e0 From d542bd140865edbaa86b9622dff6cd39f91167ee Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 10 Dec 2020 16:48:03 +0100 Subject: [PATCH 055/324] #1454 Update describe API with "initialQuery" field --- .../thehive/controllers/v1/DescribeCtrl.scala | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index 33e3fae52e..8b61b7dcb1 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -56,7 +56,7 @@ class DescribeCtrl @Inject() ( PropertyDescription("_updatedBy", "user"), PropertyDescription("_updatedAt", "date") ) - case class EntityDescription(label: String, attributes: Seq[PropertyDescription]) { + case class EntityDescription(label: String, initialQuery: String, attributes: Seq[PropertyDescription]) { def toJson: JsObject = Json.obj( "label" -> label, @@ -72,12 +72,14 @@ class DescribeCtrl @Inject() ( def describeCortexEntity( name: String, + initialQuery: String, className: String, packageName: String = "org.thp.thehive.connector.cortex.controllers.v0" ): Option[EntityDescription] = Try( EntityDescription( name, + initialQuery, injector .instanceOf(getClass.getClassLoader.loadClass(s"$packageName.$className")) .asInstanceOf[QueryCtrlV0] @@ -91,22 +93,26 @@ class DescribeCtrl @Inject() ( def entityDescriptions: Seq[EntityDescription] = cacheApi.getOrElseUpdate(s"describe.v1", cacheExpire) { Seq( - EntityDescription("case", caseCtrl.publicProperties.list.flatMap(propertyToJson("case", _))), - EntityDescription("case_task", taskCtrl.publicProperties.list.flatMap(propertyToJson("case_task", _))), - EntityDescription("alert", alertCtrl.publicProperties.list.flatMap(propertyToJson("alert", _))), - EntityDescription("case_artifact", observableCtrl.publicProperties.list.flatMap(propertyToJson("case_artifact", _))), - EntityDescription("user", userCtrl.publicProperties.list.flatMap(propertyToJson("user", _))), - EntityDescription("case_task_log", logCtrl.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), - EntityDescription("audit", auditCtrl.publicProperties.list.flatMap(propertyToJson("audit", _))), - EntityDescription("caseTemplate", caseTemplateCtrl.publicProperties.list.flatMap(propertyToJson("caseTemplate", _))), - EntityDescription("customField", customFieldCtrl.publicProperties.list.flatMap(propertyToJson("customField", _))), - EntityDescription("observableType", observableTypeCtrl.publicProperties.list.flatMap(propertyToJson("observableType", _))), - EntityDescription("organisation", organisationCtrl.publicProperties.list.flatMap(propertyToJson("organisation", _))), - EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))) -// EntityDescription("dashboard", dashboardCtrl.publicProperties.list.flatMap(propertyToJson("dashboard", _))), -// EntityDescription("page", pageCtrl.publicProperties.list.flatMap(propertyToJson("page", _))) - ) ++ describeCortexEntity("case_artifact_job", "JobCtrl") ++ - describeCortexEntity("action", "ActionCtrl") + EntityDescription("alert", "listAlert", alertCtrl.publicProperties.list.flatMap(propertyToJson("alert", _))), + EntityDescription("audit", "listAudit", auditCtrl.publicProperties.list.flatMap(propertyToJson("audit", _))), + EntityDescription("case", "listCase", caseCtrl.publicProperties.list.flatMap(propertyToJson("case", _))), + EntityDescription("caseTemplate", "listCaseTemplate", caseTemplateCtrl.publicProperties.list.flatMap(propertyToJson("caseTemplate", _))), + EntityDescription("customField", "listCustomField", customFieldCtrl.publicProperties.list.flatMap(propertyToJson("customField", _))), + // EntityDescription("dashboard", "listDashboard", dashboardCtrl.publicProperties.list.flatMap(propertyToJson("dashboard", _))), + EntityDescription("log", "listLog", logCtrl.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), + EntityDescription("observable", "listObservable", observableCtrl.publicProperties.list.flatMap(propertyToJson("case_artifact", _))), + EntityDescription( + "observableType", + "listObservableType", + observableTypeCtrl.publicProperties.list.flatMap(propertyToJson("observableType", _)) + ), + EntityDescription("organisation", "listOrganisation", organisationCtrl.publicProperties.list.flatMap(propertyToJson("organisation", _))), + // EntityDescription("page", "listPage", pageCtrl.publicProperties.list.flatMap(propertyToJson("page", _))) + EntityDescription("profile", "listProfile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))), + EntityDescription("task", "listTask", taskCtrl.publicProperties.list.flatMap(propertyToJson("case_task", _))), + EntityDescription("user", "listUser", userCtrl.publicProperties.list.flatMap(propertyToJson("user", _))) + ) ++ describeCortexEntity("job", "listJob", "JobCtrl") ++ + describeCortexEntity("action", "listAction", "ActionCtrl") } implicit val propertyDescriptionWrites: Writes[PropertyDescription] = From 1d507aa76d17246af2a536b34670281459fbd020 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 10 Dec 2020 16:48:55 +0100 Subject: [PATCH 056/324] #1454 Add APIs for task, observable and logs --- .../thp/thehive/controllers/v0/Router.scala | 44 ++++----- .../controllers/v1/ObservableCtrl.scala | 97 +++++++++++++------ .../thp/thehive/controllers/v1/Router.scala | 46 +++++++-- 3 files changed, 126 insertions(+), 61 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Router.scala b/thehive/app/org/thp/thehive/controllers/v0/Router.scala index 050122e10d..353b7a55b8 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Router.scala @@ -7,31 +7,31 @@ import play.api.routing.sird._ @Singleton class Router @Inject() ( - statsCtrl: StatsCtrl, + authenticationCtrl: AuthenticationCtrl, + alertCtrl: AlertCtrl, + attachmentCtrl: AttachmentCtrl, + auditCtrl: AuditCtrl, caseCtrl: CaseCtrl, caseTemplateCtrl: CaseTemplateCtrl, - userCtrl: UserCtrl, - organisationCtrl: OrganisationCtrl, - taskCtrl: TaskCtrl, - logCtrl: LogCtrl, - observableCtrl: ObservableCtrl, + configCtrl: ConfigCtrl, customFieldCtrl: CustomFieldCtrl, - alertCtrl: AlertCtrl, - auditCtrl: AuditCtrl, - statusCtrl: StatusCtrl, dashboardCtrl: DashboardCtrl, - authenticationCtrl: AuthenticationCtrl, - listCtrl: ListCtrl, - streamCtrl: StreamCtrl, - attachmentCtrl: AttachmentCtrl, describeCtrl: DescribeCtrl, - configCtrl: ConfigCtrl, - profileCtrl: ProfileCtrl, - shareCtrl: ShareCtrl, - tagCtrl: TagCtrl, + listCtrl: ListCtrl, + logCtrl: LogCtrl, + observableCtrl: ObservableCtrl, + organisationCtrl: OrganisationCtrl, + observableTypeCtrl: ObservableTypeCtrl, pageCtrl: PageCtrl, permissionCtrl: PermissionCtrl, - observableTypeCtrl: ObservableTypeCtrl + profileCtrl: ProfileCtrl, + taskCtrl: TaskCtrl, + shareCtrl: ShareCtrl, + statsCtrl: StatsCtrl, + userCtrl: UserCtrl, + statusCtrl: StatusCtrl, + streamCtrl: StreamCtrl, + tagCtrl: TagCtrl ) extends SimpleRouter { override def routes: Routes = { @@ -86,11 +86,11 @@ class Router @Inject() ( case POST(p"/case/artifact/$observableId/shares") => shareCtrl.shareObservable(observableId) case GET(p"/case") => caseCtrl.search - case POST(p"/case") => caseCtrl.create // Audit ok + case POST(p"/case") => caseCtrl.create // Audit ok case GET(p"/case/$caseId") => caseCtrl.get(caseId) - case PATCH(p"/case/_bulk") => caseCtrl.bulkUpdate // Not used by the frontend - case PATCH(p"/case/$caseId") => caseCtrl.update(caseId) // Audit ok - case POST(p"/case/_merge/$caseIds") => caseCtrl.merge(caseIds) // Not implemented in backend and not used by frontend + case PATCH(p"/case/_bulk") => caseCtrl.bulkUpdate // Not used by the frontend + case PATCH(p"/case/$caseId") => caseCtrl.update(caseId) // Audit ok + case POST(p"/case/_merge/$caseIds") => caseCtrl.merge(caseIds) // Not implemented in backend and not used by frontend case POST(p"/case/_search") => caseCtrl.search case POST(p"/case/_stats") => caseCtrl.stats case DELETE(p"/case/$caseId") => caseCtrl.delete(caseId) // Not used by the frontend diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index f383a7a025..69f39d30fe 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -2,7 +2,6 @@ package org.thp.thehive.controllers.v1 import java.io.FilterInputStream import java.nio.file.Files - import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader @@ -22,6 +21,7 @@ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services._ import play.api.libs.Files.DefaultTemporaryFileCreator +import play.api.libs.json.{JsArray, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} import play.api.{Configuration, Logger} @@ -29,7 +29,7 @@ import scala.collection.JavaConverters._ @Singleton class ObservableCtrl @Inject() ( - entryPoint: Entrypoint, + entrypoint: Entrypoint, @Named("with-thehive-schema") db: Database, properties: Properties, observableSrv: ObservableSrv, @@ -37,7 +37,8 @@ class ObservableCtrl @Inject() ( caseSrv: CaseSrv, organisationSrv: OrganisationSrv, temporaryFileCreator: DefaultTemporaryFileCreator, - configuration: Configuration + configuration: Configuration, + errorHandler: ErrorHandler ) extends QueryableCtrl with ObservableRenderer { @@ -79,41 +80,73 @@ class ObservableCtrl @Inject() ( ) def create(caseId: String): Action[AnyContent] = - entryPoint("create artifact") - .extract("artifact", FieldsParser[InputObservable]) + entrypoint("create observable") + .extract("observable", FieldsParser[InputObservable]) .extract("isZip", FieldsParser.boolean.optional.on("isZip")) .extract("zipPassword", FieldsParser.string.optional.on("zipPassword")) - .authTransaction(db) { implicit request => implicit graph => + .auth { implicit request => + val inputObservable: InputObservable = request.body("observable") val isZip: Option[Boolean] = request.body("isZip") val zipPassword: Option[String] = request.body("zipPassword") - val inputObservable: InputObservable = request.body("artifact") val inputAttachObs = if (isZip.contains(true)) getZipFiles(inputObservable, zipPassword) else Seq(inputObservable) - for { - case0 <- - caseSrv - .get(EntityIdOrName(caseId)) - .can(Permissions.manageObservable) - .getOrFail("Case") - observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) - observablesWithData <- - inputObservable - .data - .toTry(d => observableSrv.create(inputObservable.toObservable, observableType, d, inputObservable.tags, Nil)) - observableWithAttachment <- inputAttachObs.toTry( - _.attachment - .map(a => observableSrv.create(inputObservable.toObservable, observableType, a, inputObservable.tags, Nil)) - .flip - ) - createdObservables <- (observablesWithData ++ observableWithAttachment.flatten).toTry { richObservables => - caseSrv - .addObservable(case0, richObservables) - .map(_ => richObservables) + + db + .roTransaction { implicit graph => + for { + case0 <- + caseSrv + .get(EntityIdOrName(caseId)) + .can(Permissions.manageObservable) + .orFail(AuthorizationError("Operation not permitted")) + observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) + } yield (case0, observableType) + } + .map { + case (case0, observableType) => + val initialSuccessesAndFailures: (Seq[JsValue], Seq[JsValue]) = + inputAttachObs.foldLeft[(Seq[JsValue], Seq[JsValue])](Nil -> Nil) { + case ((successes, failures), inputObservable) => + inputObservable.attachment.fold((successes, failures)) { attachment => + db + .tryTransaction { implicit graph => + observableSrv + .create(inputObservable.toObservable, observableType, attachment, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) + } + .fold( + e => + successes -> (failures :+ errorHandler.toErrorResult(e)._2 ++ Json + .obj( + "object" -> Json + .obj("data" -> s"file:${attachment.filename}", "attachment" -> Json.obj("name" -> attachment.filename)) + )), + s => (successes :+ s) -> failures + ) + } + } + + val (successes, failures) = inputObservable + .data + .foldLeft(initialSuccessesAndFailures) { + case ((successes, failures), data) => + db + .tryTransaction { implicit graph => + observableSrv + .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) + } + .fold( + failure => (successes, failures :+ errorHandler.toErrorResult(failure)._2 ++ Json.obj("object" -> Json.obj("data" -> data))), + success => (successes :+ success, failures) + ) + } + if (failures.isEmpty) Results.Created(JsArray(successes)) + else Results.MultiStatus(Json.obj("success" -> successes, "failure" -> failures)) } - } yield Results.Created(createdObservables.toJson) } def get(observableId: String): Action[AnyContent] = - entryPoint("get observable") + entrypoint("get observable") .authRoTransaction(db) { _ => implicit graph => observableSrv .get(EntityIdOrName(observableId)) @@ -126,7 +159,7 @@ class ObservableCtrl @Inject() ( } def update(observableId: String): Action[AnyContent] = - entryPoint("update observable") + entrypoint("update observable") .extract("observable", FieldsParser.update("observable", publicProperties)) .authTransaction(db) { implicit request => implicit graph => val propertyUpdaters: Seq[PropertyUpdater] = request.body("observable") @@ -139,7 +172,7 @@ class ObservableCtrl @Inject() ( } def bulkUpdate: Action[AnyContent] = - entryPoint("bulk update") + entrypoint("bulk update") .extract("input", FieldsParser.update("observable", publicProperties)) .extract("ids", FieldsParser.seq[String].on("ids")) .authTransaction(db) { implicit request => implicit graph => @@ -154,7 +187,7 @@ class ObservableCtrl @Inject() ( } def delete(obsId: String): Action[AnyContent] = - entryPoint("delete") + entrypoint("delete") .authTransaction(db) { implicit request => implicit graph => for { observable <- diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index feffe865bb..d6f6e97945 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -7,17 +7,29 @@ import play.api.routing.sird._ @Singleton class Router @Inject() ( + authenticationCtrl: AuthenticationCtrl, + alertCtrl: AlertCtrl, + // attachmentCtrl: AttachmentCtrl, + auditCtrl: AuditCtrl, caseCtrl: CaseCtrl, caseTemplateCtrl: CaseTemplateCtrl, - userCtrl: UserCtrl, + // configCtrl: ConfigCtrl, + customFieldCtrl: CustomFieldCtrl, + // dashboardCtrl: DashboardCtrl, + describeCtrl: DescribeCtrl, + logCtrl: LogCtrl, + observableCtrl: ObservableCtrl, + observableTypeCtrl: ObservableTypeCtrl, organisationCtrl: OrganisationCtrl, + // pageCtrl: PageCtrl, + // permissionCtrl: PermissionCtrl, + profileCtrl: ProfileCtrl, taskCtrl: TaskCtrl, - customFieldCtrl: CustomFieldCtrl, - alertCtrl: AlertCtrl, - auditCtrl: AuditCtrl, - statusCtrl: StatusCtrl, - authenticationCtrl: AuthenticationCtrl, - describeCtrl: DescribeCtrl + // shareCtrl: ShareCtrl, + userCtrl: UserCtrl, + statusCtrl: StatusCtrl + // streamCtrl: StreamCtrl, + // tagCtrl: TagCtrl ) extends SimpleRouter { override def routes: Routes = { @@ -40,6 +52,14 @@ class Router @Inject() ( // case POST(p"/case/_stats") => caseCtrl.stats() // case GET(p"/case/$caseId/links") => caseCtrl.linkedCases(caseId) + case POST(p"/case/$caseId/observable") => observableCtrl.create(caseId) + case GET(p"/observable/$observableId") => observableCtrl.get(observableId) + case DELETE(p"/observable/$observableId") => observableCtrl.delete(observableId) + case PATCH(p"/observable/_bulk") => observableCtrl.bulkUpdate + case PATCH(p"/observable/$observableId") => observableCtrl.update(observableId) +// case GET(p"/observable/$observableId/similar") => observableCtrl.findSimilar(observableId) +// case POST(p"/observable/$observableId/shares") => shareCtrl.shareObservable(observableId) + case GET(p"/caseTemplate") => caseTemplateCtrl.list case POST(p"/caseTemplate") => caseTemplateCtrl.create case GET(p"/caseTemplate/$caseTemplateId") => caseTemplateCtrl.get(caseTemplateId) @@ -75,6 +95,10 @@ class Router @Inject() ( // POST /case/:caseId/task/_search controllers.TaskCtrl.findInCase(caseId) // POST /case/task/_stats controllers.TaskCtrl.stats() + case POST(p"/task/$taskId/log") => logCtrl.create(taskId) + case PATCH(p"/log/$logId") => logCtrl.update(logId) + case DELETE(p"/log/$logId") => logCtrl.delete(logId) + case GET(p"/customField") => customFieldCtrl.list case POST(p"/customField") => customFieldCtrl.create @@ -96,8 +120,16 @@ class Router @Inject() ( // POST /audit/_search controllers.AuditCtrl.find() // POST /audit/_stats controllers.AuditCtrl.stats() + case POST(p"/profile") => profileCtrl.create + case GET(p"/profile/$profileId") => profileCtrl.get(profileId) + case PATCH(p"/profile/$profileId") => profileCtrl.update(profileId) + case DELETE(p"/profile/$profileId") => profileCtrl.delete(profileId) + case GET(p"/describe/_all") => describeCtrl.describeAll case GET(p"/describe/$modelName") => describeCtrl.describe(modelName) + case GET(p"/observable/type/$idOrName") => observableTypeCtrl.get(idOrName) + case POST(p"/observable/type") => observableTypeCtrl.create + case DELETE(p"/observable/type/$idOrName") => observableTypeCtrl.delete(idOrName) } } From 7c8fdc226e432b2c774be8aa98fa0179917b9c4c Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 10 Dec 2020 16:50:04 +0100 Subject: [PATCH 057/324] #1454 Fix some queries in frontend --- .../app/scripts/controllers/case/CaseObservablesCtrl.js | 4 ++-- .../app/scripts/controllers/case/ObservablesStatsCtrl.js | 6 +++--- frontend/app/scripts/services/EntitySrv.js | 6 ++++++ frontend/app/scripts/services/api/CortexSrv.js | 2 +- frontend/app/scripts/services/api/TagSrv.js | 2 +- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js b/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js index 919c1d67b3..4045348b1f 100644 --- a/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js @@ -18,7 +18,7 @@ }; this.$onInit = function() { - $scope.filtering = new FilteringSrv('case_artifact', 'observable.list', { + $scope.filtering = new FilteringSrv('observable', 'observable.list', { version: 'v1', defaults: { showFilters: true, @@ -68,7 +68,7 @@ $scope.artifacts = new PaginatedQuerySrv({ name: 'observables', root: $scope.caseId, - objectType: 'case_artifact', + objectType: 'obserable', version: 'v1', scope: $scope, sort: $scope.filtering.context.sort, diff --git a/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js b/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js index e48194916b..f50102984c 100644 --- a/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js +++ b/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js @@ -42,7 +42,7 @@ ], { scope: $scope, rootId: caseId, - objectType: 'case_artifact', + objectType: 'observable', query: { params: { name: 'observables-by-tags-stats-' + caseId @@ -69,7 +69,7 @@ ], { scope: $scope, rootId: caseId, - objectType: 'case_artifact', + objectType: 'observable', query: { params: { name: 'observables-by-type-stats-' + caseId @@ -95,7 +95,7 @@ ], { scope: $scope, rootId: caseId, - objectType: 'case_artifact', + objectType: 'observable', query: { params: { name: 'observables-by-ioc-stats-' + caseId diff --git a/frontend/app/scripts/services/EntitySrv.js b/frontend/app/scripts/services/EntitySrv.js index f531eff88f..d20d1d97b6 100644 --- a/frontend/app/scripts/services/EntitySrv.js +++ b/frontend/app/scripts/services/EntitySrv.js @@ -19,6 +19,12 @@ caseId: entity.case.id, itemId: entity.id }; + } else if (entity._type === 'observable') { + state.name = 'app.case.observables-item'; + state.params = { + caseId: entity.case.id, + itemId: entity.id + }; } else if (entity._type === 'case_artifact_job') { state.name = 'app.case.observables-item'; state.params = { diff --git a/frontend/app/scripts/services/api/CortexSrv.js b/frontend/app/scripts/services/api/CortexSrv.js index 00921b0fae..9bd98f897a 100644 --- a/frontend/app/scripts/services/api/CortexSrv.js +++ b/frontend/app/scripts/services/api/CortexSrv.js @@ -14,7 +14,7 @@ streamObjectType: 'case_artifact_job', filter: { _parent: { - _type: 'case_artifact', + _type: 'observable', _query: { _id: observableId } diff --git a/frontend/app/scripts/services/api/TagSrv.js b/frontend/app/scripts/services/api/TagSrv.js index d90300a3ca..ce9035f11f 100644 --- a/frontend/app/scripts/services/api/TagSrv.js +++ b/frontend/app/scripts/services/api/TagSrv.js @@ -41,7 +41,7 @@ switch(entity) { case 'case': return self.fromCases(query); - case 'case_artifact': + case 'observable': return self.fromObservables(query); case 'alert': return self.fromAlerts(query); From 04afbe63f1c73d3be7b4344adeb967b39be72274 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 10 Dec 2020 22:05:25 +0100 Subject: [PATCH 058/324] #1668 WIP: Update taxonomies list admin page: add filtering, pagination, enable/disable, delete --- .../admin/taxonomy/TaxonomyListCtrl.js | 139 ++++++++++++++---- .../app/scripts/services/api/TaxonomySrv.js | 14 -- frontend/app/styles/main.css | 7 + .../views/partials/admin/taxonomy/list.html | 65 ++++++-- .../partials/admin/taxonomy/list/filters.html | 38 +++++ .../partials/admin/taxonomy/list/toolbar.html | 9 ++ 6 files changed, 218 insertions(+), 54 deletions(-) create mode 100644 frontend/app/views/partials/admin/taxonomy/list/filters.html diff --git a/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js b/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js index 47661db02f..3763c96c44 100644 --- a/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js +++ b/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js @@ -5,19 +5,43 @@ .controller('TaxonomyListCtrl', TaxonomyListCtrl) .controller('TaxonomyImportCtrl', TaxonomyImportCtrl); - function TaxonomyListCtrl($uibModal, TaxonomySrv, NotificationSrv, ModalSrv, appConfig) { + function TaxonomyListCtrl($scope, $uibModal, PaginatedQuerySrv, FilteringSrv, TaxonomySrv, NotificationSrv, ModalSrv, appConfig) { var self = this; this.appConfig = appConfig; self.load = function() { - TaxonomySrv.list() - .then(function(response) { - self.list = response; - }) - .catch(function(rejection) { - NotificationSrv.error('Taxonomies management', rejection.data, rejection.status); - }); + this.loading = true; + + // TaxonomySrv.list() + // .then(function(response) { + // self.list = response; + // }) + // .catch(function(rejection) { + // NotificationSrv.error('Taxonomies management', rejection.data, rejection.status); + // }) + // .finally(function(){ + // //self.loading = false; + // }); + + this.list = new PaginatedQuerySrv({ + name: 'taxonomies', + root: undefined, + objectType: 'taxonomy', + version: 'v1', + scope: $scope, + sort: self.filtering.context.sort, + loadAll: false, + pageSize: self.filtering.context.pageSize, + filter: this.filtering.buildQuery(), + operations: [ + {'_name': 'listTaxonomy'} + ], + extraData: ['enabled'], + onUpdate: function() { + self.loading = false; + } + }); }; self.import = function () { @@ -26,7 +50,10 @@ templateUrl: 'views/partials/admin/taxonomy/import.html', controller: 'TaxonomyImportCtrl', controllerAs: '$vm', - size: 'lg' + size: 'lg', + resolve: { + appConfig: self.appConfig + } }); modalInstance.result @@ -40,10 +67,12 @@ }); }; - this.toggleActive = function(id, active) { - TaxonomySrv.toggleActive(id, active) + this.toggleActive = function(taxonomy) { + var active = !taxonomy.extraData.enabled; + + TaxonomySrv.toggleActive(taxonomy._id, active) .then(function() { - NotificationSrv.log('Taxonomy has been successfully ' + active ? 'activated' : 'deactivated', 'success'); + NotificationSrv.log(['Taxonomy [', taxonomy.namespace, '] has been successfully', (active ? 'activated' : 'deactivated')].join(' '), 'success'); self.load(); }) @@ -54,36 +83,86 @@ }); }; - self.update = function(id, taxonomy) { - // TODO - // TaxonomySrv.update(id, _.pick(taxonomy, '...')) - TaxonomySrv.update(id, _.pick(taxonomy, '...')) - .then(function(/*response*/) { + self.remove = function(taxonomy) { + var modalInstance = ModalSrv.confirm( + 'Remove taxonomy', + 'Are you sure you want to remove the selected taxonomy?', { + flavor: 'danger', + okText: 'Yes, remove it' + } + ); + + modalInstance.result + .then(function() { + return TaxonomySrv.remove(taxonomy._id); + }) + .then(function( /*response*/ ) { self.load(); - NotificationSrv.log('Taxonomy updated successfully', 'success'); + NotificationSrv.success( + 'Taxonomy ' + taxonomy.namespace + ' has been successfully removed.' + ); }) .catch(function(err) { - NotificationSrv.error('Error', 'Taxonomy update failed', err.status); + if (err && !_.isString(err)) { + NotificationSrv.error('TaxonomyListCtrl', err.data, err.status); + } }); }; - self.create = function(taxonomy) { - TaxonomySrv.create(taxonomy) - .then(function(/*response*/) { - self.load(); - NotificationSrv.log('Taxonomy created successfully', 'success'); - }) - .catch(function(err) { - NotificationSrv.error('Error', 'Taxonomy creation failed', err.status); - }); + this.toggleFilters = function () { + this.filtering.toggleFilters(); }; - self.$onInit = function() { + this.filter = function () { + self.filtering.filter().then(this.applyFilters); + }; + + this.clearFilters = function () { + this.filtering.clearFilters() + .then(self.search); + }; + + this.removeFilter = function (index) { + self.filtering.removeFilter(index) + .then(self.search); + }; + + this.search = function () { self.load(); + self.filtering.storeContext(); + }; + this.addFilterValue = function (field, value) { + this.filtering.addFilterValue(field, value); + this.search(); + }; + + self.$onInit = function() { + //self.load(); + + self.filtering = new FilteringSrv('taxonomy', 'taxonomy.list', { + version: 'v1', + defaults: { + showFilters: true, + showStats: false, + pageSize: 15, + sort: ['+namespace'] + }, + defaultFilter: [] + }); + + self.filtering.initContext('list') + .then(function() { + self.load(); + + $scope.$watch('$vm.list.pageSize', function (newValue) { + self.filtering.setPageSize(newValue); + }); + }); }; } - function TaxonomyImportCtrl($uibModalInstance, TaxonomySrv, NotificationSrv) { + function TaxonomyImportCtrl($uibModalInstance, TaxonomySrv, NotificationSrv, appConfig) { + this.appConfig = appConfig; this.formData = {}; this.ok = function () { diff --git a/frontend/app/scripts/services/api/TaxonomySrv.js b/frontend/app/scripts/services/api/TaxonomySrv.js index 747b9a4a52..70aafedca8 100644 --- a/frontend/app/scripts/services/api/TaxonomySrv.js +++ b/frontend/app/scripts/services/api/TaxonomySrv.js @@ -2,21 +2,14 @@ 'use strict'; angular.module('theHiveServices') .service('TaxonomySrv', function($http, QuerySrv) { - // var self = this; var baseUrl = './api/v1/taxonomy'; this.list = function() { - // return $http.get(baseUrl, {params: { - // range: 'all' - // }}); - // return QuerySrv.call('v1', [ { _name: 'listTaxonomy' } ], { name:'list-taxonomies' }); - - //listTaxonomies }; this.get = function(name) { @@ -27,13 +20,6 @@ return $http.put([baseUrl, id, !!active ? 'activate' : 'deactivate'].join('/')); }; - // this.map = function() { - // return self.list() - // .then(function(response) { - // return _.indexBy(response.data, 'name'); - // }); - // }; - this.create = function(profile) { return $http.post(baseUrl, profile); }; diff --git a/frontend/app/styles/main.css b/frontend/app/styles/main.css index 0425ebbcd9..7830c6cc1a 100644 --- a/frontend/app/styles/main.css +++ b/frontend/app/styles/main.css @@ -76,6 +76,13 @@ body { text-align: center; padding: 40px; } +.loading-message { + background-color: #f5f5f5; + color: #AAA; + font-size: 18px; + text-align: center; + padding: 40px; +} .tpad50 { padding-top: 50px diff --git a/frontend/app/views/partials/admin/taxonomy/list.html b/frontend/app/views/partials/admin/taxonomy/list.html index e8eb9c5aa0..734bb087c4 100644 --- a/frontend/app/views/partials/admin/taxonomy/list.html +++ b/frontend/app/views/partials/admin/taxonomy/list.html @@ -7,26 +7,55 @@

    List of taxonomies

    -
    +
    + +
    +
    + + +
    +
    + +
    No taxnomies found.
    +
    +
    +
    + + loading taxonomies... +
    +
    +
    + -
    +
    + +
    - + + - + - + + -
    NameNamespace Description # Tags
    + +
    {{::taxonomy.namespace}} @@ -38,17 +67,33 @@

    List of taxonomies

    		 {{::taxonomy.tags.length}} - - Edit + +
    + + + + + + + + + + + + +
    - - Delete
    + +
    diff --git a/frontend/app/views/partials/admin/taxonomy/list/filters.html b/frontend/app/views/partials/admin/taxonomy/list/filters.html new file mode 100644 index 0000000000..431c826ca5 --- /dev/null +++ b/frontend/app/views/partials/admin/taxonomy/list/filters.html @@ -0,0 +1,38 @@ +
    +
    +

    Filters

    +
    +
    +
    +
    + + + + +
    +
    +
    + +
    +
    +
    +
    + + Add a filter + + + Clear + + +
    +
    +
    + +
    +
    diff --git a/frontend/app/views/partials/admin/taxonomy/list/toolbar.html b/frontend/app/views/partials/admin/taxonomy/list/toolbar.html index e7b922a3d6..87fe159685 100644 --- a/frontend/app/views/partials/admin/taxonomy/list/toolbar.html +++ b/frontend/app/views/partials/admin/taxonomy/list/toolbar.html @@ -7,6 +7,15 @@ Import taxonomies
    +
    + +
    + +
    + +
    From e9ba4aa196d3cd41e56341acf5f9f016e9f62fee Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 11 Dec 2020 06:34:13 +0100 Subject: [PATCH 059/324] #1668 WIP: add taonomy view dialog to list tags --- frontend/app/index.html | 1 + .../components/common/tag.component.js | 22 +++++++ .../admin/taxonomy/TaxonomyListCtrl.js | 52 +++++++++++---- .../components/common/tag.component.html | 1 + .../views/partials/admin/taxonomy/list.html | 10 ++- .../views/partials/admin/taxonomy/view.html | 65 +++++++++++++++++++ 6 files changed, 135 insertions(+), 16 deletions(-) create mode 100644 frontend/app/scripts/components/common/tag.component.js create mode 100644 frontend/app/views/components/common/tag.component.html create mode 100644 frontend/app/views/partials/admin/taxonomy/view.html diff --git a/frontend/app/index.html b/frontend/app/index.html index 9a4760768f..57ad3fda5a 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -139,6 +139,7 @@ + diff --git a/frontend/app/scripts/components/common/tag.component.js b/frontend/app/scripts/components/common/tag.component.js new file mode 100644 index 0000000000..8876df5802 --- /dev/null +++ b/frontend/app/scripts/components/common/tag.component.js @@ -0,0 +1,22 @@ +(function() { + 'use strict'; + + angular.module('theHiveComponents') + .component('tag', { + controller: function() { + this.$onInit = function() { + this.tag = _.without([ + this.value.namespace, + ':', + this.value.predicate, + this.value.value ? ("=\"" + this.value.value + "\"") : null + ], null).join(''); + }; + }, + controllerAs: '$ctrl', + templateUrl: 'views/components/common/tag.component.html', + bindings: { + value: '<' + } + }); +})(); diff --git a/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js b/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js index 3763c96c44..78a1d5c29b 100644 --- a/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js +++ b/frontend/app/scripts/controllers/admin/taxonomy/TaxonomyListCtrl.js @@ -3,6 +3,7 @@ angular.module('theHiveControllers') .controller('TaxonomyListCtrl', TaxonomyListCtrl) + .controller('TaxonomyDialogCtrl', TaxonomyDialogCtrl) .controller('TaxonomyImportCtrl', TaxonomyImportCtrl); function TaxonomyListCtrl($scope, $uibModal, PaginatedQuerySrv, FilteringSrv, TaxonomySrv, NotificationSrv, ModalSrv, appConfig) { @@ -13,17 +14,6 @@ self.load = function() { this.loading = true; - // TaxonomySrv.list() - // .then(function(response) { - // self.list = response; - // }) - // .catch(function(rejection) { - // NotificationSrv.error('Taxonomies management', rejection.data, rejection.status); - // }) - // .finally(function(){ - // //self.loading = false; - // }); - this.list = new PaginatedQuerySrv({ name: 'taxonomies', root: undefined, @@ -44,6 +34,32 @@ }); }; + self.show = function(taxonomy) { + // var modalInstance = $uibModal.open({ + + $uibModal.open({ + animation: true, + templateUrl: 'views/partials/admin/taxonomy/view.html', + controller: 'TaxonomyDialogCtrl', + controllerAs: '$modal', + size: 'max', + resolve: { + taxonomy: angular.copy(taxonomy) + } + }); + + // modalInstance.result + // .then(function() { + // self.load(); + // }) + // .catch(function(err){ + // if(err && !_.isString(err)) { + // NotificationSrv.error('Taxonomies import', err.data, err.status); + // } + // }); + }; + + self.import = function () { var modalInstance = $uibModal.open({ animation: true, @@ -137,8 +153,6 @@ }; self.$onInit = function() { - //self.load(); - self.filtering = new FilteringSrv('taxonomy', 'taxonomy.list', { version: 'v1', defaults: { @@ -161,6 +175,18 @@ }; } + function TaxonomyDialogCtrl($uibModalInstance, TaxonomySrv, NotificationSrv, taxonomy) { + this.taxonomy = taxonomy; + + this.ok = function () { + $uibModalInstance.close(); + }; + + this.cancel = function () { + $uibModalInstance.dismiss('cancel'); + }; + } + function TaxonomyImportCtrl($uibModalInstance, TaxonomySrv, NotificationSrv, appConfig) { this.appConfig = appConfig; this.formData = {}; diff --git a/frontend/app/views/components/common/tag.component.html b/frontend/app/views/components/common/tag.component.html new file mode 100644 index 0000000000..9646fdddcf --- /dev/null +++ b/frontend/app/views/components/common/tag.component.html @@ -0,0 +1 @@ +{{$ctrl.tag}} diff --git a/frontend/app/views/partials/admin/taxonomy/list.html b/frontend/app/views/partials/admin/taxonomy/list.html index 734bb087c4..6d10f519f5 100644 --- a/frontend/app/views/partials/admin/taxonomy/list.html +++ b/frontend/app/views/partials/admin/taxonomy/list.html @@ -44,6 +44,7 @@

    List of taxonomies

    Namespace Description + Version # Tags @@ -58,12 +59,15 @@

    List of taxonomies

    - {{::taxonomy.namespace}} + {{::taxonomy.namespace}}
    {{::taxonomy.description}} + + {{::taxonomy.version}} + {{::taxonomy.tags.length}} @@ -78,8 +82,8 @@

    List of taxonomies

    }[!!taxonomy.extraData.enabled]"> - - + + diff --git a/frontend/app/views/partials/admin/taxonomy/view.html b/frontend/app/views/partials/admin/taxonomy/view.html new file mode 100644 index 0000000000..3a50127b1c --- /dev/null +++ b/frontend/app/views/partials/admin/taxonomy/view.html @@ -0,0 +1,65 @@ +
    + +
    +

    {{::$modal.taxonomy.namespace}} taxonomy

    +
    + +
    +
    +
    +
    + +
    {{::$modal.taxonomy.namespace}}
    +
    +
    +
    +
    + +
    {{::$modal.taxonomy.version}}
    +
    +
    +
    + +
    + +
    {{::$modal.taxonomy.description}}
    +
    + +
    +
    + + + + + + + + + + + + + + + + + + + +
    TagPredicateValueColor
    + + + + {{::tag.predicate}}{{::tag.value || '-'}}{{::tag.colour}} + + + +
    +
    +
    +
    +
    + + +
    +
    From 745fcea71a3f7dba4a5bc1f50ea50f44841097f9 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 11 Dec 2020 09:19:48 +0100 Subject: [PATCH 060/324] #1668 WIP: allow filtering by taxonomy status --- frontend/app/views/partials/admin/taxonomy/list.html | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/app/views/partials/admin/taxonomy/list.html b/frontend/app/views/partials/admin/taxonomy/list.html index 6d10f519f5..f8aa93d385 100644 --- a/frontend/app/views/partials/admin/taxonomy/list.html +++ b/frontend/app/views/partials/admin/taxonomy/list.html @@ -52,10 +52,12 @@

    List of taxonomies

    - + + +
    From eae7bbc091331eaf679fce3a05e89c1afdb2fcb2 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Fri, 11 Dec 2020 18:55:26 +0100 Subject: [PATCH 061/324] WIP Changed tag colour property type - Cortex tests failing --- ScalliGraph | 2 +- .../scala/org/thp/thehive/dto/v0/Tag.scala | 4 +- .../scala/org/thp/thehive/dto/v1/Tag.scala | 2 +- .../src/main/scala/org/thp/misp/dto/Tag.scala | 7 +-- .../misp/services/MispImportSrvTest.scala | 2 +- .../thehive/controllers/v0/AlertCtrl.scala | 2 +- .../thp/thehive/controllers/v0/CaseCtrl.scala | 2 +- .../controllers/v0/CaseTemplateCtrl.scala | 2 +- .../controllers/v0/ObservableCtrl.scala | 12 +++-- .../thp/thehive/controllers/v0/TagCtrl.scala | 12 ++--- .../controllers/v1/ObservableCtrl.scala | 10 ++--- .../thehive/controllers/v1/Properties.scala | 44 +++++++------------ .../thehive/controllers/v1/TaxonomyCtrl.scala | 4 +- .../controllers/v1/TaxonomyRenderer.scala | 8 ++-- thehive/app/org/thp/thehive/models/Tag.scala | 15 +++---- .../models/TheHiveSchemaDefinition.scala | 11 +++++ .../thp/thehive/services/AttachmentSrv.scala | 2 +- .../app/org/thp/thehive/services/TagSrv.scala | 18 ++------ .../controllers/v0/ConfigCtrlTest.scala | 8 +++- .../controllers/v1/TaxonomyCtrlTest.scala | 14 +++--- .../test/org/thp/thehive/models/TagTest.scala | 16 +++---- thehive/test/resources/data/Tag.json | 22 +++++----- 22 files changed, 97 insertions(+), 122 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 856e64f3e1..9aa06293e3 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 856e64f3e1b262821a9d5b8c402ebc13f7562f18 +Subproject commit 9aa06293e32254466d1c5d7ae089755fdfbbe4e0 diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Tag.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Tag.scala index 46cdc7fd5e..d994f6fc38 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Tag.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Tag.scala @@ -2,13 +2,13 @@ package org.thp.thehive.dto.v0 import play.api.libs.json.{Json, OFormat, OWrites} -case class InputTag(namespace: String, predicate: String, value: Option[String], description: Option[String], colour: Option[Int]) +case class InputTag(namespace: String, predicate: String, value: Option[String], description: Option[String], colour: Option[String]) object InputTag { implicit val writes: OWrites[InputTag] = Json.writes[InputTag] } -case class OutputTag(namespace: String, predicate: String, value: Option[String], description: Option[String], colour: Int) +case class OutputTag(namespace: String, predicate: String, value: Option[String], description: Option[String], colour: String) object OutputTag { implicit val format: OFormat[OutputTag] = Json.format[OutputTag] diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala index 3b536c867c..13f6d33193 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Tag.scala @@ -7,7 +7,7 @@ case class OutputTag( predicate: String, value: Option[String], description: Option[String], - colour: Int + colour: String ) object OutputTag { diff --git a/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala b/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala index 683b1ee489..eef50fdcdc 100644 --- a/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala +++ b/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala @@ -6,7 +6,7 @@ import play.api.libs.json._ case class Tag( id: Option[String], name: String, - colour: Option[Int], + colour: Option[String], exportable: Option[Boolean] ) @@ -14,10 +14,7 @@ object Tag { implicit val reads: Reads[Tag] = ((JsPath \ "id").readNullable[String] and (JsPath \ "name").read[String] and - (JsPath \ "colour").readNullable[String].map { - case Some(c) if c.headOption.contains('#') => Some(Integer.parseUnsignedInt(c.tail, 16)) - case _ => None - } and + (JsPath \ "colour").readNullable[String] and (JsPath \ "exportable").readNullable[Boolean])(Tag.apply _) implicit val writes: Writes[Tag] = Json.writes[Tag] diff --git a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala index 915ef429c0..4062eaef86 100644 --- a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala +++ b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala @@ -65,7 +65,7 @@ class MispImportSrvTest(implicit ec: ExecutionContext) extends PlaySpecification attributeCount = Some(11), distribution = 1, attributes = Nil, - tags = Seq(Tag(Some("1"), "TH-test", Some(0x36a3a3), None), Tag(Some("2"), "TH-test-2", Some(0x1ac7c7), None)) + tags = Seq(Tag(Some("1"), "TH-test", Some("#36a3a3"), None), Tag(Some("2"), "TH-test-2", Some("#1ac7c7"), None)) ) ) } diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index d0ba0e18fe..7d8ac401ea 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -391,7 +391,7 @@ class PublicAlert @Inject() ( val namespace = UMapping.string.getProperty(v, "namespace") val predicate = UMapping.string.getProperty(v, "predicate") val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString + Tag(namespace, predicate, value, None, "#000000").toString }, Converter.identity[String] ) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index 71dff80ff5..afcbf55831 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -245,7 +245,7 @@ class PublicCase @Inject() ( val namespace = UMapping.string.getProperty(v, "namespace") val predicate = UMapping.string.getProperty(v, "predicate") val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString + Tag(namespace, predicate, value, None, "#000000").toString }, Converter.identity[String] ) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala index 5a1d824314..e8845fd84c 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala @@ -129,7 +129,7 @@ class PublicCaseTemplate @Inject() ( val namespace = UMapping.string.getProperty(v, "namespace") val predicate = UMapping.string.getProperty(v, "predicate") val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString + Tag(namespace, predicate, value, None, "#000000").toString }, Converter.identity[String] ) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 629711357d..2b280f0eae 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -1,13 +1,8 @@ package org.thp.thehive.controllers.v0 -import java.io.FilterInputStream -import java.nio.file.Files - -import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ -import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ @@ -27,6 +22,9 @@ import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.libs.json.{JsArray, JsObject, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} +import java.io.FilterInputStream +import java.nio.file.Files +import javax.inject.{Inject, Named, Singleton} import scala.collection.JavaConverters._ import scala.util.Success @@ -214,7 +212,7 @@ class ObservableCtrl @Inject() ( } } - private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = + private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] = observable.attachment.toSeq.flatMap { attachment => val zipFile = new ZipFile(attachment.filepath.toFile) val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] @@ -288,7 +286,7 @@ class PublicObservable @Inject() ( val namespace = UMapping.string.getProperty(v, "namespace") val predicate = UMapping.string.getProperty(v, "predicate") val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString + Tag(namespace, predicate, value, None, "#000000").toString }, Converter.identity[String] ) diff --git a/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala index 45a85f7abf..a6045ef70f 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala @@ -17,8 +17,6 @@ import org.thp.thehive.services.TagSrv import play.api.libs.json.{JsNumber, JsObject, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} -import scala.util.Try - class TagCtrl @Inject() ( override val entrypoint: Entrypoint, @Named("with-thehive-schema") override val db: Database, @@ -68,13 +66,10 @@ class TagCtrl @Inject() ( colour = (entry \ "colour") .asOpt[String] - .map(parseColour) - .getOrElse(0) // black + .getOrElse("#000000") e = (entry \ "description").asOpt[String] orElse (entry \ "expanded").asOpt[String] } yield Tag(namespace, predicate, Some(v), e, colour) - def parseColour(colour: String): Int = if (colour(0) == '#') Try(Integer.parseUnsignedInt(colour.tail, 16)).getOrElse(0) else 0 - private def distinct(valueOpt: Option[String], acc: (Seq[JsObject], Seq[String]), v: JsObject): (Seq[JsObject], Seq[String]) = if (valueOpt.isDefined && acc._2.contains(valueOpt.get)) acc else (acc._1 :+ v, valueOpt.fold(acc._2)(acc._2 :+ _)) @@ -90,8 +85,7 @@ class TagCtrl @Inject() ( colour = (predicate \ "colour") .asOpt[String] - .map(parseColour) - .getOrElse(0) // black + .getOrElse("#000000") } yield Tag(namespace, v, None, e, colour) def get(tagId: String): Action[AnyContent] = @@ -142,7 +136,7 @@ class PublicTag @Inject() (tagSrv: TagSrv) extends PublicData { val namespace = UMapping.string.getProperty(v, "namespace") val predicate = UMapping.string.getProperty(v, "predicate") val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString + Tag(namespace, predicate, value, None, "#000000").toString }, Converter.identity[String] ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index f383a7a025..c81e9e7c9b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -1,13 +1,8 @@ package org.thp.thehive.controllers.v1 -import java.io.FilterInputStream -import java.nio.file.Files - -import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ -import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -25,6 +20,9 @@ import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.mvc.{Action, AnyContent, Results} import play.api.{Configuration, Logger} +import java.io.FilterInputStream +import java.nio.file.Files +import javax.inject.{Inject, Named, Singleton} import scala.collection.JavaConverters._ @Singleton @@ -196,7 +194,7 @@ class ObservableCtrl @Inject() ( } } - private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = + private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] = observable.attachment.toSeq.flatMap { attachment => val zipFile = new ZipFile(attachment.filepath.toFile) val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 54ff5f31a4..0c986745af 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -1,9 +1,6 @@ package org.thp.thehive.controllers.v1 -import java.lang.{Long => JLong} -import java.util.Date - -import javax.inject.{Inject, Named, Singleton} +import org.apache.tinkerpop.gremlin.structure.Vertex import org.thp.scalligraph.controllers.{FPathElem, FPathEmpty} import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query.{PublicProperties, PublicPropertyListBuilder} @@ -26,6 +23,9 @@ import org.thp.thehive.services.UserOps._ import org.thp.thehive.services._ import play.api.libs.json.{JsObject, JsValue, Json} +import java.lang.{Long => JLong} +import java.util.Date +import javax.inject.{Inject, Named, Singleton} import scala.util.Failure @Singleton @@ -65,12 +65,7 @@ class Properties @Inject() ( cases .tags .graphMap[String, String, Converter.Identity[String]]( - { v => - val namespace = UMapping.string.getProperty(v, "namespace") - val predicate = UMapping.string.getProperty(v, "predicate") - val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString - }, + vertexToTag, Converter.identity[String] ) ) @@ -171,12 +166,7 @@ class Properties @Inject() ( cases .tags .graphMap[String, String, Converter.Identity[String]]( - { v => - val namespace = UMapping.string.getProperty(v, "namespace") - val predicate = UMapping.string.getProperty(v, "predicate") - val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString - }, + vertexToTag, Converter.identity[String] ) ) @@ -356,12 +346,7 @@ class Properties @Inject() ( cases .tags .graphMap[String, String, Converter.Identity[String]]( - { v => - val namespace = UMapping.string.getProperty(v, "namespace") - val predicate = UMapping.string.getProperty(v, "predicate") - val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString - }, + vertexToTag, Converter.identity[String] ) ) @@ -465,12 +450,7 @@ class Properties @Inject() ( cases .tags .graphMap[String, String, Converter.Identity[String]]( - { v => - val namespace = UMapping.string.getProperty(v, "namespace") - val predicate = UMapping.string.getProperty(v, "predicate") - val value = UMapping.string.optional.getProperty(v, "value") - Tag(namespace, predicate, value, None, 0).toString - }, + vertexToTag, Converter.identity[String] ) ) @@ -500,4 +480,12 @@ class Properties @Inject() ( .property("version", UMapping.int)(_.field.readonly) .property("enabled", UMapping.boolean)(_.select(_.enabled).readonly) .build + + private def vertexToTag: Vertex => String = { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, "#000000").toString + } + } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala index cd73717175..77fcb66925 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala @@ -49,7 +49,7 @@ class TaxonomyCtrl @Inject() ( { case (OutputParam(from, to, extraData), taxoSteps, authContext) => taxoSteps.richPage(from, to, extraData.contains("total")) { - _.richTaxonomyWithCustomRenderer(taxoStatsRenderer(extraData - "total")(authContext)) + _.richTaxonomyWithCustomRenderer(taxoStatsRenderer(extraData - "total")) } } ) @@ -114,7 +114,7 @@ class TaxonomyCtrl @Inject() ( value.predicate, Some(e.value), e.expanded, - e.colour.map(tagSrv.parseTagColour).getOrElse(tagSrv.defaultColour) + e.colour.getOrElse(tagSrv.defaultColour) ) ) }) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala index 07835754be..b5b45a8b89 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaxonomyRenderer.scala @@ -1,10 +1,9 @@ package org.thp.thehive.controllers.v1 -import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.thehive.services.TaxonomyOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.models.Taxonomy +import org.thp.thehive.services.TaxonomyOps._ import play.api.libs.json._ import java.util.{Map => JMap} @@ -14,9 +13,8 @@ trait TaxonomyRenderer { def enabledStats: Traversal.V[Taxonomy] => Traversal[JsValue, Boolean, Converter[JsValue, Boolean]] = _.enabled.domainMap(l => JsBoolean(l)) - def taxoStatsRenderer(extraData: Set[String])(implicit - authContext: AuthContext - ): Traversal.V[Taxonomy] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { traversal => + def taxoStatsRenderer(extraData: Set[String]): + Traversal.V[Taxonomy] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { traversal => def addData[G]( name: String )(f: Traversal.V[Taxonomy] => Traversal[JsValue, G, Converter[JsValue, G]]): Traversal[JsObject, JMap[String, Any], Converter[ diff --git a/thehive/app/org/thp/thehive/models/Tag.scala b/thehive/app/org/thp/thehive/models/Tag.scala index 3ad58979a5..ee1264f61e 100644 --- a/thehive/app/org/thp/thehive/models/Tag.scala +++ b/thehive/app/org/thp/thehive/models/Tag.scala @@ -1,12 +1,9 @@ package org.thp.thehive.models -import java.util.Date - -import org.thp.scalligraph.{BuildVertexEntity, EntityId} -import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType} +import org.thp.scalligraph.BuildVertexEntity +import org.thp.scalligraph.models.{DefineIndex, IndexType} import play.api.Logger -import scala.util.Try import scala.util.matching.Regex @DefineIndex(IndexType.unique, "namespace", "predicate", "value") @@ -16,7 +13,7 @@ case class Tag( predicate: String, value: Option[String], description: Option[String], - colour: Int + colour: String ) { override def hashCode(): Int = 31 * (31 * value.## + predicate.##) + namespace.## @@ -35,15 +32,15 @@ case class Tag( object Tag { lazy val logger: Logger = Logger(getClass) - val tagColour: Regex = "(.*)#(\\p{XDigit}{6})".r + val tagColour: Regex = "(.*)(#\\p{XDigit}{6})".r val namespacePredicateValue: Regex = "([^\".:=]+)[.:]([^\".=]+)=\"?([^\"]+)\"?".r val namespacePredicate: Regex = "([^\".:=]+)[.]([^\".=]+)".r val PredicateValue: Regex = "([^\".:=]+)[=:]\"?([^\"]+)\"?".r val predicate: Regex = "([^\".:=]+)".r - def fromString(tagName: String, defaultNamespace: String, defaultColour: Int = 0): Tag = { + def fromString(tagName: String, defaultNamespace: String, defaultColour: String = "#000000"): Tag = { val (name, colour) = tagName match { - case tagColour(n, c) => n -> Try(Integer.parseUnsignedInt(c, 16)).getOrElse(defaultColour) + case tagColour(n, c) => n -> c case _ => tagName -> defaultColour } name match { diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index c3af62d20a..caa97fca8d 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -57,6 +57,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { case error => logger.warn(s"Unable to remove lock on property $name: $error") } } + // TODO remove unused commented code ? // def removeIndexLock(name: String): Try[Unit] = // db.managementTransaction { mgmt => // Try(mgmt.setConsistency(mgmt.getGraphIndex(name), ConsistencyModifier.DEFAULT)) @@ -138,6 +139,16 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { Try(traversal.unsafeHas("name", "admin").raw.property("permissions", "manageTaxonomy").iterate()) Success(()) } + .updateGraph("Remove colour property for Tags", "Tag") { traversal => + traversal.removeProperty("colour").iterate() + Success(()) + } + .removeProperty("Tag", "colour", usedOnlyByThisModel = true) + .addProperty[String]("Tag", "colour") + .updateGraph("Add property colour for Tags ", "Tag") { traversal => + traversal.raw.property("colour", "#000000").iterate() + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala index cc3165c5a3..fefca23f5f 100644 --- a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala +++ b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala @@ -81,7 +81,7 @@ object AttachmentOps { implicit class AttachmentOpsDefs(traversal: Traversal.V[Attachment]) { def getByAttachmentId(attachmentId: String): Traversal.V[Attachment] = traversal.has(_.attachmentId, attachmentId) - def visible(implicit authContext: AuthContext): Traversal.V[Attachment] = traversal // TODO + def visible: Traversal.V[Attachment] = traversal // TODO } } diff --git a/thehive/app/org/thp/thehive/services/TagSrv.scala b/thehive/app/org/thp/thehive/services/TagSrv.scala index d937be0d92..ddc6ab87da 100644 --- a/thehive/app/org/thp/thehive/services/TagSrv.scala +++ b/thehive/app/org/thp/thehive/services/TagSrv.scala @@ -29,20 +29,10 @@ class TagSrv @Inject() (appConfig: ApplicationConfig, @Named("integrity-check-ac def defaultNamespace: String = defaultNamespaceConfig.get - private val defaultColourConfig: ConfigItem[String, Int] = - appConfig.mapItem[String, Int]( - "tags.defaultColour", - "Default colour of the automatically created tags", - { - case s if s(0) == '#' => parseTagColour(s.tail) - case _ => defaultColour - } - ) - - def defaultColour: Int = defaultColourConfig.get - - // TODO Duplication in Tag.scala - def parseTagColour(c: String) = Try(Integer.parseUnsignedInt(c, 16)).getOrElse(defaultColour) + private val defaultColourConfig: ConfigItem[String, String] = + appConfig.item[String]("tags.defaultColour", "Default colour of the automatically created tags") + + def defaultColour: String = defaultColourConfig.get def parseString(tagName: String): Tag = Tag.fromString(tagName, defaultNamespace, defaultColour) diff --git a/thehive/test/org/thp/thehive/controllers/v0/ConfigCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/ConfigCtrlTest.scala index 46caf3274b..17e88bb138 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/ConfigCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/ConfigCtrlTest.scala @@ -6,6 +6,9 @@ import play.api.libs.json.{JsObject, Json} import play.api.test.{FakeRequest, PlaySpecification} class ConfigCtrlTest extends PlaySpecification with TestAppBuilder { + +// TODO leave unused code ? +// // def getList = { // val request = FakeRequest("GET", "/api/config") // .withHeaders("user" -> "admin@thehive.local") @@ -36,9 +39,10 @@ class ConfigCtrlTest extends PlaySpecification with TestAppBuilder { status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") - app[TagSrv].defaultColour must beEqualTo(0xff00) + app[TagSrv].defaultColour must beEqualTo("#00FF00") } - +// TODO leave unused tests ? +// // "get user specific configuration" in testApp { app => // val request = FakeRequest("GET", "/api/config/user/organisation") // .withHeaders("user" -> "admin@thehive.local") diff --git a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala index 917a6fb6f3..21674a24c5 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala @@ -68,9 +68,9 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { "A test taxonomy", 1, List( - OutputTag("test-taxo", "pred1", Some("entry1"), None, 0), - OutputTag("test-taxo", "pred2", Some("entry2"), None, 0), - OutputTag("test-taxo", "pred2", Some("entry21"), None, 0) + OutputTag("test-taxo", "pred1", Some("entry1"), None, "#000000"), + OutputTag("test-taxo", "pred2", Some("entry2"), None, "#000000"), + OutputTag("test-taxo", "pred2", Some("entry21"), None, "#000000") ) ) } @@ -123,7 +123,7 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { "taxonomy1", "The taxonomy 1", 1, - List(OutputTag("taxonomy1", "pred1", Some("value1"), None, 0)) + List(OutputTag("taxonomy1", "pred1", Some("value1"), None, "#000000")) ) } @@ -144,7 +144,7 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[TaxonomyCtrl].importZip(request) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") - contentAsString(result) must not contain("Failure") + contentAsString(result) must not contain "Failure" contentAsJson(result).as[JsArray].value.size must beEqualTo(2) } @@ -156,7 +156,7 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[TaxonomyCtrl].importZip(request) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") - contentAsString(result) must not contain("Failure") + contentAsString(result) must not contain "Failure" contentAsJson(result).as[JsArray].value.size must beEqualTo(2) } @@ -168,7 +168,7 @@ class TaxonomyCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[TaxonomyCtrl].importZip(request) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") - contentAsString(result) must not contain("Failure") + contentAsString(result) must not contain "Failure" contentAsJson(result).as[JsArray].value.size must beEqualTo(1) } diff --git a/thehive/test/org/thp/thehive/models/TagTest.scala b/thehive/test/org/thp/thehive/models/TagTest.scala index 29a9021c47..24a4c59a58 100644 --- a/thehive/test/org/thp/thehive/models/TagTest.scala +++ b/thehive/test/org/thp/thehive/models/TagTest.scala @@ -4,43 +4,43 @@ import play.api.test.PlaySpecification class TagTest extends PlaySpecification { val defaultNamespace: String = "_default_namespace_" - val defaultColor: Int = 0xffff00 + val defaultColour: String = "#ffff00" - def parseTag(s: String): Tag = Tag.fromString(s, defaultNamespace, defaultColor) + def parseTag(s: String): Tag = Tag.fromString(s, defaultNamespace, defaultColour) "tag" should { "be parsed from key:value" in { val tag = parseTag("Module:atest_blah_blah") - tag must beEqualTo(Tag(defaultNamespace, "Module", Some("atest_blah_blah"), None, defaultColor)) + tag must beEqualTo(Tag(defaultNamespace, "Module", Some("atest_blah_blah"), None, defaultColour)) tag.toString must beEqualTo("Module=\"atest_blah_blah\"") } "be parsed from key:value=" in { val tag = parseTag("Id:7SeUoB3IBABD+tMh2PjVJYg==") - tag must beEqualTo(Tag(defaultNamespace, "Id", Some("7SeUoB3IBABD+tMh2PjVJYg=="), None, defaultColor)) + tag must beEqualTo(Tag(defaultNamespace, "Id", Some("7SeUoB3IBABD+tMh2PjVJYg=="), None, defaultColour)) tag.toString must beEqualTo("Id=\"7SeUoB3IBABD+tMh2PjVJYg==\"") } "be parsed from key: value" in { val tag = parseTag("domain: google.com") - tag must beEqualTo(Tag(defaultNamespace, "domain", Some("google.com"), None, defaultColor)) + tag must beEqualTo(Tag(defaultNamespace, "domain", Some("google.com"), None, defaultColour)) tag.toString must beEqualTo("domain=\"google.com\"") } "be parsed from key: a.b.c.d" in { val tag = parseTag("ip: 8.8.8.8") - tag must beEqualTo(Tag(defaultNamespace, "ip", Some("8.8.8.8"), None, defaultColor)) + tag must beEqualTo(Tag(defaultNamespace, "ip", Some("8.8.8.8"), None, defaultColour)) tag.toString must beEqualTo("ip=\"8.8.8.8\"") } "be parsed with colour" in { val tag = parseTag("ip:8.8.8.8#FF00FF") - tag must beEqualTo(Tag(defaultNamespace, "ip", Some("8.8.8.8"), None, 0xFF00FF)) + tag must beEqualTo(Tag(defaultNamespace, "ip", Some("8.8.8.8"), None, "#FF00FF")) tag.toString must beEqualTo("ip=\"8.8.8.8\"") } "be parsed with hash sign and colour" in { val tag = parseTag("case:#42#FF00FF") - tag must beEqualTo(Tag(defaultNamespace, "case", Some("#42"), None, 0xFF00FF)) + tag must beEqualTo(Tag(defaultNamespace, "case", Some("#42"), None, "#FF00FF")) tag.toString must beEqualTo("case=\"#42\"") } } diff --git a/thehive/test/resources/data/Tag.json b/thehive/test/resources/data/Tag.json index 094be1895a..30908714a4 100644 --- a/thehive/test/resources/data/Tag.json +++ b/thehive/test/resources/data/Tag.json @@ -4,76 +4,76 @@ "namespace": "testNamespace", "predicate": "testPredicate", "value": "t1", - "colour": 0 + "colour": "#000000" }, { "id": "tagt2", "namespace": "testNamespace", "predicate": "testPredicate", "value": "t2", - "colour": 0 + "colour": "#000000" }, { "id": "tagt3", "namespace": "testNamespace", "predicate": "testPredicate", "value": "t3", - "colour": 0 + "colour": "#000000" }, { "id": "tagalert", "namespace": "testNamespace", "predicate": "testPredicate", "value": "alert", - "colour": 0 + "colour": "#000000" }, { "id": "tagtest", "namespace": "testNamespace", "predicate": "testPredicate", "value": "test", - "colour": 0 + "colour": "#000000" }, { "id": "tagspam", "namespace": "testNamespace", "predicate": "testPredicate", "value": "spam", - "colour": 0 + "colour": "#000000" }, { "id": "tagsrc:mail", "namespace": "testNamespace", "predicate": "testPredicate", "value": "src:mail", - "colour": 0 + "colour": "#000000" }, { "id": "tagtestDomain", "namespace": "testNamespace", "predicate": "testPredicate", "value": "testDomain", - "colour": 0 + "colour": "#000000" }, { "id": "taghello", "namespace": "testNamespace", "predicate": "testPredicate", "value": "hello", - "colour": 0 + "colour": "#000000" }, { "id": "tagworld", "namespace": "testNamespace", "predicate": "testPredicate", "value": "world", - "colour": 0 + "colour": "#000000" }, { "id": "taxonomy-tag1", "namespace": "taxonomy1", "predicate": "pred1", "value": "value1", - "colour": 0 + "colour": "#000000" } ] \ No newline at end of file From fb928e03c504393182482474a97a8f5c05e44d24 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 12 Dec 2020 17:21:55 +0100 Subject: [PATCH 062/324] #1454 Update the all the FilteringSrv instances to use the new entity names from APIs v1 --- .../app/scripts/components/alert/AlertObservableListCmp.js | 2 +- frontend/app/scripts/controllers/MainPageCtrl.js | 2 +- frontend/app/scripts/controllers/case/CaseTasksCtrl.js | 2 +- frontend/app/views/partials/case/case.list.html | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertObservableListCmp.js b/frontend/app/scripts/components/alert/AlertObservableListCmp.js index 8344ca5c71..1e7b032827 100644 --- a/frontend/app/scripts/components/alert/AlertObservableListCmp.js +++ b/frontend/app/scripts/components/alert/AlertObservableListCmp.js @@ -7,7 +7,7 @@ var self = this; self.$onInit = function() { - this.filtering = new FilteringSrv('case_artifact', 'alert.dialog.observables', { + this.filtering = new FilteringSrv('observable', 'alert.dialog.observables', { version: 'v1', defaults: { showFilters: false, diff --git a/frontend/app/scripts/controllers/MainPageCtrl.js b/frontend/app/scripts/controllers/MainPageCtrl.js index 8320212b90..e1980eb669 100644 --- a/frontend/app/scripts/controllers/MainPageCtrl.js +++ b/frontend/app/scripts/controllers/MainPageCtrl.js @@ -31,7 +31,7 @@ self.view.data = 'waitingtasks'; } - self.filtering = new FilteringSrv('case_task', $stateParams.viewId + '.list', { + self.filtering = new FilteringSrv('task', $stateParams.viewId + '.list', { version: 'v1', defaults: { showFilters: true, diff --git a/frontend/app/scripts/controllers/case/CaseTasksCtrl.js b/frontend/app/scripts/controllers/case/CaseTasksCtrl.js index ae277fea63..da9ee1f483 100755 --- a/frontend/app/scripts/controllers/case/CaseTasksCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseTasksCtrl.js @@ -19,7 +19,7 @@ $scope.collapseOptions = {}; this.$onInit = function() { - $scope.filtering = new FilteringSrv('case_task', 'task.list', { + $scope.filtering = new FilteringSrv('task', 'task.list', { version: 'v1', defaults: { showFilters: true, diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 36e3c63db6..358b7f54e5 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -76,8 +76,8 @@

    List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount}

    - - + +
    (Closed at {{currentCase.endDate | showDate}} as {{$vm.CaseResolutionStatus[currentCase.resolutionStatus]}}) From fe52ccb465c7658cb090030e28a44a684a810c29 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 29 Oct 2020 16:22:45 +0100 Subject: [PATCH 063/324] Cascade remove from Share --- .../thp/thehive/controllers/v0/CaseCtrl.scala | 2 +- .../thp/thehive/controllers/v1/CaseCtrl.scala | 2 +- .../org/thp/thehive/services/CaseSrv.scala | 8 ++++++ .../org/thp/thehive/services/ShareSrv.scala | 28 ++++++++++++++++--- .../org/thp/thehive/services/TaskSrv.scala | 15 ++++++++-- 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index f8fcd8aeed..184f97db13 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -144,7 +144,7 @@ class CaseCtrl @Inject() ( .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase) .getOrFail("Case") - _ <- caseSrv.remove(c) + _ <- caseSrv.cascadeRemove(c) } yield Results.NoContent } diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index 0440346733..d58fb5c89d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -117,7 +117,7 @@ class CaseCtrl @Inject() ( .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase) .getOrFail("Case") - _ <- caseSrv.remove(c) + _ <- caseSrv.cascadeRemove(c) } yield Results.NoContent } diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index d2e2e0c88f..dda62648f5 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -198,6 +198,14 @@ class CaseSrv @Inject() ( } yield () } + def cascadeRemove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + // We let ShareSrv handle all cascade deletions (Case, Tasks, Logs and Observables) + for { + organisation <- organisationSrv.getOrFail(authContext.organisation) + share <- shareSrv.get(`case`, organisation._id).getOrFail("Case") + } yield shareSrv.cascadeRemove(share._id) + } + def remove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { val details = Json.obj("number" -> `case`.number, "title" -> `case`.title) for { diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index e0d1ae4c7d..ed114dd0bd 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -27,7 +27,8 @@ class ShareSrv @Inject() ( auditSrv: AuditSrv, caseSrvProvider: Provider[CaseSrv], taskSrv: TaskSrv, - observableSrvProvider: Provider[ObservableSrv] + observableSrvProvider: Provider[ObservableSrv], + organisationSrv: OrganisationSrv ) extends VertexSrv[Share] { lazy val caseSrv: CaseSrv = caseSrvProvider.get lazy val observableSrv: ObservableSrv = observableSrvProvider.get @@ -90,11 +91,30 @@ class ShareSrv @Inject() ( // caseSrv.get(`case`).in[ShareCase].filter(_.in[OrganisationShare])._id.getOrFail().flatMap(remove(_)) // FIXME add organisation ? def remove(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + unshareCase(shareId).flatMap(_ => Try(get(shareId).remove())) + + def cascadeRemove(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + for { + organisation <- organisationSrv.getOrFail(authContext.organisation) + share <- getOrFail(shareId) + // Remove ShareTask, then remove task for each task + tasks <- Try(get(share).tasks.toSeq) + _ <- tasks.toTry(t => removeShareTasks(t, organisation).flatMap(_ => taskSrv.remove(t))) + // Remove observables + observables <- Try(get(share).observables.toSeq) + _ <- observables.toTry(o => removeShareObservable(o, organisation).flatMap(_ => observableSrv.remove(o))) + // Unshare then remove the case + _ <- unshareCase(shareId) + caze <- get(share).`case`.getOrFail("Case") + _ <- caseSrv.remove(caze) + } yield Try(get(shareId).remove()) + } + + def unshareCase(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { case0 <- get(shareId).`case`.getOrFail("Case") organisation <- get(shareId).organisation.getOrFail("Organisation") - _ <- auditSrv.share.unshareCase(case0, organisation) - } yield get(shareId).remove() + } yield auditSrv.share.unshareCase(case0, organisation) /** * Unshare Task for a given Organisation @@ -118,7 +138,7 @@ class ShareSrv @Inject() ( } yield shareTaskSrv.get(shareTask).remove() /** - * Unshare Task for a given Organisation + * Unshare Observable for a given Organisation * @param observable observable to unshare * @param organisation organisation the task must be unshared with * @return diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index 574d074392..c77725126f 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -21,11 +21,15 @@ import play.api.libs.json.{JsNull, JsObject, Json} import scala.util.{Failure, Success, Try} @Singleton -class TaskSrv @Inject() (caseSrvProvider: Provider[CaseSrv], auditSrv: AuditSrv)(implicit - @Named("with-thehive-schema") db: Database +class TaskSrv @Inject() ( + @Named("with-thehive-schema") implicit val db: Database, + caseSrvProvider: Provider[CaseSrv], + logSrvProvider: Provider[LogSrv], + auditSrv: AuditSrv ) extends VertexSrv[Task] { lazy val caseSrv: CaseSrv = caseSrvProvider.get + lazy val logSrv: LogSrv = logSrvProvider.get val caseTemplateTaskSrv = new EdgeSrv[CaseTemplateTask, CaseTemplate, Task] val taskUserSrv = new EdgeSrv[TaskUser, Task, User] val taskLogSrv = new EdgeSrv[TaskLog, Task, Log] @@ -64,6 +68,13 @@ class TaskSrv @Inject() (caseSrvProvider: Provider[CaseSrv], auditSrv: AuditSrv) .map(_ => get(task).remove()) } + def cascadeRemove(task: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + for { + task <- getOrFail(task._id) + logs <- Try(get(task).logs.toSeq) + _ <- logs.toTry(l => logSrv.cascadeRemove(l)) + } yield remove(task) + override def update( traversal: Traversal.V[Task], propertyUpdaters: Seq[PropertyUpdater] From 1c9a1ae773906c57a2517102bdc0f4ccd38d310f Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 29 Oct 2020 17:39:01 +0100 Subject: [PATCH 064/324] Created deletion audits before deleting entities --- .../app/org/thp/thehive/services/ObservableSrv.scala | 7 +++++++ thehive/app/org/thp/thehive/services/ShareSrv.scala | 12 ++++++++---- thehive/app/org/thp/thehive/services/TaskSrv.scala | 12 +----------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index b95f40f7aa..493c87d119 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -177,6 +177,13 @@ class ObservableSrv @Inject() ( case Some(alert) => alertSrv.removeObservable(alert, observable) } + // Same as remove but with no Audit creation + def cascadeRemove(observable: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + get(observable).alert.headOption match { + case None => Try(get(observable).remove()) + case Some(alert) => alertSrv.removeObservable(alert, observable) + } + override def update( traversal: Traversal.V[Observable], propertyUpdaters: Seq[PropertyUpdater] diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index ed114dd0bd..360b0143bf 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -97,12 +97,16 @@ class ShareSrv @Inject() ( for { organisation <- organisationSrv.getOrFail(authContext.organisation) share <- getOrFail(shareId) - // Remove ShareTask, then remove task for each task + // Create deletion Audit, remove ShareTask, then remove task. All that for each task tasks <- Try(get(share).tasks.toSeq) - _ <- tasks.toTry(t => removeShareTasks(t, organisation).flatMap(_ => taskSrv.remove(t))) - // Remove observables + _ <- tasks.toTry(t => auditSrv.task.delete(t, share) + .flatMap(_ => removeShareTasks(t, organisation) + .flatMap(_ => taskSrv.cascadeRemove(t)))) + // Create deletion Audit, remove ShareObservable, then remove observable. All that for each observable observables <- Try(get(share).observables.toSeq) - _ <- observables.toTry(o => removeShareObservable(o, organisation).flatMap(_ => observableSrv.remove(o))) + _ <- observables.toTry(o => auditSrv.observable.delete(o, share) + .flatMap(_ => removeShareObservable(o, organisation) + .flatMap(_ => observableSrv.cascadeRemove(o)))) // Unshare then remove the case _ <- unshareCase(shareId) caze <- get(share).`case`.getOrFail("Case") diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index c77725126f..6a81764d2f 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -50,17 +50,7 @@ class TaskSrv @Inject() ( def remove(task: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = get(task).caseTemplate.headOption match { - case None => - get(task) - .shares - .toIterator - .toTry { share => - auditSrv - .task - .delete(task, share) - .map(_ => get(task).remove()) - } - .map(_ => ()) + case None => Try(get(task).remove()) case Some(caseTemplate) => auditSrv .caseTemplate From d7cd42531a7c2f597b65841338a411bc3a26241c Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Fri, 30 Oct 2020 11:01:55 +0100 Subject: [PATCH 065/324] Removed deleted property from Log --- .../connector/cortex/services/ActionOperationSrv.scala | 2 +- dto/src/main/scala/org/thp/thehive/dto/v1/Log.scala | 1 - .../main/scala/org/thp/thehive/migration/th3/Conversion.scala | 3 +-- thehive/app/org/thp/thehive/controllers/v0/Conversion.scala | 3 +-- thehive/app/org/thp/thehive/controllers/v1/Conversion.scala | 1 - thehive/app/org/thp/thehive/models/Log.scala | 3 +-- .../app/org/thp/thehive/models/TheHiveSchemaDefinition.scala | 4 ++++ 7 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala index 2e2722b805..a683855c04 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala @@ -90,7 +90,7 @@ class ActionOperationSrv @Inject() ( case AddLogToTask(content, _) => for { t <- relatedTask.fold[Try[Task with Entity]](Failure(InternalError("Unable to apply action AddLogToTask without task")))(Success(_)) - _ <- logSrv.create(Log(content, new Date(), deleted = false), t) + _ <- logSrv.create(Log(content, new Date()), t) } yield updateOperation(operation) case AddArtifactToCase(_, dataType, dataMessage) => diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Log.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Log.scala index 4e4ba55519..11cad043aa 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Log.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Log.scala @@ -17,7 +17,6 @@ case class OutputLog( message: String, date: Date, attachment: Option[OutputAttachment] = None, - deleted: Boolean, owner: String, extraData: JsObject ) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index af1545e564..8a5b88bd82 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -152,12 +152,11 @@ trait Conversion { metaData <- json.validate[MetaData] message <- (json \ "message").validate[String] date <- (json \ "startDate").validate[Date] - deleted = (json \ "status").asOpt[String].contains("Deleted") attachment = (json \ "attachment") .asOpt[Attachment] .map(a => InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id))) - } yield InputLog(metaData, Log(message, date, deleted), attachment.toSeq) + } yield InputLog(metaData, Log(message, date), attachment.toSeq) } implicit val alertReads: Reads[InputAlert] = Reads[InputAlert] { json => diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index f972afd972..bb8570cd00 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -315,7 +315,7 @@ object Conversion { .withFieldComputed(_.message, _.message) .withFieldComputed(_.startDate, _._createdAt) .withFieldComputed(_.owner, _._createdBy) - .withFieldComputed(_.status, l => if (l.deleted) "Deleted" else "Ok") + .withFieldComputed(_.status, _ => "Ok") .withFieldComputed(_.attachment, _.attachments.headOption.map(_.toValue)) .transform ) @@ -326,7 +326,6 @@ object Conversion { inputLog .into[Log] .withFieldConst(_.date, new Date) - .withFieldConst(_.deleted, false) .transform } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index ac556fca70..90858715cc 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -423,7 +423,6 @@ object Conversion { inputLog .into[Log] .withFieldConst(_.date, new Date) - .withFieldConst(_.deleted, false) .transform } diff --git a/thehive/app/org/thp/thehive/models/Log.scala b/thehive/app/org/thp/thehive/models/Log.scala index 4c5c54bb41..78db90848f 100644 --- a/thehive/app/org/thp/thehive/models/Log.scala +++ b/thehive/app/org/thp/thehive/models/Log.scala @@ -9,7 +9,7 @@ import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} case class LogAttachment() @BuildVertexEntity -case class Log(message: String, date: Date, deleted: Boolean) +case class Log(message: String, date: Date) case class RichLog(log: Log with Entity, attachments: Seq[Attachment with Entity]) { def _id: EntityId = log._id @@ -19,5 +19,4 @@ case class RichLog(log: Log with Entity, attachments: Seq[Attachment with Entity def _updatedAt: Option[Date] = log._updatedAt def date: Date = log.date def message: String = log.message - def deleted: Boolean = log.deleted } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index eeab7f15fd..80092279b7 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -84,6 +84,10 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .iterate() Success(()) } + .updateGraph("Remove deleted property from logs", "Log") { traversal => + traversal.removeProperty("deleted") + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() From 6820d73cfd0edc3b7e12cb8de9850da433ef2a76 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Fri, 30 Oct 2020 14:45:24 +0100 Subject: [PATCH 066/324] Deleted edge removal when cascade deleting a Share --- .../thehive/models/TheHiveSchemaDefinition.scala | 3 ++- thehive/app/org/thp/thehive/services/ShareSrv.scala | 13 +++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 80092279b7..c9514cdb2c 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -84,7 +84,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .iterate() Success(()) } - .updateGraph("Remove deleted property from logs", "Log") { traversal => + .updateGraph("Remove deleted logs and deleted property from logs", "Log") { traversal => + traversal.unsafeHas("deleted", "true").remove() traversal.removeProperty("deleted") Success(()) } diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index 360b0143bf..f5cdcaf816 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -97,18 +97,15 @@ class ShareSrv @Inject() ( for { organisation <- organisationSrv.getOrFail(authContext.organisation) share <- getOrFail(shareId) - // Create deletion Audit, remove ShareTask, then remove task. All that for each task + // Create deletion Audit, then remove task. All that for each task tasks <- Try(get(share).tasks.toSeq) _ <- tasks.toTry(t => auditSrv.task.delete(t, share) - .flatMap(_ => removeShareTasks(t, organisation) - .flatMap(_ => taskSrv.cascadeRemove(t)))) - // Create deletion Audit, remove ShareObservable, then remove observable. All that for each observable + .flatMap(_ => taskSrv.cascadeRemove(t))) + // Create deletion Audit, then remove observable. All that for each observable observables <- Try(get(share).observables.toSeq) _ <- observables.toTry(o => auditSrv.observable.delete(o, share) - .flatMap(_ => removeShareObservable(o, organisation) - .flatMap(_ => observableSrv.cascadeRemove(o)))) - // Unshare then remove the case - _ <- unshareCase(shareId) + .flatMap(_ => observableSrv.cascadeRemove(o))) + // Remove the case caze <- get(share).`case`.getOrFail("Case") _ <- caseSrv.remove(caze) } yield Try(get(shareId).remove()) From a226f5af6872c3bcbe687f4adf0f46c0868cac45 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 2 Nov 2020 18:14:04 +0100 Subject: [PATCH 067/324] WIP Unshared tasks and obs instead of deleting --- .../thp/thehive/controllers/v0/Conversion.scala | 2 +- .../models/TheHiveSchemaDefinition.scala | 1 + .../app/org/thp/thehive/services/CaseSrv.scala | 3 +-- .../app/org/thp/thehive/services/ShareSrv.scala | 17 +++++++++++++---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index bb8570cd00..69a5a230db 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -315,7 +315,7 @@ object Conversion { .withFieldComputed(_.message, _.message) .withFieldComputed(_.startDate, _._createdAt) .withFieldComputed(_.owner, _._createdBy) - .withFieldComputed(_.status, _ => "Ok") + .withFieldConst(_.status, "Ok") .withFieldComputed(_.attachment, _.attachments.headOption.map(_.toValue)) .transform ) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index c9514cdb2c..405664239d 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -89,6 +89,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { traversal.removeProperty("deleted") Success(()) } + .removeProperty(model = "Log", propertyName = "deleted", usedOnlyByThisModel = true) val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index dda62648f5..c3ff769d16 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -201,8 +201,7 @@ class CaseSrv @Inject() ( def cascadeRemove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { // We let ShareSrv handle all cascade deletions (Case, Tasks, Logs and Observables) for { - organisation <- organisationSrv.getOrFail(authContext.organisation) - share <- shareSrv.get(`case`, organisation._id).getOrFail("Case") + share <- shareSrv.get(`case`, authContext.organisation).getOrFail("Case") } yield shareSrv.cascadeRemove(share._id) } diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index f5cdcaf816..b74ef83bab 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -97,14 +97,23 @@ class ShareSrv @Inject() ( for { organisation <- organisationSrv.getOrFail(authContext.organisation) share <- getOrFail(shareId) + // Create deletion Audit, then remove task. All that for each task - tasks <- Try(get(share).tasks.toSeq) - _ <- tasks.toTry(t => auditSrv.task.delete(t, share) + tasks <- Try(get(share).tasks) + tasksToUnshare <- Try(tasks.filter(_.inE[ShareTask].count.is(P.gt(1))).toSeq) + tasksToDelete <- Try(tasks.filter(_.inE[ShareTask].count.is(P.eq(1))).toSeq) + _ <- tasksToUnshare.toTry(t => removeShareTasks(t, organisation)) + _ <- tasksToDelete.toTry(t => auditSrv.task.delete(t, share) .flatMap(_ => taskSrv.cascadeRemove(t))) + // Create deletion Audit, then remove observable. All that for each observable - observables <- Try(get(share).observables.toSeq) - _ <- observables.toTry(o => auditSrv.observable.delete(o, share) + obs <- Try(get(share).observables) + obsToUnshare <- Try(obs.filter(_.inE[ShareObservable].count.is(P.gt(1))).toSeq) + obsToDelete <- Try(obs.filter(_.inE[ShareObservable].count.is(P.eq(1))).toSeq) + _ <- obsToUnshare.toTry(o => removeShareObservable(o, organisation)) + _ <- obsToDelete.toTry(o => auditSrv.observable.delete(o, share) .flatMap(_ => observableSrv.cascadeRemove(o))) + // Remove the case caze <- get(share).`case`.getOrFail("Case") _ <- caseSrv.remove(caze) From 502d937e3cb4bede316c7af5340847ae6b0addfd Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 2 Nov 2020 18:17:46 +0100 Subject: [PATCH 068/324] WIP Removed deleted status from TheHive3 migration --- .../org/thp/thehive/migration/th3/Input.scala | 54 +++++-------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala index c6ce3faca7..78ae6eec3e 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala @@ -9,8 +9,8 @@ import akka.stream.scaladsl.Source import akka.util.ByteString import com.google.inject.Guice import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.searches.queries.RangeQuery import com.sksamuel.elastic4s.searches.queries.term.TermsQuery +import com.sksamuel.elastic4s.searches.queries.{Query, RangeQuery} import javax.inject.{Inject, Singleton} import net.codingwell.scalaguice.ScalaModule import org.thp.thehive.migration @@ -23,7 +23,7 @@ import play.api.{Configuration, Logger} import scala.collection.immutable import scala.concurrent.{ExecutionContext, Future} -import scala.reflect.{classTag, ClassTag} +import scala.reflect.{ClassTag, classTag} import scala.util.{Failure, Success, Try} object Input { @@ -223,44 +223,18 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe )._2 override def listCaseTaskLogs(filter: Filter): Source[Try[(String, InputLog)], NotUsed] = - dbFind(Some("all"), Nil)(indexName => - search(indexName).query( - bool( - Seq( - termQuery("relations", "case_task_log"), - hasParentQuery( - "case_task", - hasParentQuery("case", bool(caseFilter(filter), Nil, Nil), score = false), - score = false - ) - ), - Nil, - Nil - ) - ) - )._1 - .readWithParent[InputLog](json => Try((json \ "_parent").as[String])) + listCaseTaskLogs(bool(caseFilter(filter), Nil, Nil)) override def countCaseTaskLogs(filter: Filter): Future[Long] = - dbFind(Some("0-0"), Nil)(indexName => - search(indexName) - .query( - bool( - Seq( - termQuery("relations", "case_task_log"), - hasParentQuery( - "case_task", - hasParentQuery("case", bool(caseFilter(filter), Nil, Nil), score = false), - score = false - ) - ), - Nil, - Nil - ) - ) - )._2 + countCaseTaskLogs(bool(caseFilter(filter), Nil, Nil)) override def listCaseTaskLogs(caseId: String): Source[Try[(String, InputLog)], NotUsed] = + listCaseTaskLogs(idsQuery(caseId)) + + override def countCaseTaskLogs(caseId: String): Future[Long] = + countCaseTaskLogs(idsQuery(caseId)) + + private def listCaseTaskLogs(query: Query): Source[Try[(String, InputLog)], NotUsed] = dbFind(Some("all"), Nil)(indexName => search(indexName).query( bool( @@ -268,18 +242,18 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe termQuery("relations", "case_task_log"), hasParentQuery( "case_task", - hasParentQuery("case", idsQuery(caseId), score = false), + hasParentQuery("case", query, score = false), score = false ) ), Nil, - Nil + Seq(termQuery("status", "deleted")) ) ) )._1 .readWithParent[InputLog](json => Try((json \ "_parent").as[String])) - override def countCaseTaskLogs(caseId: String): Future[Long] = + private def countCaseTaskLogs(query: Query): Future[Long] = dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( @@ -288,7 +262,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe termQuery("relations", "case_task_log"), hasParentQuery( "case_task", - hasParentQuery("case", idsQuery(caseId), score = false), + hasParentQuery("case", query, score = false), score = false ) ), From ef65f65370c197649c5ba1ee2b13dbd26fdbfc32 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Fri, 27 Nov 2020 21:33:34 +0100 Subject: [PATCH 069/324] WIP cascade remove test --- .../thehive/controllers/v0/ShareCtrl.scala | 4 +- .../org/thp/thehive/services/AlertSrv.scala | 3 +- .../thp/thehive/services/AttachmentSrv.scala | 6 +- .../app/org/thp/thehive/services/LogSrv.scala | 5 +- .../thp/thehive/services/ObservableSrv.scala | 19 ++- .../org/thp/thehive/services/ShareSrv.scala | 40 ++---- .../org/thp/thehive/services/TaskSrv.scala | 9 +- .../org/thp/thehive/DatabaseBuilder.scala | 2 + .../thp/thehive/services/CaseSrvTest.scala | 120 +++++++++++++++++- thehive/test/resources/data/Attachment.json | 20 +++ thehive/test/resources/data/Case.json | 24 ++++ thehive/test/resources/data/Log.json | 6 + .../test/resources/data/LogAttachment.json | 3 + thehive/test/resources/data/Observable.json | 21 +++ .../resources/data/ObservableAttachment.json | 4 +- .../resources/data/OrganisationShare.json | 3 + thehive/test/resources/data/Share.json | 5 +- thehive/test/resources/data/ShareCase.json | 5 +- .../test/resources/data/ShareObservable.json | 6 +- thehive/test/resources/data/ShareTask.json | 7 +- thehive/test/resources/data/Task.json | 27 ++++ thehive/test/resources/data/TaskLog.json | 4 +- 22 files changed, 291 insertions(+), 52 deletions(-) create mode 100644 thehive/test/resources/data/LogAttachment.json diff --git a/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala index 4d3c6f890c..c9a9c5ad8d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala @@ -115,7 +115,7 @@ class ShareCtrl @Inject() ( organisations.toTry { organisationName => organisationSrv .getOrFail(EntityIdOrName(organisationName)) - .flatMap(shareSrv.removeShareTasks(task, _)) + .flatMap(shareSrv.unshareTask(task, _)) } } .map(_ => Results.NoContent) @@ -133,7 +133,7 @@ class ShareCtrl @Inject() ( organisations.toTry { organisationName => organisationSrv .getOrFail(EntityIdOrName(organisationName)) - .flatMap(shareSrv.removeShareObservable(observable, _)) + .flatMap(shareSrv.unshareObservable(observable, _)) } } .map(_ => Results.NoContent) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 49c91f328d..f4d187a833 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -137,8 +137,7 @@ class AlertSrv @Inject() ( def removeObservable(alert: Alert with Entity, observable: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = observableSrv .get(observable) - .inE[AlertObservable] - .filter(_.outV.hasId(alert._id)) + .filter(_.inE[AlertObservable].outV.hasId(alert._id)) .getOrFail("Observable") .flatMap { alertObservable => alertObservableSrv.get(alertObservable).remove() diff --git a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala index cc3165c5a3..98abc01423 100644 --- a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala +++ b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala @@ -71,9 +71,11 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage def exists(attachment: Attachment with Entity): Boolean = storageSrv.exists("attachment", attachment.attachmentId) - def cascadeRemove(attachment: Attachment with Entity)(implicit graph: Graph): Try[Unit] = - // TODO handle Storage data removal + def cascadeRemove(attachment: Attachment with Entity)(implicit graph: Graph): Try[Unit] = { + // TODO storageSrv delete data ? + // TODO check attachmentId not used by other Attachment vertices Try(get(attachment).remove()) + } } diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 9e0aa4f5b5..9cbbb0293e 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -58,9 +58,8 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv, taskSr for { _ <- get(log).attachments.toIterator.toTry(attachmentSrv.cascadeRemove(_)) task <- get(log).task.getOrFail("Task") - _ = get(log).remove() - _ <- auditSrv.log.delete(log, Some(task)) - } yield () + _ <- auditSrv.log.delete(log, Some(task)) + } yield get(log).remove() override def update( traversal: Traversal.V[Log], diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 493c87d119..dae2df195a 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -174,15 +174,22 @@ class ObservableSrv @Inject() ( .map(_ => get(observable).remove()) } .map(_ => ()) - case Some(alert) => alertSrv.removeObservable(alert, observable) + case Some(alert) => for { + _ <- Try(get(observable).remove()) + } yield auditSrv.observableInAlert.delete(observable, Some(alert)) } // Same as remove but with no Audit creation - def cascadeRemove(observable: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - get(observable).alert.headOption match { - case None => Try(get(observable).remove()) - case Some(alert) => alertSrv.removeObservable(alert, observable) - } + def cascadeRemove(observable: Observable with Entity, share: Share with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + val alert = get(observable).alert.headOption + if (alert.isDefined) { auditSrv.observableInAlert.delete(observable, alert) } + for { + attachments <- Try(get(observable).attachments.toSeq) + _ <- attachments.toTry(attachmentSrv.cascadeRemove(_)) + _ <- auditSrv.observable.delete(observable, share) + // TODO handle Jobs ? + } yield Try(get(observable).remove()) + } override def update( traversal: Traversal.V[Observable], diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index b74ef83bab..7d0b54bef2 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -98,25 +98,23 @@ class ShareSrv @Inject() ( organisation <- organisationSrv.getOrFail(authContext.organisation) share <- getOrFail(shareId) - // Create deletion Audit, then remove task. All that for each task + + tasks <- Try(get(share).tasks) - tasksToUnshare <- Try(tasks.filter(_.inE[ShareTask].count.is(P.gt(1))).toSeq) + tasksToUnshare <- Try(tasks.clone().filter(_.inE[ShareTask].count.is(P.gt(1))).toSeq) tasksToDelete <- Try(tasks.filter(_.inE[ShareTask].count.is(P.eq(1))).toSeq) - _ <- tasksToUnshare.toTry(t => removeShareTasks(t, organisation)) - _ <- tasksToDelete.toTry(t => auditSrv.task.delete(t, share) - .flatMap(_ => taskSrv.cascadeRemove(t))) + _ <- tasksToUnshare.toTry(unshareTask(_, organisation)) + _ <- tasksToDelete.toTry(taskSrv.cascadeRemove(_, share)) - // Create deletion Audit, then remove observable. All that for each observable obs <- Try(get(share).observables) - obsToUnshare <- Try(obs.filter(_.inE[ShareObservable].count.is(P.gt(1))).toSeq) + obsToUnshare <- Try(obs.clone().filter(_.inE[ShareObservable].count.is(P.gt(1))).toSeq) obsToDelete <- Try(obs.filter(_.inE[ShareObservable].count.is(P.eq(1))).toSeq) - _ <- obsToUnshare.toTry(o => removeShareObservable(o, organisation)) - _ <- obsToDelete.toTry(o => auditSrv.observable.delete(o, share) - .flatMap(_ => observableSrv.cascadeRemove(o))) + _ <- obsToUnshare.toTry(unshareObservable(_, organisation)) + _ <- obsToDelete.toTry(observableSrv.cascadeRemove(_, share)) // Remove the case caze <- get(share).`case`.getOrFail("Case") - _ <- caseSrv.remove(caze) + _ <- if (share.owner) caseSrv.remove(caze) else unshareCase(shareId) } yield Try(get(shareId).remove()) } @@ -124,15 +122,11 @@ class ShareSrv @Inject() ( for { case0 <- get(shareId).`case`.getOrFail("Case") organisation <- get(shareId).organisation.getOrFail("Organisation") - } yield auditSrv.share.unshareCase(case0, organisation) + shareCase <- get(shareId).`case`.inE[ShareCase].getOrFail("Case") + _ <- auditSrv.share.unshareCase(case0, organisation) + } yield shareCaseSrv.get(shareCase).remove() - /** - * Unshare Task for a given Organisation - * @param task task to unshare - * @param organisation organisation the task must be unshared with - * @return - */ - def removeShareTasks( + def unshareTask( task: Task with Entity, organisation: Organisation with Entity )(implicit graph: Graph, authContext: AuthContext): Try[Unit] = @@ -147,13 +141,7 @@ class ShareSrv @Inject() ( _ <- auditSrv.share.unshareTask(task, case0, organisation) } yield shareTaskSrv.get(shareTask).remove() - /** - * Unshare Observable for a given Organisation - * @param observable observable to unshare - * @param organisation organisation the task must be unshared with - * @return - */ - def removeShareObservable( + def unshareObservable( observable: Observable with Entity, organisation: Organisation with Entity )(implicit graph: Graph, authContext: AuthContext): Try[Unit] = diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index 6a81764d2f..e09bc0ecab 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -58,11 +58,12 @@ class TaskSrv @Inject() ( .map(_ => get(task).remove()) } - def cascadeRemove(task: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def cascadeRemove(task: Task with Entity, share: Share with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { - task <- getOrFail(task._id) - logs <- Try(get(task).logs.toSeq) - _ <- logs.toTry(l => logSrv.cascadeRemove(l)) + task <- getOrFail(task._id) + logs <- Try(get(task).logs.toSeq) + _ <- logs.toTry(logSrv.cascadeRemove(_)) + _ <- auditSrv.task.delete(task, share) } yield remove(task) override def update( diff --git a/thehive/test/org/thp/thehive/DatabaseBuilder.scala b/thehive/test/org/thp/thehive/DatabaseBuilder.scala index 51767a822f..37dabd31b7 100644 --- a/thehive/test/org/thp/thehive/DatabaseBuilder.scala +++ b/thehive/test/org/thp/thehive/DatabaseBuilder.scala @@ -107,6 +107,8 @@ class DatabaseBuilder @Inject() ( createEdge(taskSrv.taskUserSrv, taskSrv, userSrv, FieldsParser[TaskUser], idMap) createEdge(taskSrv.taskLogSrv, taskSrv, logSrv, FieldsParser[TaskLog], idMap) + createEdge(logSrv.logAttachmentSrv, logSrv, attachmentSrv, FieldsParser[LogAttachment], idMap) + createEdge(caseSrv.caseUserSrv, caseSrv, userSrv, FieldsParser[CaseUser], idMap) createEdge(caseSrv.mergedFromSrv, caseSrv, caseSrv, FieldsParser[MergedFrom], idMap) createEdge(caseSrv.caseCaseTemplateSrv, caseSrv, caseTemplateSrv, FieldsParser[CaseCaseTemplate], idMap) diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index 258c2c38cc..fc3cb11f76 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -11,6 +11,9 @@ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{CreateError, EntityName} import org.thp.thehive.TestAppBuilder import org.thp.thehive.models._ +import org.thp.thehive.services.TaskOps._ +import org.thp.thehive.services.LogOps._ +import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ import play.api.libs.json.Json @@ -207,7 +210,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "get correct next case number" in testApp { app => app[Database].roTransaction { implicit graph => - app[CaseSrv].nextCaseNumber shouldEqual 4 + app[CaseSrv].nextCaseNumber shouldEqual 6 } } @@ -419,5 +422,120 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[Database].roTransaction(implicit graph => app[CaseSrv].get(EntityName("1")).linkedCases must not(beEmpty)) } } + + "cascade remove, case not shared" in testApp { app => + app[Database].roTransaction { implicit graph => + implicit val authContext: AuthContext = DummyUserSrv(organisation = "cert", permissions = Set(Permissions.manageCase)).authContext + + def caze = app[CaseSrv].startTraversal.has(_.number, 5).getOrFail("Case") + caze must beSuccessfulTry + + val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-simple")._id.getOrFail("Task").get + def taskDelete = app[TaskSrv].get(taskId).getOrFail("Task") + def logs = app[TaskSrv].get(taskId).logs.toSeq.size + def logsAttach = app[TaskSrv].get(taskId).logs.attachments.toSeq.size + taskDelete must beSuccessfulTry + logs must beEqualTo(1) + logsAttach must beEqualTo(1) + + val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-simple")._id.getOrFail("Observable").get + def obsDelete = app[ObservableSrv].get(obsId).getOrFail("Observable") + def obsAttach = app[ObservableSrv].get(obsId).attachments.toSeq.size + obsDelete must beSuccessfulTry + obsAttach must beEqualTo(1) + + app[CaseSrv].cascadeRemove(caze.get) must beASuccessfulTry + + taskDelete must beAFailedTry // TODO it's a Success, why ??? + logs must beEqualTo(0) + logsAttach must beEqualTo(0) + obsDelete must beAFailedTry + obsAttach must beEqualTo(0) + caze must beAFailedTry + } + } + + "cascade remove, shared" in testApp { app => + app[Database].roTransaction { implicit graph => + // Check users of soc have access to case 4 + implicit val authContext: AuthContext = DummyUserSrv(organisation = "soc", permissions = Set(Permissions.manageCase)).authContext + + def caze = app[CaseSrv].startTraversal.has(_.number, 4).getOrFail("Case") + caze must beSuccessfulTry + + val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-unshare")._id.getOrFail("Task").get + def taskUnshare = app[TaskSrv].get(taskId).getOrFail("Task") + taskUnshare must beSuccessfulTry + + val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-unshare")._id.getOrFail("Observable").get + def obsUnshare = app[ObservableSrv].get(obsId).getOrFail("Observable") + obsUnshare must beSuccessfulTry + } + + app[Database].roTransaction { implicit graph => + // Check entities & cascade remove the case + implicit val authContext: AuthContext = DummyUserSrv(organisation = "cert", permissions = Set(Permissions.manageCase)).authContext + + def caze = app[CaseSrv].startTraversal.has(_.number, 4).getOrFail("Case") + caze must beSuccessfulTry + + val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-delete")._id.getOrFail("Task").get + val taskId2 = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-unshare")._id.getOrFail("Task").get + def taskDelete = app[TaskSrv].get(taskId).getOrFail("Task") + def taskUnshare = app[TaskSrv].get(taskId2).getOrFail("Task") + def logs = app[TaskSrv].get(taskId).logs.toSeq.size + def logs2 = app[TaskSrv].get(taskId2).logs.toSeq.size + def taskAttach = app[TaskSrv].get(taskId).logs.attachments.toSeq.size + def taskAttach2 = app[TaskSrv].get(taskId2).logs.attachments.toSeq.size + taskDelete must beSuccessfulTry + logs must beEqualTo(1) + taskAttach must beEqualTo(1) + taskUnshare must beSuccessfulTry + logs2 must beEqualTo(0) + taskAttach2 must beEqualTo(0) + + val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-delete")._id.getOrFail("Observable").get + val obsId2 = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-unshare")._id.getOrFail("Observable").get + def obsDelete = app[ObservableSrv].get(obsId).getOrFail("Observable") + def obsUnshare = app[ObservableSrv].get(obsId2).getOrFail("Observable") + def obsAttach = app[ObservableSrv].get(obsId).attachments.toSeq.size + def obsAttach2 = app[ObservableSrv].get(obsId2).attachments.toSeq.size + obsDelete must beSuccessfulTry + obsAttach must beEqualTo(1) + obsUnshare must beSuccessfulTry + obsAttach2 must beEqualTo(0) + + app[CaseSrv].cascadeRemove(caze.get) must beASuccessfulTry + + caze must beASuccessfulTry + taskDelete must beAFailedTry + taskUnshare must beASuccessfulTry + logs must beEqualTo(0) + taskAttach must beEqualTo(0) + logs2 must beEqualTo(0) + taskAttach2 must beEqualTo(0) + obsDelete must beAFailedTry + obsUnshare must beASuccessfulTry + obsAttach must beEqualTo(0) + obsAttach2 must beEqualTo(0) + } + + app[Database].roTransaction { implicit graph => + // Users of soc should still have access to case + implicit val authContext: AuthContext = DummyUserSrv(organisation = "soc", permissions = Set(Permissions.manageCase)).authContext + + def caze = app[CaseSrv].startTraversal.has(_.number, 4).getOrFail("Case") + caze must beSuccessfulTry + + val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-unshare")._id.getOrFail("Task").get + def taskUnshare = app[TaskSrv].get(taskId).getOrFail("Task") + taskUnshare must beSuccessfulTry + + val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-unshare")._id.getOrFail("Observable").get + def obsUnshare = app[ObservableSrv].get(obsId).getOrFail("Observable") + obsUnshare must beSuccessfulTry + } + } + } } diff --git a/thehive/test/resources/data/Attachment.json b/thehive/test/resources/data/Attachment.json index 6c28345222..af587ae943 100644 --- a/thehive/test/resources/data/Attachment.json +++ b/thehive/test/resources/data/Attachment.json @@ -10,5 +10,25 @@ "905138a85e85e74344e90d25dba7299e" ], "attachmentId": "a4bf1f6be616bf6a0de2ff6264de43a64bb768d38c783ec2bc74b5d4dcf5f889" + }, + { + "id": "cascade-remove-attachment1", + "name": "cascade1.txt", + "size": 13, + "contentType": "text/plain", + "hashes": [ + "905138a85e85e74344e90d25dba7299e" + ], + "attachmentId": "b4bf1f6be616bf6a0de2ff6264de43a64bb768d38c783ec2bc74b5d4dcf5f889" + }, + { + "id": "cascade-remove-attachment2", + "name": "cascade2.txt", + "size": 13, + "contentType": "text/plain", + "hashes": [ + "a4bf1f6be616bf6a0de2ff6264de43a64bb768d38c783ec2bc74b5d4dcf5f889" + ], + "attachmentId": "c4bf1f6be616bf6a0de2ff6264de43a64bb768d38c783ec2bc74b5d4dcf5f889" } ] \ No newline at end of file diff --git a/thehive/test/resources/data/Case.json b/thehive/test/resources/data/Case.json index e7e28530e8..3e31a40e48 100644 --- a/thehive/test/resources/data/Case.json +++ b/thehive/test/resources/data/Case.json @@ -34,5 +34,29 @@ "tlp": 2, "pap": 2, "status": "Open" + }, + { + "id": "case4", + "number": 4, + "title": "case#4", + "description": "description of case cascade remove", + "severity": 2, + "startDate": 1531667370000, + "flag": false, + "tlp": 2, + "pap": 2, + "status": "Open" + }, + { + "id": "case5", + "number": 5, + "title": "case#5", + "description": "description of case cascade remove", + "severity": 2, + "startDate": 1531667370000, + "flag": false, + "tlp": 2, + "pap": 2, + "status": "Open" } ] \ No newline at end of file diff --git a/thehive/test/resources/data/Log.json b/thehive/test/resources/data/Log.json index 594a72f724..ce1f5ed27f 100644 --- a/thehive/test/resources/data/Log.json +++ b/thehive/test/resources/data/Log.json @@ -4,5 +4,11 @@ "message": "log for action test", "date": 1566303875000, "deleted": false + }, + { + "id": "log-cascade-remove", + "message": "log cascade remove", + "date": 1566303875000, + "deleted": false } ] \ No newline at end of file diff --git a/thehive/test/resources/data/LogAttachment.json b/thehive/test/resources/data/LogAttachment.json new file mode 100644 index 0000000000..70f9e4e039 --- /dev/null +++ b/thehive/test/resources/data/LogAttachment.json @@ -0,0 +1,3 @@ +[ + {"from": "log-cascade-remove", "to": "cascade-remove-attachment2"} +] \ No newline at end of file diff --git a/thehive/test/resources/data/Observable.json b/thehive/test/resources/data/Observable.json index 88e79ee5cc..3e333545dc 100644 --- a/thehive/test/resources/data/Observable.json +++ b/thehive/test/resources/data/Observable.json @@ -33,5 +33,26 @@ "tlp": 1, "ioc": true, "sighted": true + }, + { + "id": "obs-cascade-remove-simple", + "message": "obs-cascade-remove-simple", + "tlp": 1, + "ioc": true, + "sighted": true + }, + { + "id": "obs-cascade-remove-unshare", + "message": "obs-cascade-remove-unshare", + "tlp": 1, + "ioc": true, + "sighted": true + }, + { + "id": "obs-cascade-remove-delete", + "message": "obs-cascade-remove-delete", + "tlp": 1, + "ioc": true, + "sighted": true } ] diff --git a/thehive/test/resources/data/ObservableAttachment.json b/thehive/test/resources/data/ObservableAttachment.json index e96a0e3fa3..8aa6ad9726 100644 --- a/thehive/test/resources/data/ObservableAttachment.json +++ b/thehive/test/resources/data/ObservableAttachment.json @@ -1,3 +1,5 @@ [ - {"from": "helloworld", "to": "attachment-hello"} + {"from": "helloworld", "to": "attachment-hello"}, + {"from": "obs-cascade-remove-simple", "to": "cascade-remove-attachment1"}, + {"from": "obs-cascade-remove-delete", "to": "cascade-remove-attachment1"} ] \ No newline at end of file diff --git a/thehive/test/resources/data/OrganisationShare.json b/thehive/test/resources/data/OrganisationShare.json index 36fb4352f3..af6330927e 100644 --- a/thehive/test/resources/data/OrganisationShare.json +++ b/thehive/test/resources/data/OrganisationShare.json @@ -1,6 +1,9 @@ [ {"from": "cert", "to": "case1-cert"}, {"from": "cert", "to": "case2-cert"}, + {"from": "cert", "to": "cascade-remove-simple"}, + {"from": "cert", "to": "share-cascade-remove"}, + {"from": "soc", "to": "share-cascade-remove2"}, {"from": "soc", "to": "case2-soc"}, {"from": "soc", "to": "case3-soc"} ] \ No newline at end of file diff --git a/thehive/test/resources/data/Share.json b/thehive/test/resources/data/Share.json index f10a76f4e2..1e8f5df498 100644 --- a/thehive/test/resources/data/Share.json +++ b/thehive/test/resources/data/Share.json @@ -2,5 +2,8 @@ {"id": "case1-cert", "owner": true}, {"id": "case2-cert", "owner": true}, {"id": "case2-soc", "owner": false}, - {"id": "case3-soc", "owner": true} + {"id": "case3-soc", "owner": true}, + {"id": "cascade-remove-simple", "owner": true}, + {"id": "share-cascade-remove", "owner": true}, + {"id": "share-cascade-remove2", "owner": true} ] \ No newline at end of file diff --git a/thehive/test/resources/data/ShareCase.json b/thehive/test/resources/data/ShareCase.json index f3e1b02f94..4979fbf35d 100644 --- a/thehive/test/resources/data/ShareCase.json +++ b/thehive/test/resources/data/ShareCase.json @@ -2,5 +2,8 @@ {"from": "case1-cert", "to": "case1"}, {"from": "case2-cert", "to": "case2"}, {"from": "case2-soc", "to": "case2"}, - {"from": "case3-soc", "to": "case3"} + {"from": "case3-soc", "to": "case3"}, + {"from": "cascade-remove-simple", "to": "case5"}, + {"from": "share-cascade-remove", "to": "case4"}, + {"from": "share-cascade-remove2", "to": "case4"} ] \ No newline at end of file diff --git a/thehive/test/resources/data/ShareObservable.json b/thehive/test/resources/data/ShareObservable.json index 136f6ec8bf..bd33b9726e 100644 --- a/thehive/test/resources/data/ShareObservable.json +++ b/thehive/test/resources/data/ShareObservable.json @@ -1,5 +1,9 @@ [ {"from": "case1-cert", "to": "h.fr"}, {"from": "case2-cert", "to": "helloworld"}, - {"from": "case2-soc", "to": "helloworld"} + {"from": "case2-soc", "to": "helloworld"}, + {"from": "cascade-remove-simple", "to": "obs-cascade-remove-simple"}, + {"from": "share-cascade-remove", "to": "obs-cascade-remove-delete"}, + {"from": "share-cascade-remove", "to": "obs-cascade-remove-unshare"}, + {"from": "share-cascade-remove2", "to": "obs-cascade-remove-unshare"} ] \ No newline at end of file diff --git a/thehive/test/resources/data/ShareTask.json b/thehive/test/resources/data/ShareTask.json index 075623d6ef..99092b810a 100644 --- a/thehive/test/resources/data/ShareTask.json +++ b/thehive/test/resources/data/ShareTask.json @@ -3,5 +3,10 @@ {"from": "case1-cert", "to": "task2"}, {"from": "case2-soc", "to": "task3"}, {"from": "case2-cert", "to": "task4"}, - {"from": "case3-soc", "to": "task5"} + {"from": "case3-soc", "to": "task5"}, + {"from": "cascade-remove-simple", "to": "task-cascade-remove-simple"}, + {"from": "share-cascade-remove", "to": "task-cascade-remove-delete"}, + {"from": "share-cascade-remove", "to": "task-cascade-remove-unshare"}, + {"from": "share-cascade-remove2", "to": "task-cascade-remove-unshare"} + ] \ No newline at end of file diff --git a/thehive/test/resources/data/Task.json b/thehive/test/resources/data/Task.json index 06db6dc2b3..48fa35f58f 100644 --- a/thehive/test/resources/data/Task.json +++ b/thehive/test/resources/data/Task.json @@ -43,5 +43,32 @@ "status": "Waiting", "flag": true, "order": 0 + }, + { + "id": "task-cascade-remove-simple", + "title": "task-cascade-remove-simple", + "group": "group1", + "description": "description task simple", + "status": "Waiting", + "flag": true, + "order": 0 + }, + { + "id": "task-cascade-remove-unshare", + "title": "task-cascade-remove-unshare", + "group": "group1", + "description": "description task unshare", + "status": "Waiting", + "flag": true, + "order": 0 + }, + { + "id": "task-cascade-remove-delete", + "title": "task-cascade-remove-delete", + "group": "group1", + "description": "description task delete", + "status": "Waiting", + "flag": true, + "order": 0 } ] \ No newline at end of file diff --git a/thehive/test/resources/data/TaskLog.json b/thehive/test/resources/data/TaskLog.json index be493e0362..5848ae884b 100644 --- a/thehive/test/resources/data/TaskLog.json +++ b/thehive/test/resources/data/TaskLog.json @@ -1,3 +1,5 @@ [ - {"from": "task4", "to": "log1"} + {"from": "task4", "to": "log1"}, + {"from": "task-cascade-remove-simple", "to": "log-cascade-remove"}, + {"from": "task-cascade-remove-delete", "to": "log-cascade-remove"} ] \ No newline at end of file From 7403497b57210d23d3a5f3c74832c8b910282ceb Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 30 Nov 2020 10:58:43 +0100 Subject: [PATCH 070/324] Fixed cascade remove tests --- thehive/app/org/thp/thehive/services/ShareSrv.scala | 10 ++++------ .../test/org/thp/thehive/services/CaseSrvTest.scala | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index 7d0b54bef2..fdee200713 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -98,8 +98,6 @@ class ShareSrv @Inject() ( organisation <- organisationSrv.getOrFail(authContext.organisation) share <- getOrFail(shareId) - - tasks <- Try(get(share).tasks) tasksToUnshare <- Try(tasks.clone().filter(_.inE[ShareTask].count.is(P.gt(1))).toSeq) tasksToDelete <- Try(tasks.filter(_.inE[ShareTask].count.is(P.eq(1))).toSeq) @@ -112,10 +110,10 @@ class ShareSrv @Inject() ( _ <- obsToUnshare.toTry(unshareObservable(_, organisation)) _ <- obsToDelete.toTry(observableSrv.cascadeRemove(_, share)) - // Remove the case - caze <- get(share).`case`.getOrFail("Case") - _ <- if (share.owner) caseSrv.remove(caze) else unshareCase(shareId) - } yield Try(get(shareId).remove()) + cazeDelete <- Try(get(share).`case`.filter(_.inE[ShareCase].count.is(P.eq(1))).toSeq) + _ <- if (cazeDelete.nonEmpty) caseSrv.remove(cazeDelete.head) + else unshareCase(shareId) + } yield get(shareId).remove() } def unshareCase(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index fc3cb11f76..4364aa6916 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -424,7 +424,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { } "cascade remove, case not shared" in testApp { app => - app[Database].roTransaction { implicit graph => + app[Database].transaction { implicit graph => implicit val authContext: AuthContext = DummyUserSrv(organisation = "cert", permissions = Set(Permissions.manageCase)).authContext def caze = app[CaseSrv].startTraversal.has(_.number, 5).getOrFail("Case") @@ -446,7 +446,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[CaseSrv].cascadeRemove(caze.get) must beASuccessfulTry - taskDelete must beAFailedTry // TODO it's a Success, why ??? + taskDelete must beAFailedTry logs must beEqualTo(0) logsAttach must beEqualTo(0) obsDelete must beAFailedTry @@ -472,7 +472,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { obsUnshare must beSuccessfulTry } - app[Database].roTransaction { implicit graph => + app[Database].transaction { implicit graph => // Check entities & cascade remove the case implicit val authContext: AuthContext = DummyUserSrv(organisation = "cert", permissions = Set(Permissions.manageCase)).authContext From 3f444d19410e1f8715cb4c6fa41e4a16d06ce76e Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 30 Nov 2020 11:42:21 +0100 Subject: [PATCH 071/324] Fixed tests for cascade remove --- .../test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala | 2 +- thehive/test/resources/data/Case.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala index f677c55520..c341d9f6a2 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala @@ -327,7 +327,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { status(result) must_=== 200 val resultCase = contentAsJson(result) - (resultCase \ "count").asOpt[Int] must beSome(2) + (resultCase \ "count").asOpt[Int] must beSome(4) (resultCase \ "testNamespace:testPredicate=\"t1\"" \ "count").asOpt[Int] must beSome(2) (resultCase \ "testNamespace:testPredicate=\"t2\"" \ "count").asOpt[Int] must beSome(1) (resultCase \ "testNamespace:testPredicate=\"t3\"" \ "count").asOpt[Int] must beSome(1) diff --git a/thehive/test/resources/data/Case.json b/thehive/test/resources/data/Case.json index 3e31a40e48..9e43ee55ef 100644 --- a/thehive/test/resources/data/Case.json +++ b/thehive/test/resources/data/Case.json @@ -40,7 +40,7 @@ "number": 4, "title": "case#4", "description": "description of case cascade remove", - "severity": 2, + "severity": 3, "startDate": 1531667370000, "flag": false, "tlp": 2, @@ -52,7 +52,7 @@ "number": 5, "title": "case#5", "description": "description of case cascade remove", - "severity": 2, + "severity": 3, "startDate": 1531667370000, "flag": false, "tlp": 2, From 5ca5dc305644ffee4fde3a5f9adf2d1fd8a77116 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 30 Nov 2020 16:23:49 +0100 Subject: [PATCH 072/324] Cascade remove for (shared) attachments --- ScalliGraph | 2 +- .../app/org/thp/thehive/services/AttachmentSrv.scala | 11 +++++++---- thehive/app/org/thp/thehive/services/LogSrv.scala | 5 +++-- .../app/org/thp/thehive/services/ObservableSrv.scala | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index f6a4d2165c..c22e0468a2 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit f6a4d2165c26826c5b28db1a513ade15dfb060f2 +Subproject commit c22e0468a25ed93cc079383b166ab508f6ec8853 diff --git a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala index 98abc01423..7364cbcc58 100644 --- a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala +++ b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala @@ -71,10 +71,13 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage def exists(attachment: Attachment with Entity): Boolean = storageSrv.exists("attachment", attachment.attachmentId) - def cascadeRemove(attachment: Attachment with Entity)(implicit graph: Graph): Try[Unit] = { - // TODO storageSrv delete data ? - // TODO check attachmentId not used by other Attachment vertices - Try(get(attachment).remove()) + def cascadeRemove(attachment: Attachment with Entity)(implicit graph: Graph): Unit = { + val attachments = startTraversal.has(_.attachmentId, attachment.attachmentId).limit(2).getCount + if (attachments == 1) { + storageSrv.delete("attachment", attachment.attachmentId) + } + + get(attachment).remove() } } diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 9cbbb0293e..013353111d 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -54,12 +54,13 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv, taskSr _ <- auditSrv.log.update(log, task, Json.obj("attachment" -> attachment.name)) } yield attachment - def cascadeRemove(log: Log with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def cascadeRemove(log: Log with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + get(log).attachments.toIterator.foreach(attachmentSrv.cascadeRemove(_)) for { - _ <- get(log).attachments.toIterator.toTry(attachmentSrv.cascadeRemove(_)) task <- get(log).task.getOrFail("Task") _ <- auditSrv.log.delete(log, Some(task)) } yield get(log).remove() + } override def update( traversal: Traversal.V[Log], diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index dae2df195a..7cdb1960f6 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -185,7 +185,7 @@ class ObservableSrv @Inject() ( if (alert.isDefined) { auditSrv.observableInAlert.delete(observable, alert) } for { attachments <- Try(get(observable).attachments.toSeq) - _ <- attachments.toTry(attachmentSrv.cascadeRemove(_)) + _ <- attachments.toTry(a => Try(attachmentSrv.cascadeRemove(a))) _ <- auditSrv.observable.delete(observable, share) // TODO handle Jobs ? } yield Try(get(observable).remove()) From bb6e919151a36e270d438ff3320aa98059834561 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 1 Dec 2020 10:06:58 +0100 Subject: [PATCH 073/324] Fixed bug share / unshare --- thehive/app/org/thp/thehive/services/ShareSrv.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index fdee200713..0feb6f431a 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -120,7 +120,11 @@ class ShareSrv @Inject() ( for { case0 <- get(shareId).`case`.getOrFail("Case") organisation <- get(shareId).organisation.getOrFail("Organisation") - shareCase <- get(shareId).`case`.inE[ShareCase].getOrFail("Case") + shareCase <- get(shareId) + .`case` + .inE[ShareCase] + .filter(_.outV.v[Share].byOrganisation(organisation._id)) + .getOrFail("Case") _ <- auditSrv.share.unshareCase(case0, organisation) } yield shareCaseSrv.get(shareCase).remove() From dd5f913c5103992454f961d73d1faff0b493323b Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 16 Dec 2020 15:13:28 +0100 Subject: [PATCH 074/324] Added scalafmt align rule --- .scalafmt.conf | 1 + ScalliGraph | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index 97286da352..4a6c535e14 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -7,6 +7,7 @@ maxColumn = 150 align.openParenCallSite = false align.openParenDefnSite = false +align.tokens.add = [{code = "must"}] newlines.alwaysBeforeTopLevelStatements = false rewrite.rules = [ RedundantBraces diff --git a/ScalliGraph b/ScalliGraph index c22e0468a2..7f2f8c4fd0 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit c22e0468a25ed93cc079383b166ab508f6ec8853 +Subproject commit 7f2f8c4fd04f254f76545090350fb80917582d00 From 9bdfa7486291d24490a6b77308430097ba2f5814 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 17 Dec 2020 15:51:41 +0100 Subject: [PATCH 075/324] WIP Renamed functions & moved deletion responsibilty --- .../org/thp/thehive/migration/th3/Input.scala | 4 +- .../thp/thehive/controllers/v0/CaseCtrl.scala | 2 +- .../thp/thehive/controllers/v0/LogCtrl.scala | 2 +- .../controllers/v0/TheHiveQueryExecutor.scala | 2 +- .../thp/thehive/controllers/v1/CaseCtrl.scala | 2 +- .../thp/thehive/controllers/v1/LogCtrl.scala | 2 +- .../app/org/thp/thehive/models/Share.scala | 1 + .../models/TheHiveSchemaDefinition.scala | 2 +- .../thp/thehive/services/AttachmentSrv.scala | 7 +- .../org/thp/thehive/services/CaseSrv.scala | 20 ++- .../app/org/thp/thehive/services/LogSrv.scala | 7 +- .../thp/thehive/services/ObservableSrv.scala | 44 +++--- .../org/thp/thehive/services/ShareSrv.scala | 39 ++---- .../org/thp/thehive/services/TaskSrv.scala | 38 +++-- .../thp/thehive/services/CaseSrvTest.scala | 132 ++++++++++-------- thehive/test/resources/data/Share.json | 2 +- 16 files changed, 164 insertions(+), 142 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala index 78ae6eec3e..6beff002a1 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala @@ -23,7 +23,7 @@ import play.api.{Configuration, Logger} import scala.collection.immutable import scala.concurrent.{ExecutionContext, Future} -import scala.reflect.{ClassTag, classTag} +import scala.reflect.{classTag, ClassTag} import scala.util.{Failure, Success, Try} object Input { @@ -267,7 +267,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe ) ), Nil, - Nil + Seq(termQuery("status", "deleted")) ) ) )._2 diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index 184f97db13..c7d67e0154 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -144,7 +144,7 @@ class CaseCtrl @Inject() ( .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase) .getOrFail("Case") - _ <- caseSrv.cascadeRemove(c) + _ <- caseSrv.shareDelete(c) } yield Results.NoContent } diff --git a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala index ca1bc469a4..a75cc2d5d5 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala @@ -69,7 +69,7 @@ class LogCtrl @Inject() ( .authTransaction(db) { implicit req => implicit graph => for { log <- logSrv.get(EntityIdOrName(logId)).can(Permissions.manageTask).getOrFail("Log") - _ <- logSrv.cascadeRemove(log) + _ <- logSrv.delete(log) } yield Results.NoContent } } diff --git a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala index b55bbac9f1..d5d881bf86 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala @@ -77,7 +77,7 @@ class TheHiveQueryExecutor @Inject() ( case tpe if SubType(tpe, ru.typeOf[Traversal.V[Log]]) => ru.typeOf[Traversal.V[Observable]] } override val customFilterQuery: FilterQuery = FilterQuery(db, publicProperties) { (tpe, globalParser) => - FieldsParser.debug("parentChildFilter") { + FieldsParser("parentChildFilter") { case (_, FObjOne("_parent", ParentIdFilter(_, parentId))) if parentTypes.isDefinedAt(tpe) => Good(new ParentIdInputFilter(parentId)) case (path, FObjOne("_parent", ParentQueryFilter(_, parentFilterField))) if parentTypes.isDefinedAt(tpe) => diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index d58fb5c89d..4c03f6a20f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -117,7 +117,7 @@ class CaseCtrl @Inject() ( .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase) .getOrFail("Case") - _ <- caseSrv.cascadeRemove(c) + _ <- caseSrv.shareDelete(c) } yield Results.NoContent } diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala index fcd7e2be74..47751459a5 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala @@ -84,7 +84,7 @@ class LogCtrl @Inject() ( .authTransaction(db) { implicit req => implicit graph => for { log <- logSrv.get(EntityIdOrName(logId)).can(Permissions.manageTask).getOrFail("Log") - _ <- logSrv.cascadeRemove(log) + _ <- logSrv.delete(log) } yield Results.NoContent } } diff --git a/thehive/app/org/thp/thehive/models/Share.scala b/thehive/app/org/thp/thehive/models/Share.scala index 0b5fc13646..ee9d909d71 100644 --- a/thehive/app/org/thp/thehive/models/Share.scala +++ b/thehive/app/org/thp/thehive/models/Share.scala @@ -29,6 +29,7 @@ case class RichShare(share: Share with Entity, caseId: EntityId, organisationNam def owner: Boolean = share.owner } +// TODO what to do with this unused code ? //object RichShare { // def apply(`case`: Case with Entity, organisation: Organisation with Entity, profile: Profile with Entity): RichShare = // RichShare(`case`._id, organisation.name, profile.permissions) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 405664239d..9340e92d24 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -85,7 +85,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { Success(()) } .updateGraph("Remove deleted logs and deleted property from logs", "Log") { traversal => - traversal.unsafeHas("deleted", "true").remove() + traversal.clone().unsafeHas("deleted", "true").remove() traversal.removeProperty("deleted") Success(()) } diff --git a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala index 7364cbcc58..59d784e69f 100644 --- a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala +++ b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala @@ -71,13 +71,12 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage def exists(attachment: Attachment with Entity): Boolean = storageSrv.exists("attachment", attachment.attachmentId) - def cascadeRemove(attachment: Attachment with Entity)(implicit graph: Graph): Unit = { + override def delete(attachment: Attachment with Entity)(implicit graph: Graph): Try[Unit] = { val attachments = startTraversal.has(_.attachmentId, attachment.attachmentId).limit(2).getCount - if (attachments == 1) { + if (attachments == 1) storageSrv.delete("attachment", attachment.attachmentId) - } - get(attachment).remove() + Try(get(attachment).remove()) } } diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index c3ff769d16..b4f0740d56 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -1,8 +1,8 @@ package org.thp.thehive.services import java.util.{Map => JMap} - import akka.actor.ActorRef + import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.{Order, P} import org.apache.tinkerpop.gremlin.structure.{Graph, Vertex} @@ -11,6 +11,7 @@ import org.thp.scalligraph.controllers.FPathElem import org.thp.scalligraph.models._ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ +import org.thp.scalligraph.traversal.Converter.Identity import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} import org.thp.scalligraph.{CreateError, EntityIdOrName, EntityName, RichOptionTry, RichSeq} @@ -198,11 +199,17 @@ class CaseSrv @Inject() ( } yield () } - def cascadeRemove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { - // We let ShareSrv handle all cascade deletions (Case, Tasks, Logs and Observables) + def shareDelete(caze: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + val details = Json.obj("number" -> caze.number, "title" -> caze.title) + for { - share <- shareSrv.get(`case`, authContext.organisation).getOrFail("Case") - } yield shareSrv.cascadeRemove(share._id) + share <- shareSrv.get(caze, authContext.organisation).getOrFail("Share") + org <- organisationSrv.get(authContext.organisation).getOrFail("Organisation") + _ <- auditSrv.`case`.delete(caze, org, Some(details)) + _ <- + if (share.owner) remove(caze) + else shareSrv.unshareCase(share._id) + } yield shareSrv.delete(share._id) } def remove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { @@ -500,6 +507,9 @@ object CaseOps { .toSeq } + def isShared: Traversal[Boolean, Boolean, Identity[Boolean]] = + traversal.choose(_.inE[ShareCase].count.is(P.gt(1)), true, false) + def richCase(implicit authContext: AuthContext): Traversal[RichCase, JMap[String, Any], Converter[RichCase, JMap[String, Any]]] = traversal .project( diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 013353111d..b1b979d1f7 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -54,13 +54,10 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv, taskSr _ <- auditSrv.log.update(log, task, Json.obj("attachment" -> attachment.name)) } yield attachment - def cascadeRemove(log: Log with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { - get(log).attachments.toIterator.foreach(attachmentSrv.cascadeRemove(_)) + override def delete(log: Log with Entity)(implicit graph: Graph): Try[Unit] = for { - task <- get(log).task.getOrFail("Task") - _ <- auditSrv.log.delete(log, Some(task)) + _ <- get(log).attachments.toSeq.toTry(attachmentSrv.delete(_)) } yield get(log).remove() - } override def update( traversal: Traversal.V[Log], diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 7cdb1960f6..7dd65d396b 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -1,15 +1,15 @@ package org.thp.thehive.services import java.util.{Map => JMap} - import javax.inject.{Inject, Named, Provider, Singleton} -import org.apache.tinkerpop.gremlin.process.traversal.{P => JP} +import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.{Graph, Vertex} import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ +import org.thp.scalligraph.traversal.Converter.Identity import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} import org.thp.scalligraph.utils.Hash @@ -28,12 +28,15 @@ class ObservableSrv @Inject() ( dataSrv: DataSrv, attachmentSrv: AttachmentSrv, tagSrv: TagSrv, - caseSrvProvider: Provider[CaseSrv], + organisationSrv: OrganisationSrv, auditSrv: AuditSrv, + shareSrvProvider: Provider[ShareSrv], + caseSrvProvider: Provider[CaseSrv], alertSrvProvider: Provider[AlertSrv] )(implicit @Named("with-thehive-schema") db: Database ) extends VertexSrv[Observable] { + lazy val shareSrv: ShareSrv = shareSrvProvider.get lazy val caseSrv: CaseSrv = caseSrvProvider.get lazy val alertSrv: AlertSrv = alertSrvProvider.get val observableKeyValueSrv = new EdgeSrv[ObservableKeyValue, Observable, KeyValue] @@ -174,22 +177,26 @@ class ObservableSrv @Inject() ( .map(_ => get(observable).remove()) } .map(_ => ()) - case Some(alert) => for { - _ <- Try(get(observable).remove()) - } yield auditSrv.observableInAlert.delete(observable, Some(alert)) + case Some(alert) => + for { + _ <- Try(get(observable).remove()) + } yield auditSrv.observableInAlert.delete(observable, Some(alert)) } // Same as remove but with no Audit creation - def cascadeRemove(observable: Observable with Entity, share: Share with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { - val alert = get(observable).alert.headOption - if (alert.isDefined) { auditSrv.observableInAlert.delete(observable, alert) } - for { - attachments <- Try(get(observable).attachments.toSeq) - _ <- attachments.toTry(a => Try(attachmentSrv.cascadeRemove(a))) - _ <- auditSrv.observable.delete(observable, share) - // TODO handle Jobs ? - } yield Try(get(observable).remove()) - } + def delete(obs: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + if (get(obs).isShared.head) + for { + orga <- organisationSrv.getOrFail(authContext.organisation) + } yield shareSrv.unshareObservable(obs, orga) + else { + val alert = get(obs).alert.headOption + val attachments = get(obs).attachments.toSeq + if (alert.isDefined) auditSrv.observableInAlert.delete(obs, alert) + for { + _ <- attachments.toTry(attachmentSrv.delete(_)) + } yield get(obs).remove() + } override def update( traversal: Traversal.V[Observable], @@ -254,6 +261,9 @@ object ObservableOps { def origin: Traversal.V[Organisation] = shares.has(_.owner, true).organisation + def isShared: Traversal[Boolean, Boolean, Identity[Boolean]] = + traversal.choose(_.inE[ShareObservable].count.is(P.gt(1)), true, false) + def richObservable: Traversal[RichObservable, JMap[String, Any], Converter[RichObservable, JMap[String, Any]]] = traversal .project( @@ -364,7 +374,7 @@ object ObservableOps { _.out[ObservableAttachment] .in[ObservableAttachment] // FIXME this doesn't work. Link must be done with attachmentId ) - .where(JP.without(originLabel.name)) + .where(P.without(originLabel.name)) .dedup .v[Observable] } diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index 0feb6f431a..b6bec199bb 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -93,39 +93,26 @@ class ShareSrv @Inject() ( def remove(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = unshareCase(shareId).flatMap(_ => Try(get(shareId).remove())) - def cascadeRemove(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + def delete(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { - organisation <- organisationSrv.getOrFail(authContext.organisation) - share <- getOrFail(shareId) - - tasks <- Try(get(share).tasks) - tasksToUnshare <- Try(tasks.clone().filter(_.inE[ShareTask].count.is(P.gt(1))).toSeq) - tasksToDelete <- Try(tasks.filter(_.inE[ShareTask].count.is(P.eq(1))).toSeq) - _ <- tasksToUnshare.toTry(unshareTask(_, organisation)) - _ <- tasksToDelete.toTry(taskSrv.cascadeRemove(_, share)) - - obs <- Try(get(share).observables) - obsToUnshare <- Try(obs.clone().filter(_.inE[ShareObservable].count.is(P.gt(1))).toSeq) - obsToDelete <- Try(obs.filter(_.inE[ShareObservable].count.is(P.eq(1))).toSeq) - _ <- obsToUnshare.toTry(unshareObservable(_, organisation)) - _ <- obsToDelete.toTry(observableSrv.cascadeRemove(_, share)) - - cazeDelete <- Try(get(share).`case`.filter(_.inE[ShareCase].count.is(P.eq(1))).toSeq) - _ <- if (cazeDelete.nonEmpty) caseSrv.remove(cazeDelete.head) - else unshareCase(shareId) + share <- getOrFail(shareId) + tasks = get(share).tasks.toSeq + obs = get(share).observables.toSeq + _ <- tasks.toTry(taskSrv.delete(_)) + _ <- obs.toTry(observableSrv.delete(_)) } yield get(shareId).remove() - } def unshareCase(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { case0 <- get(shareId).`case`.getOrFail("Case") organisation <- get(shareId).organisation.getOrFail("Organisation") - shareCase <- get(shareId) - .`case` - .inE[ShareCase] - .filter(_.outV.v[Share].byOrganisation(organisation._id)) - .getOrFail("Case") - _ <- auditSrv.share.unshareCase(case0, organisation) + shareCase <- + get(shareId) + .`case` + .inE[ShareCase] + .filter(_.outV.v[Share].byOrganisation(organisation._id)) + .getOrFail("Case") + _ <- auditSrv.share.unshareCase(case0, organisation) } yield shareCaseSrv.get(shareCase).remove() def unshareTask( diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index e09bc0ecab..48d1996415 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -1,8 +1,9 @@ package org.thp.thehive.services +import org.apache.tinkerpop.gremlin.process.traversal.P + import java.util import java.util.Date - import javax.inject.{Inject, Named, Provider, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.EntityIdOrName @@ -10,6 +11,7 @@ import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ +import org.thp.scalligraph.traversal.Converter.Identity import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.models.{TaskStatus, _} @@ -23,16 +25,19 @@ import scala.util.{Failure, Success, Try} @Singleton class TaskSrv @Inject() ( @Named("with-thehive-schema") implicit val db: Database, + shareSrvProvider: Provider[ShareSrv], caseSrvProvider: Provider[CaseSrv], logSrvProvider: Provider[LogSrv], + organisationSrv: OrganisationSrv, auditSrv: AuditSrv ) extends VertexSrv[Task] { - lazy val caseSrv: CaseSrv = caseSrvProvider.get - lazy val logSrv: LogSrv = logSrvProvider.get - val caseTemplateTaskSrv = new EdgeSrv[CaseTemplateTask, CaseTemplate, Task] - val taskUserSrv = new EdgeSrv[TaskUser, Task, User] - val taskLogSrv = new EdgeSrv[TaskLog, Task, Log] + lazy val shareSrv: ShareSrv = shareSrvProvider.get + lazy val caseSrv: CaseSrv = caseSrvProvider.get + lazy val logSrv: LogSrv = logSrvProvider.get + val caseTemplateTaskSrv = new EdgeSrv[CaseTemplateTask, CaseTemplate, Task] + val taskUserSrv = new EdgeSrv[TaskUser, Task, User] + val taskLogSrv = new EdgeSrv[TaskLog, Task, Log] def create(e: Task, owner: Option[User with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichTask] = for { @@ -50,7 +55,7 @@ class TaskSrv @Inject() ( def remove(task: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = get(task).caseTemplate.headOption match { - case None => Try(get(task).remove()) + case None => Try(get(task).remove()) case Some(caseTemplate) => auditSrv .caseTemplate @@ -58,13 +63,15 @@ class TaskSrv @Inject() ( .map(_ => get(task).remove()) } - def cascadeRemove(task: Task with Entity, share: Share with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - for { - task <- getOrFail(task._id) - logs <- Try(get(task).logs.toSeq) - _ <- logs.toTry(logSrv.cascadeRemove(_)) - _ <- auditSrv.task.delete(task, share) - } yield remove(task) + def delete(t: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + if (get(t).isShared.head) + for { + orga <- organisationSrv.getOrFail(authContext.organisation) + } yield shareSrv.unshareTask(t, orga) + else + for { + _ <- get(t).logs.toSeq.toTry(logSrv.delete(_)) + } yield remove(t) override def update( traversal: Traversal.V[Task], @@ -161,6 +168,9 @@ object TaskOps { .users(Permissions.manageTask) .dedup + def isShared: Traversal[Boolean, Boolean, Identity[Boolean]] = + traversal.choose(_.inE[ShareTask].count.is(P.gt(1)), true, false) + def richTask: Traversal[RichTask, util.Map[String, Any], Converter[RichTask, util.Map[String, Any]]] = traversal .project( diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index 4364aa6916..5531a6557c 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -132,9 +132,9 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { richCase.tlp must_=== 2 richCase.pap must_=== 2 richCase.status must_=== CaseStatus.Open - richCase.summary must beNone + richCase.summary must beNone richCase.impactStatus must beNone - richCase.assignee must beSome("socuser@thehive.local") + richCase.assignee must beSome("socuser@thehive.local") CustomField("boolean1", "boolean1", "boolean custom field", CustomFieldType.boolean, mandatory = false, options = Nil) richCase.customFields.map(f => (f.name, f.typeName, f.value)) must contain( allOf[(String, String, Option[Any])]( @@ -185,7 +185,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[Database].transaction { implicit graph => app[CaseSrv].getOrFail(EntityName("3")) must beSuccessfulTry.which { `case`: Case with Entity => app[CaseSrv].setOrCreateCustomField(`case`, EntityName("boolean1"), Some(true), None) must beSuccessfulTry - app[CaseSrv].getCustomField(`case`, EntityName("boolean1")).flatMap(_.value) must beSome.which(_ == true) + app[CaseSrv].getCustomField(`case`, EntityName("boolean1")).flatMap(_.value) must beSome.which(_ == true) } } } @@ -194,7 +194,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[Database].transaction { implicit graph => app[CaseSrv].getOrFail(EntityName("3")) must beSuccessfulTry.which { `case`: Case with Entity => app[CaseSrv].setOrCreateCustomField(`case`, EntityName("boolean1"), Some(false), None) must beSuccessfulTry - app[CaseSrv].getCustomField(`case`, EntityName("boolean1")).flatMap(_.value) must beSome.which(_ == false) + app[CaseSrv].getCustomField(`case`, EntityName("boolean1")).flatMap(_.value) must beSome.which(_ == false) } } } @@ -358,7 +358,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { ) .get - app[CaseSrv].get(c7._id).resolutionStatus.exists must beFalse + app[CaseSrv].get(c7._id).resolutionStatus.exists must beFalse app[Database].tryTransaction(implicit graph => app[CaseSrv].setResolutionStatus(c7.`case`, "Duplicated")) must beSuccessfulTry app[Database].roTransaction(implicit graph => app[CaseSrv].get(c7._id).resolutionStatus.exists must beTrue) app[Database].tryTransaction(implicit graph => app[CaseSrv].unsetResolutionStatus(c7.`case`)) must beSuccessfulTry @@ -430,21 +430,21 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { def caze = app[CaseSrv].startTraversal.has(_.number, 5).getOrFail("Case") caze must beSuccessfulTry - val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-simple")._id.getOrFail("Task").get - def taskDelete = app[TaskSrv].get(taskId).getOrFail("Task") - def logs = app[TaskSrv].get(taskId).logs.toSeq.size - def logsAttach = app[TaskSrv].get(taskId).logs.attachments.toSeq.size - taskDelete must beSuccessfulTry - logs must beEqualTo(1) - logsAttach must beEqualTo(1) - - val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-simple")._id.getOrFail("Observable").get - def obsDelete = app[ObservableSrv].get(obsId).getOrFail("Observable") - def obsAttach = app[ObservableSrv].get(obsId).attachments.toSeq.size + def taskTraversal = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-simple") + def taskDelete = taskTraversal.getOrFail("Task") + def logs = taskTraversal.logs.toSeq.size + def logsAttach = taskTraversal.logs.attachments.toSeq.size + taskDelete must beSuccessfulTry + logs must beEqualTo(1) + logsAttach must beEqualTo(1) + + def obsTraversal = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-simple") + def obsDelete = obsTraversal.getOrFail("Observable") + def obsAttach = obsTraversal.attachments.toSeq.size obsDelete must beSuccessfulTry obsAttach must beEqualTo(1) - app[CaseSrv].cascadeRemove(caze.get) must beASuccessfulTry + app[CaseSrv].shareDelete(caze.get) must beASuccessfulTry taskDelete must beAFailedTry logs must beEqualTo(0) @@ -460,16 +460,20 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { // Check users of soc have access to case 4 implicit val authContext: AuthContext = DummyUserSrv(organisation = "soc", permissions = Set(Permissions.manageCase)).authContext - def caze = app[CaseSrv].startTraversal.has(_.number, 4).getOrFail("Case") - caze must beSuccessfulTry - - val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-unshare")._id.getOrFail("Task").get - def taskUnshare = app[TaskSrv].get(taskId).getOrFail("Task") - taskUnshare must beSuccessfulTry - - val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-unshare")._id.getOrFail("Observable").get - def obsUnshare = app[ObservableSrv].get(obsId).getOrFail("Observable") - obsUnshare must beSuccessfulTry + app[CaseSrv] + .startTraversal + .has(_.number, 4) + .getOrFail("Case") must beSuccessfulTry + + app[TaskSrv] + .startTraversal + .has(_.title, "task-cascade-remove-unshare") + .getOrFail("Task") must beSuccessfulTry + + app[ObservableSrv] + .startTraversal + .has(_.message, "obs-cascade-remove-unshare") + .getOrFail("Observable") must beSuccessfulTry } app[Database].transaction { implicit graph => @@ -479,43 +483,43 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { def caze = app[CaseSrv].startTraversal.has(_.number, 4).getOrFail("Case") caze must beSuccessfulTry - val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-delete")._id.getOrFail("Task").get - val taskId2 = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-unshare")._id.getOrFail("Task").get - def taskDelete = app[TaskSrv].get(taskId).getOrFail("Task") - def taskUnshare = app[TaskSrv].get(taskId2).getOrFail("Task") - def logs = app[TaskSrv].get(taskId).logs.toSeq.size - def logs2 = app[TaskSrv].get(taskId2).logs.toSeq.size - def taskAttach = app[TaskSrv].get(taskId).logs.attachments.toSeq.size - def taskAttach2 = app[TaskSrv].get(taskId2).logs.attachments.toSeq.size - taskDelete must beSuccessfulTry + def taskTraversal = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-delete") + def taskTraversal2 = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-unshare") + def taskDelete = taskTraversal.getOrFail("Task") + def taskUnshare = taskTraversal2.getOrFail("Task") + def logs = taskTraversal.logs.toSeq.size + def logs2 = taskTraversal2.logs.toSeq.size + def taskAttach = taskTraversal.logs.attachments.toSeq.size + def taskAttach2 = taskTraversal2.logs.attachments.toSeq.size + taskDelete must beSuccessfulTry logs must beEqualTo(1) taskAttach must beEqualTo(1) - taskUnshare must beSuccessfulTry - logs2 must beEqualTo(0) + taskUnshare must beSuccessfulTry + logs2 must beEqualTo(0) taskAttach2 must beEqualTo(0) - val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-delete")._id.getOrFail("Observable").get - val obsId2 = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-unshare")._id.getOrFail("Observable").get - def obsDelete = app[ObservableSrv].get(obsId).getOrFail("Observable") - def obsUnshare = app[ObservableSrv].get(obsId2).getOrFail("Observable") - def obsAttach = app[ObservableSrv].get(obsId).attachments.toSeq.size - def obsAttach2 = app[ObservableSrv].get(obsId2).attachments.toSeq.size - obsDelete must beSuccessfulTry + def obsTraversal = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-delete") + def obsTraversal2 = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-unshare") + def obsDelete = obsTraversal.getOrFail("Observable") + def obsUnshare = obsTraversal2.getOrFail("Observable") + def obsAttach = obsTraversal.attachments.toSeq.size + def obsAttach2 = obsTraversal2.attachments.toSeq.size + obsDelete must beSuccessfulTry obsAttach must beEqualTo(1) - obsUnshare must beSuccessfulTry + obsUnshare must beSuccessfulTry obsAttach2 must beEqualTo(0) - app[CaseSrv].cascadeRemove(caze.get) must beASuccessfulTry + app[CaseSrv].shareDelete(caze.get) must beASuccessfulTry - caze must beASuccessfulTry - taskDelete must beAFailedTry - taskUnshare must beASuccessfulTry + caze must beASuccessfulTry + taskDelete must beAFailedTry + taskUnshare must beASuccessfulTry logs must beEqualTo(0) taskAttach must beEqualTo(0) logs2 must beEqualTo(0) taskAttach2 must beEqualTo(0) - obsDelete must beAFailedTry - obsUnshare must beASuccessfulTry + obsDelete must beAFailedTry + obsUnshare must beASuccessfulTry obsAttach must beEqualTo(0) obsAttach2 must beEqualTo(0) } @@ -524,16 +528,20 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { // Users of soc should still have access to case implicit val authContext: AuthContext = DummyUserSrv(organisation = "soc", permissions = Set(Permissions.manageCase)).authContext - def caze = app[CaseSrv].startTraversal.has(_.number, 4).getOrFail("Case") - caze must beSuccessfulTry - - val taskId = app[TaskSrv].startTraversal.has(_.title, "task-cascade-remove-unshare")._id.getOrFail("Task").get - def taskUnshare = app[TaskSrv].get(taskId).getOrFail("Task") - taskUnshare must beSuccessfulTry - - val obsId = app[ObservableSrv].startTraversal.has(_.message, "obs-cascade-remove-unshare")._id.getOrFail("Observable").get - def obsUnshare = app[ObservableSrv].get(obsId).getOrFail("Observable") - obsUnshare must beSuccessfulTry + app[CaseSrv] + .startTraversal + .has(_.number, 4) + .getOrFail("Case") must beSuccessfulTry + + app[TaskSrv] + .startTraversal + .has(_.title, "task-cascade-remove-unshare") + .getOrFail("Task") must beSuccessfulTry + + app[ObservableSrv] + .startTraversal + .has(_.message, "obs-cascade-remove-unshare") + .getOrFail("Observable") must beSuccessfulTry } } diff --git a/thehive/test/resources/data/Share.json b/thehive/test/resources/data/Share.json index 1e8f5df498..1025f74bc9 100644 --- a/thehive/test/resources/data/Share.json +++ b/thehive/test/resources/data/Share.json @@ -4,6 +4,6 @@ {"id": "case2-soc", "owner": false}, {"id": "case3-soc", "owner": true}, {"id": "cascade-remove-simple", "owner": true}, - {"id": "share-cascade-remove", "owner": true}, + {"id": "share-cascade-remove", "owner": false}, {"id": "share-cascade-remove2", "owner": true} ] \ No newline at end of file From 14b1e3065185741e32e19484dc5693726ed1f341 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 17 Dec 2020 17:43:24 +0100 Subject: [PATCH 076/324] Unit tests ok --- ScalliGraph | 2 +- thehive/app/org/thp/thehive/services/CaseSrv.scala | 12 +++++------- .../app/org/thp/thehive/services/ObservableSrv.scala | 2 +- thehive/app/org/thp/thehive/services/ShareSrv.scala | 2 +- thehive/app/org/thp/thehive/services/TaskSrv.scala | 2 +- .../test/org/thp/thehive/services/CaseSrvTest.scala | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 7f2f8c4fd0..ddbc847ef3 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 7f2f8c4fd04f254f76545090350fb80917582d00 +Subproject commit ddbc847ef30f2507e1287d894ad2191d873a0a87 diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index b4f0740d56..e6d286f58f 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -207,20 +207,18 @@ class CaseSrv @Inject() ( org <- organisationSrv.get(authContext.organisation).getOrFail("Organisation") _ <- auditSrv.`case`.delete(caze, org, Some(details)) _ <- - if (share.owner) remove(caze) + if (share.owner) delete(caze) else shareSrv.unshareCase(share._id) - } yield shareSrv.delete(share._id) + _ <- shareSrv.delete(share._id) + } yield () } - def remove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + private[services] def delete(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { val details = Json.obj("number" -> `case`.number, "title" -> `case`.title) for { organisation <- organisationSrv.getOrFail(authContext.organisation) _ <- auditSrv.`case`.delete(`case`, organisation, Some(details)) - } yield { - get(`case`).share.remove() - get(`case`).remove() - } + } yield get(`case`).remove() } override def getByName(name: String)(implicit graph: Graph): Traversal.V[Case] = diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 7dd65d396b..5b24adbb7f 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -184,7 +184,7 @@ class ObservableSrv @Inject() ( } // Same as remove but with no Audit creation - def delete(obs: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + private[services] def delete(obs: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = if (get(obs).isShared.head) for { orga <- organisationSrv.getOrFail(authContext.organisation) diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index b6bec199bb..e1d8262d00 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -93,7 +93,7 @@ class ShareSrv @Inject() ( def remove(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = unshareCase(shareId).flatMap(_ => Try(get(shareId).remove())) - def delete(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + private[services] def delete(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { share <- getOrFail(shareId) tasks = get(share).tasks.toSeq diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index 48d1996415..e6c618756a 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -63,7 +63,7 @@ class TaskSrv @Inject() ( .map(_ => get(task).remove()) } - def delete(t: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + private[services] def delete(t: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = if (get(t).isShared.head) for { orga <- organisationSrv.getOrFail(authContext.organisation) diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index 5531a6557c..82ab0826c3 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -313,7 +313,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { ) .get - app[Database].tryTransaction(implicit graph => app[CaseSrv].remove(c1.`case`)) must beSuccessfulTry + app[Database].tryTransaction(implicit graph => app[CaseSrv].delete(c1.`case`)) must beSuccessfulTry app[Database].roTransaction { implicit graph => app[CaseSrv].get(c1._id).exists must beFalse } From 1b51cefe2e4313bae253c1ae243e8f2fae8043d5 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 22 Dec 2020 15:05:45 +0100 Subject: [PATCH 077/324] Code review changes --- .../thp/thehive/controllers/v0/ShareCtrl.scala | 4 ++-- .../thp/thehive/controllers/v1/Router.scala | 9 +++++---- .../thp/thehive/controllers/v1/TaskCtrl.scala | 13 +++++++++++++ .../app/org/thp/thehive/services/CaseSrv.scala | 7 ++++--- .../thp/thehive/services/ObservableSrv.scala | 3 ++- .../org/thp/thehive/services/ShareSrv.scala | 18 +++--------------- .../app/org/thp/thehive/services/TaskSrv.scala | 2 +- .../org/thp/thehive/services/CaseSrvTest.scala | 2 +- 8 files changed, 31 insertions(+), 27 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala index c9a9c5ad8d..8bca28c143 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala @@ -97,7 +97,7 @@ class ShareCtrl @Inject() ( .has(_.owner, false) ._id .orFail(AuthorizationError("Operation not permitted")) - _ <- shareSrv.remove(shareId) + _ <- shareSrv.unshareCase(shareId) } yield () } .map(_ => Results.NoContent) @@ -147,7 +147,7 @@ class ShareCtrl @Inject() ( else if (shareSrv.get(shareId).has(_.owner, true).exists) Failure(AuthorizationError("You can't remove initial shares")) else - shareSrv.remove(shareId) + shareSrv.unshareCase(shareId) def updateShare(shareId: String): Action[AnyContent] = entrypoint("update share") diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index feffe865bb..897516f315 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -68,10 +68,11 @@ class Router @Inject() ( // case GET(p"/share/$shareId") => shareCtrl.get(shareId) // case PATCH(p"/share/$shareId") => shareCtrl.update(shareId) - case GET(p"/task") => taskCtrl.list - case POST(p"/task") => taskCtrl.create - case GET(p"/task/$taskId") => taskCtrl.get(taskId) - case PATCH(p"/task/$taskId") => taskCtrl.update(taskId) + case GET(p"/task") => taskCtrl.list + case POST(p"/task") => taskCtrl.create + case GET(p"/task/$taskId") => taskCtrl.get(taskId) + case PATCH(p"/task/$taskId") => taskCtrl.update(taskId) + case DELETE(p"/task/$taskId") => taskCtrl.delete(taskId) // POST /case/:caseId/task/_search controllers.TaskCtrl.findInCase(caseId) // POST /case/task/_stats controllers.TaskCtrl.stats() diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala index 6ffdbb1b81..0b8d531e6f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala @@ -110,4 +110,17 @@ class TaskCtrl @Inject() ( ) .map(_ => Results.NoContent) } + + def delete(taskId: String): Action[AnyContent] = + entrypoint("delete task") + .authTransaction(db) { implicit request => implicit graph => + for { + t <- + taskSrv + .get(EntityIdOrName(taskId)) + .can(Permissions.manageTask) + .getOrFail("Task") + _ <- taskSrv.delete(t) + } yield Results.NoContent + } } diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index e6d286f58f..100deecafd 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -209,15 +209,16 @@ class CaseSrv @Inject() ( _ <- if (share.owner) delete(caze) else shareSrv.unshareCase(share._id) - _ <- shareSrv.delete(share._id) } yield () } - private[services] def delete(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + private def delete(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { val details = Json.obj("number" -> `case`.number, "title" -> `case`.title) for { organisation <- organisationSrv.getOrFail(authContext.organisation) - _ <- auditSrv.`case`.delete(`case`, organisation, Some(details)) + shares = get(`case`).shares.toSeq + _ <- shares.toTry(s => shareSrv.unshareCase(s._id)) + _ <- auditSrv.`case`.delete(`case`, organisation, Some(details)) } yield get(`case`).remove() } diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 5b24adbb7f..a7d6912acc 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -180,7 +180,8 @@ class ObservableSrv @Inject() ( case Some(alert) => for { _ <- Try(get(observable).remove()) - } yield auditSrv.observableInAlert.delete(observable, Some(alert)) + _ <- auditSrv.observableInAlert.delete(observable, Some(alert)) + } yield () } // Same as remove but with no Audit creation diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index e1d8262d00..692977cae7 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -87,13 +87,7 @@ class ShareSrv @Inject() ( } yield newShareProfile } -// def remove(`case`: Case with Entity, organisationId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = -// caseSrv.get(`case`).in[ShareCase].filter(_.in[OrganisationShare])._id.getOrFail().flatMap(remove(_)) // FIXME add organisation ? - - def remove(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - unshareCase(shareId).flatMap(_ => Try(get(shareId).remove())) - - private[services] def delete(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + private def delete(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { share <- getOrFail(shareId) tasks = get(share).tasks.toSeq @@ -106,14 +100,8 @@ class ShareSrv @Inject() ( for { case0 <- get(shareId).`case`.getOrFail("Case") organisation <- get(shareId).organisation.getOrFail("Organisation") - shareCase <- - get(shareId) - .`case` - .inE[ShareCase] - .filter(_.outV.v[Share].byOrganisation(organisation._id)) - .getOrFail("Case") - _ <- auditSrv.share.unshareCase(case0, organisation) - } yield shareCaseSrv.get(shareCase).remove() + _ <- auditSrv.share.unshareCase(case0, organisation) + } yield delete(shareId) def unshareTask( task: Task with Entity, diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index e6c618756a..48d1996415 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -63,7 +63,7 @@ class TaskSrv @Inject() ( .map(_ => get(task).remove()) } - private[services] def delete(t: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def delete(t: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = if (get(t).isShared.head) for { orga <- organisationSrv.getOrFail(authContext.organisation) diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index 82ab0826c3..7093197d9f 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -313,7 +313,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { ) .get - app[Database].tryTransaction(implicit graph => app[CaseSrv].delete(c1.`case`)) must beSuccessfulTry + app[Database].tryTransaction(implicit graph => app[CaseSrv].shareDelete(c1.`case`)) must beSuccessfulTry app[Database].roTransaction { implicit graph => app[CaseSrv].get(c1._id).exists must beFalse } From 0d815bca0ca1961645ed656e4f49df57a4d7d0b0 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 22 Dec 2020 15:14:01 +0100 Subject: [PATCH 078/324] Added delete route for observable --- .../app/org/thp/thehive/controllers/v1/ObservableCtrl.scala | 2 +- thehive/app/org/thp/thehive/controllers/v1/Router.scala | 3 +++ thehive/app/org/thp/thehive/services/ObservableSrv.scala | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index f383a7a025..d52a35f49d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -154,7 +154,7 @@ class ObservableCtrl @Inject() ( } def delete(obsId: String): Action[AnyContent] = - entryPoint("delete") + entryPoint("delete observable") .authTransaction(db) { implicit request => implicit graph => for { observable <- diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 897516f315..e9d8860441 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -11,6 +11,7 @@ class Router @Inject() ( caseTemplateCtrl: CaseTemplateCtrl, userCtrl: UserCtrl, organisationCtrl: OrganisationCtrl, + observableCtrl: ObservableCtrl, taskCtrl: TaskCtrl, customFieldCtrl: CustomFieldCtrl, alertCtrl: AlertCtrl, @@ -68,6 +69,8 @@ class Router @Inject() ( // case GET(p"/share/$shareId") => shareCtrl.get(shareId) // case PATCH(p"/share/$shareId") => shareCtrl.update(shareId) + case DELETE(p"/observable/$observableId") => observableCtrl.delete(observableId) + case GET(p"/task") => taskCtrl.list case POST(p"/task") => taskCtrl.create case GET(p"/task/$taskId") => taskCtrl.get(taskId) diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index a7d6912acc..a76af52ccc 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -185,7 +185,7 @@ class ObservableSrv @Inject() ( } // Same as remove but with no Audit creation - private[services] def delete(obs: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def delete(obs: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = if (get(obs).isShared.head) for { orga <- organisationSrv.getOrFail(authContext.organisation) From 9d0a4e64e519cb1637fcd69f8a462fd4698daffd Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 22 Dec 2020 15:26:43 +0100 Subject: [PATCH 079/324] Used delete instead of remove & fixed compilation warnings --- .../thp/thehive/controllers/v0/ObservableCtrl.scala | 10 ++++------ .../thp/thehive/controllers/v1/ObservableCtrl.scala | 12 +++++------- .../app/org/thp/thehive/services/ObservableSrv.scala | 3 ++- thehive/app/org/thp/thehive/services/ShareSrv.scala | 3 ++- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 629711357d..6cdf3219de 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -1,13 +1,8 @@ package org.thp.thehive.controllers.v0 -import java.io.FilterInputStream -import java.nio.file.Files - -import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ -import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ @@ -27,6 +22,9 @@ import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.libs.json.{JsArray, JsObject, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} +import java.io.FilterInputStream +import java.nio.file.Files +import javax.inject.{Inject, Named, Singleton} import scala.collection.JavaConverters._ import scala.util.Success @@ -214,7 +212,7 @@ class ObservableCtrl @Inject() ( } } - private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = + private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] = observable.attachment.toSeq.flatMap { attachment => val zipFile = new ZipFile(attachment.filepath.toFile) val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index d52a35f49d..27592a3ea7 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -1,13 +1,8 @@ package org.thp.thehive.controllers.v1 -import java.io.FilterInputStream -import java.nio.file.Files - -import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ -import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -25,6 +20,9 @@ import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.mvc.{Action, AnyContent, Results} import play.api.{Configuration, Logger} +import java.io.FilterInputStream +import java.nio.file.Files +import javax.inject.{Inject, Named, Singleton} import scala.collection.JavaConverters._ @Singleton @@ -162,7 +160,7 @@ class ObservableCtrl @Inject() ( .get(EntityIdOrName(obsId)) .can(Permissions.manageObservable) .getOrFail("Observable") - _ <- observableSrv.remove(observable) + _ <- observableSrv.delete(observable) } yield Results.NoContent } @@ -196,7 +194,7 @@ class ObservableCtrl @Inject() ( } } - private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = + private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] = observable.attachment.toSeq.flatMap { attachment => val zipFile = new ZipFile(attachment.filepath.toFile) val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index a76af52ccc..3a7ab94ba8 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -189,7 +189,8 @@ class ObservableSrv @Inject() ( if (get(obs).isShared.head) for { orga <- organisationSrv.getOrFail(authContext.organisation) - } yield shareSrv.unshareObservable(obs, orga) + _ <- shareSrv.unshareObservable(obs, orga) + } yield () else { val alert = get(obs).alert.headOption val attachments = get(obs).attachments.toSeq diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index 692977cae7..28484c29b0 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -101,7 +101,8 @@ class ShareSrv @Inject() ( case0 <- get(shareId).`case`.getOrFail("Case") organisation <- get(shareId).organisation.getOrFail("Organisation") _ <- auditSrv.share.unshareCase(case0, organisation) - } yield delete(shareId) + _ <- delete(shareId) + } yield () def unshareTask( task: Task with Entity, From adb899f987e748f1d1d5cd1747e2aafc4be4d9ce Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 26 Dec 2020 09:58:09 +0100 Subject: [PATCH 080/324] #1725 Retry to connect to cassandra until it is ready --- ScalliGraph | 2 +- .../app/org/thp/thehive/controllers/v1/ObservableCtrl.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index ddbc847ef3..2426d2e5f1 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit ddbc847ef30f2507e1287d894ad2191d873a0a87 +Subproject commit 2426d2e5f19ae24ed7c114d5ebd3c902027a6679 diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index 2f660a8561..49c9f3e3b2 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -43,8 +43,7 @@ class ObservableCtrl @Inject() ( attachmentSrv: AttachmentSrv, errorHandler: ErrorHandler, temporaryFileCreator: DefaultTemporaryFileCreator, - configuration: Configuration, - errorHandler: ErrorHandler + configuration: Configuration ) extends QueryableCtrl with ObservableRenderer { From 38dd9632b5707fa74e57c8e48a9e170dca0ea5e9 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 5 Jan 2021 14:09:51 +0100 Subject: [PATCH 081/324] Property refactoring --- thehive/app/org/thp/thehive/models/Audit.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/models/Audit.scala b/thehive/app/org/thp/thehive/models/Audit.scala index a97ebad3e8..38c636ddca 100644 --- a/thehive/app/org/thp/thehive/models/Audit.scala +++ b/thehive/app/org/thp/thehive/models/Audit.scala @@ -11,8 +11,15 @@ import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} @BuildEdgeEntity[Audit, User] case class AuditUser() -@DefineIndex(IndexType.basic, "requestId", "mainAction") @BuildVertexEntity +@DefineIndex(IndexType.basic, "requestId", "mainAction") +@DefineIndex(IndexType.standard, "requestId") +@DefineIndex(IndexType.standard, "action") +@DefineIndex(IndexType.standard, "mainAction") +@DefineIndex(IndexType.standard, "objectId") +@DefineIndex(IndexType.standard, "objectType") +@DefineIndex(IndexType.standard, "_createdAt") +@DefineIndex(IndexType.standard, "_updatedAt") case class Audit( requestId: String, action: String, From 4e3e54438837f6594538a47a69b6a840c57cb95a Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 5 Jan 2021 14:11:36 +0100 Subject: [PATCH 082/324] #1731 Wrap graph to include database instance --- ScalliGraph | 2 +- conf/logback.xml | 2 + .../resources/play/reference-overrides.conf | 4 +- .../controllers/v0/CortexQueryExecutor.scala | 7 +- .../connector/cortex/models/Action.scala | 7 +- .../cortex/models/SchemaUpdaterActor.scala | 76 ---------- .../models/SchemaUpdaterSerializer.scala | 25 ---- .../cortex/services/ActionOperationSrv.scala | 3 +- .../connector/cortex/services/ActionSrv.scala | 6 +- .../cortex/services/AnalyzerTemplateSrv.scala | 5 +- .../cortex/services/CortexAuditSrv.scala | 6 +- .../cortex/services/EntityHelper.scala | 2 +- .../connector/cortex/services/JobSrv.scala | 7 +- .../notification/notifiers/RunAnalyzer.scala | 2 +- .../notification/notifiers/RunResponder.scala | 3 +- .../thehive/migration/th4/NoAuditSrv.scala | 3 +- .../thp/thehive/migration/th4/Output.scala | 3 +- .../misp/services/MispExportSrv.scala | 3 +- .../misp/services/MispImportSrv.scala | 2 +- project/Dependencies.scala | 51 +++---- .../org/thp/thehive/controllers/dav/VFS.scala | 2 +- .../thehive/controllers/v0/AlertCtrl.scala | 9 +- .../controllers/v0/AuditRenderer.scala | 2 +- .../thp/thehive/controllers/v0/CaseCtrl.scala | 10 +- .../controllers/v0/CaseTemplateCtrl.scala | 6 +- .../thehive/controllers/v0/ConfigCtrl.scala | 2 +- .../controllers/v0/DashboardCtrl.scala | 8 +- .../thehive/controllers/v0/DescribeCtrl.scala | 2 +- .../thp/thehive/controllers/v0/ListCtrl.scala | 2 +- .../controllers/v0/ObservableCtrl.scala | 32 ++--- .../thehive/controllers/v0/QueryCtrl.scala | 6 +- .../thehive/controllers/v0/ShareCtrl.scala | 2 +- .../thp/thehive/controllers/v0/TaskCtrl.scala | 4 +- .../controllers/v0/TheHiveQueryExecutor.scala | 7 +- .../thp/thehive/controllers/v0/UserCtrl.scala | 8 +- .../thehive/controllers/v1/AlertCtrl.scala | 4 +- .../thp/thehive/controllers/v1/CaseCtrl.scala | 2 +- .../thehive/controllers/v1/Properties.scala | 116 ++++++---------- .../app/org/thp/thehive/models/Audit.scala | 9 +- .../thehive/models/SchemaUpdaterActor.scala | 130 ------------------ .../models/SchemaUpdaterSerializer.scala | 27 ---- .../models/TheHiveSchemaDefinition.scala | 3 +- .../org/thp/thehive/services/AlertSrv.scala | 3 +- .../thp/thehive/services/AttachmentSrv.scala | 3 +- .../org/thp/thehive/services/AuditSrv.scala | 6 +- .../org/thp/thehive/services/CaseSrv.scala | 7 +- .../thehive/services/CaseTemplateSrv.scala | 5 +- .../org/thp/thehive/services/ConfigSrv.scala | 3 +- .../thp/thehive/services/CustomFieldSrv.scala | 2 +- .../thp/thehive/services/DashboardSrv.scala | 4 +- .../org/thp/thehive/services/DataSrv.scala | 6 +- .../org/thp/thehive/services/FlowActor.scala | 7 +- .../thehive/services/ImpactStatusSrv.scala | 4 +- .../services/IntegrityCheckActor.scala | 19 +-- .../thp/thehive/services/KeyValueSrv.scala | 2 +- .../app/org/thp/thehive/services/LogSrv.scala | 3 +- .../thp/thehive/services/ObservableSrv.scala | 5 +- .../thehive/services/ObservableTypeSrv.scala | 4 +- .../thehive/services/OrganisationSrv.scala | 5 +- .../org/thp/thehive/services/PageSrv.scala | 3 +- .../org/thp/thehive/services/ProfileSrv.scala | 4 +- .../thp/thehive/services/ReportTagSrv.scala | 3 +- .../services/ResolutionStatusSrv.scala | 4 +- .../org/thp/thehive/services/RoleSrv.scala | 3 +- .../org/thp/thehive/services/ShareSrv.scala | 7 +- .../thp/thehive/services/TOTPAuthSrv.scala | 3 +- .../app/org/thp/thehive/services/TagSrv.scala | 5 +- .../org/thp/thehive/services/TaskSrv.scala | 3 +- .../org/thp/thehive/services/UserSrv.scala | 12 +- .../notification/NotificationActor.scala | 3 +- .../notification/notifiers/AppendToFile.scala | 7 +- .../notification/notifiers/Emailer.scala | 6 +- .../notification/notifiers/Mattermost.scala | 14 +- .../notification/notifiers/Notifier.scala | 6 +- .../notification/notifiers/Webhook.scala | 6 +- .../notification/triggers/CaseCreated.scala | 2 +- .../notification/triggers/FilteredEvent.scala | 3 +- .../notification/triggers/GlobalTrigger.scala | 2 +- .../notification/triggers/LogInMyTask.scala | 2 +- .../notification/triggers/TaskAssigned.scala | 2 +- .../notification/triggers/Trigger.scala | 6 +- .../thehive/services/th3/Aggregation.scala | 11 +- thehive/conf/play/reference-overrides.conf | 4 +- .../org/thp/thehive/DatabaseBuilder.scala | 3 +- thehive/test/org/thp/thehive/QueryTest.scala | 3 + .../thp/thehive/services/CaseSrvTest.scala | 2 +- 86 files changed, 257 insertions(+), 572 deletions(-) delete mode 100644 cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterActor.scala delete mode 100644 cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterSerializer.scala delete mode 100644 thehive/app/org/thp/thehive/models/SchemaUpdaterActor.scala delete mode 100644 thehive/app/org/thp/thehive/models/SchemaUpdaterSerializer.scala create mode 100644 thehive/test/org/thp/thehive/QueryTest.scala diff --git a/ScalliGraph b/ScalliGraph index 2426d2e5f1..a612bcd28d 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 2426d2e5f19ae24ed7c114d5ebd3c902027a6679 +Subproject commit a612bcd28dffd617d29ecfd5d0511e3d5ed3666d diff --git a/conf/logback.xml b/conf/logback.xml index 52ba1c47b9..94caf258db 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -36,6 +36,8 @@ + +