Skip to content

Commit

Permalink
#1731 Add index for tags
Browse files Browse the repository at this point in the history
  • Loading branch information
To-om committed Jan 27, 2021
1 parent cee616f commit f048c7a
Show file tree
Hide file tree
Showing 17 changed files with 166 additions and 148 deletions.
27 changes: 6 additions & 21 deletions frontend/app/scripts/services/api/TagSrv.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,28 +5,13 @@

var self = this;

var getTags = function(objectType, term) {
var getTags = function(objectType, term) { // TODO remove objectType parameter (not used anymore)
var defer = $q.defer();
var operations = [
{ _name: 'listTag' }
];

if(objectType) {
operations.push({ _name: objectType });
}

operations.push({
_name: 'filter',
_like: {
_field: 'text',
_value: '*' + term + '*'
}
});

operations.push({ _name: 'text' });

// Get the list
QuerySrv.call('v0', operations, {name: 'tags-auto-complete'})
QuerySrv.call('v0', [
{ _name: 'listTag' },
{ _name: 'autoComplete', freeTag: term, limit: 10 },
{ _name: 'text' }
], {name: 'tags-auto-complete'})
.then(function(data) {
defer.resolve(_.map(_.unique(data), function(tag) {
return {text: tag};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,10 @@ class Output @Inject() (
body(graph)(getAuthContext(userId))
}

def getTag(tagName: String)(implicit graph: Graph, authContext: AuthContext): Try[Tag with Entity] =
cache.getOrElseUpdate(s"tag-$tagName")(tagSrv.createEntity(Tag.fromString(tagName, tagSrv.defaultNamespace, tagSrv.defaultColour)))
def getTag(tagName: String, organisationId: String)(implicit graph: Graph, authContext: AuthContext): Try[Tag with Entity] =
cache.getOrElseUpdate(s"tag-$tagName")(
tagSrv.createEntity(Tag(s"_freetags_$organisationId", tagName, None, None, tagSrv.freeTagColour))
)

override def organisationExists(inputOrganisation: InputOrganisation): Boolean = organisations.contains(inputOrganisation.organisation.name)

Expand Down Expand Up @@ -540,7 +542,7 @@ class Output @Inject() (
.logFailure(s"Unable to set case template ${ct.name} to case #${createdCase.number}")
}
inputCase.`case`.tags.foreach { tagName =>
getTag(tagName)
getTag(tagName, organisationIds.head.value)
.flatMap(tag => caseSrv.caseTagSrv.create(CaseTag(), createdCase, tag))
.logFailure(s"Unable to add tag $tagName to case #${createdCase.number}")
}
Expand Down Expand Up @@ -667,7 +669,7 @@ class Output @Inject() (
_ = updateMetaData(observable, inputObservable.metaData)
_ <- observableSrv.observableObservableType.create(ObservableObservableType(), observable, observableType)
_ = inputObservable.observable.tags.foreach { tagName =>
getTag(tagName)
getTag(tagName, organisationIds.head.value)
.foreach(tag => observableSrv.observableTagSrv.create(ObservableTag(), observable, tag))
}
} yield observable
Expand Down Expand Up @@ -711,11 +713,11 @@ class Output @Inject() (
authTransaction(inputAlert.metaData.createdBy) { implicit graph => implicit authContext =>
logger.debug(s"Create alert ${inputAlert.alert.`type`}:${inputAlert.alert.source}:${inputAlert.alert.sourceRef}")
val `case` = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption)
val tags = inputAlert.alert.tags.flatMap(getTag(_).toOption)
for {
organisation <- getOrganisation(inputAlert.organisation)
createdAlert <- alertSrv.createEntity(inputAlert.alert.copy(organisationId = organisation._id, caseId = `case`.map(_._id)))
_ = updateMetaData(createdAlert, inputAlert.metaData)
tags = inputAlert.alert.tags.flatMap(getTag(_, organisation._id.value).toOption)
_ = updateMetaData(createdAlert, inputAlert.metaData)
_ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), createdAlert, organisation)
_ <-
inputAlert
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,6 @@ class PublicObservable @Inject() (
override val pageQuery: ParamQuery[OutputParam] =
Query.withParam[OutputParam, Traversal.V[Observable], IteratorOutput](
"page",
FieldsParser[OutputParam],
{
case (OutputParam(from, to, withStats, 0), observableSteps, authContext) =>
observableSteps
Expand Down Expand Up @@ -418,7 +417,7 @@ class PublicObservable @Inject() (
observableSrv
.get(vertex)(graph)
.getOrFail("Observable")
.flatMap(observable => observableSrv.updateTagNames(observable, value)(graph, authContext))
.flatMap(observable => observableSrv.updateTags(observable, value)(graph, authContext))
.map(_ => Json.obj("tags" -> value))
}
)
Expand Down
17 changes: 15 additions & 2 deletions thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ import org.thp.scalligraph.models.{Database, Entity, UMapping}
import org.thp.scalligraph.query._
import org.thp.scalligraph.traversal.TraversalOps._
import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal}
import org.thp.scalligraph.utils.FunctionalCondition.When
import org.thp.scalligraph.{EntityIdOrName, RichSeq}
import org.thp.thehive.controllers.v0.Conversion._
import org.thp.thehive.models.{Permissions, Tag}
import org.thp.thehive.services.TagOps._
import org.thp.thehive.services.TagSrv
import org.thp.thehive.services.{OrganisationSrv, TagSrv}
import play.api.libs.json.{JsNumber, JsObject, JsValue, Json}
import play.api.mvc.{Action, AnyContent, Results}

Expand Down Expand Up @@ -99,8 +100,10 @@ class TagCtrl @Inject() (
}
}

case class TagHint(freeTag: Option[String], namespace: Option[String], predicate: Option[String], value: Option[String], limit: Option[Long])

@Singleton
class PublicTag @Inject() (tagSrv: TagSrv) extends PublicData {
class PublicTag @Inject() (tagSrv: TagSrv, organisationSrv: OrganisationSrv) extends PublicData {
override val entityName: String = "tag"
override val initialQuery: Query = Query.init[Traversal.V[Tag]]("listTag", (graph, _) => tagSrv.startTraversal(graph))
override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Tag], IteratorOutput](
Expand All @@ -118,6 +121,16 @@ class PublicTag @Inject() (tagSrv: TagSrv) extends PublicData {
Query[Traversal.V[Tag], Traversal.V[Tag]]("fromCase", (tagSteps, _) => tagSteps.fromCase),
Query[Traversal.V[Tag], Traversal.V[Tag]]("fromObservable", (tagSteps, _) => tagSteps.fromObservable),
Query[Traversal.V[Tag], Traversal.V[Tag]]("fromAlert", (tagSteps, _) => tagSteps.fromAlert),
Query.withParam[TagHint, Traversal.V[Tag], Traversal.V[Tag]](
"autoComplete",
(tagHint, tags, authContext) =>
tagHint
.freeTag
.fold(tags.autoComplete(tagHint.namespace, tagHint.predicate, tagHint.value)(authContext))(
tags.autoComplete(organisationSrv, _)(authContext)
)
.merge(tagHint.limit)(_.limit(_))
),
Query[Traversal.V[Tag], Traversal[String, Vertex, Converter[String, Vertex]]]("text", (tagSteps, _) => tagSteps.displayName),
Query.output[String, Traversal[String, Vertex, Converter[String, Vertex]]]
)
Expand Down
10 changes: 1 addition & 9 deletions thehive/app/org/thp/thehive/controllers/v1/Properties.scala
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ class Properties @Inject() (
_.field.custom { (_, value, vertex, graph, authContext) =>
observableSrv
.getOrFail(vertex)(graph)
.flatMap(observable => observableSrv.updateTagNames(observable, value)(graph, authContext))
.flatMap(observable => observableSrv.updateTags(observable, value)(graph, authContext))
.map(_ => Json.obj("tags" -> value))
}
)
Expand All @@ -393,12 +393,4 @@ class Properties @Inject() (
.property("version", UMapping.int)(_.field.readonly)
.property("enabled", UMapping.boolean)(_.select(_.enabled).readonly)
.build

private def vertexToTag: Vertex => String = { v =>
val namespace = UMapping.string.getProperty(v, "namespace")
val predicate = UMapping.string.getProperty(v, "predicate")
val value = UMapping.string.optional.getProperty(v, "value")
Tag(namespace, predicate, value, None, "#000000").toString
}

}
5 changes: 2 additions & 3 deletions thehive/app/org/thp/thehive/controllers/v1/TaxonomyCtrl.scala
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ class TaxonomyCtrl @Inject() (
override val pageQuery: ParamQuery[OutputParam] =
Query.withParam[OutputParam, Traversal.V[Taxonomy], IteratorOutput](
"page",
FieldsParser[OutputParam],
{
case (OutputParam(from, to, extraData), taxoSteps, authContext) =>
taxoSteps.richPage(from, to, extraData.contains("total")) {
Expand Down Expand Up @@ -105,12 +104,12 @@ class TaxonomyCtrl @Inject() (
// Create tags
val tagValues = inputTaxo.values.getOrElse(Seq())
val tags = tagValues.flatMap { value =>
value.entry.map(e => Tag(inputTaxo.namespace, value.predicate, Some(e.value), e.expanded, e.colour.getOrElse(tagSrv.defaultColour)))
value.entry.map(e => Tag(inputTaxo.namespace, value.predicate, Some(e.value), e.expanded, e.colour.getOrElse(tagSrv.freeTagColour)))
}

// Create a tag for predicates with no tags associated
val predicateWithNoTags = inputTaxo.predicates.map(_.value).diff(tagValues.map(_.predicate))
val allTags = tags ++ predicateWithNoTags.map(p => Tag(inputTaxo.namespace, p, None, None, tagSrv.defaultColour))
val allTags = tags ++ predicateWithNoTags.map(p => Tag(inputTaxo.namespace, p, None, None, tagSrv.freeTagColour))

if (inputTaxo.namespace.isEmpty)
Failure(BadRequestError(s"A taxonomy with no namespace cannot be imported"))
Expand Down
22 changes: 13 additions & 9 deletions thehive/app/org/thp/thehive/models/Tag.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import play.api.Logger
import scala.util.matching.Regex

@DefineIndex(IndexType.unique, "namespace", "predicate", "value")
@DefineIndex(IndexType.fulltext, "namespace", "predicate", "value", "description")
@BuildVertexEntity
case class Tag(
namespace: String,
Expand All @@ -17,10 +18,13 @@ case class Tag(
) {
override def hashCode(): Int = 31 * (31 * value.## + predicate.##) + namespace.##

override def equals(obj: Any): Boolean = obj match {
case Tag(n, p, v, _, _) => n == namespace && p == predicate && v == value
case _ => false
}
override def equals(obj: Any): Boolean =
obj match {
case Tag(n, p, v, _, _) => n == namespace && p == predicate && v == value
case _ => false
}

lazy val isFreeTag: Boolean = namespace.startsWith("_freetags_")

override def canEqual(that: Any): Boolean = that.isInstanceOf[Tag]

Expand All @@ -35,8 +39,8 @@ object Tag {
val tagColour: Regex = "(.*)(#\\p{XDigit}{6})".r
val namespacePredicateValue: Regex = "([^\".:=]+)[.:]([^\".=]+)=\"?([^\"]+)\"?".r
val namespacePredicate: Regex = "([^\".:=]+)[.]([^\".=]+)".r
val PredicateValue: Regex = "([^\".:=]+)[=:]\"?([^\"]+)\"?".r
val predicate: Regex = "([^\".:=]+)".r
// val PredicateValue: Regex = "([^\".:=]+)[=:]\"?([^\"]+)\"?".r
val predicate: Regex = "([^\".:=]+)".r

def fromString(tagName: String, defaultNamespace: String, defaultColour: String = "#000000"): Tag = {
val (name, colour) = tagName match {
Expand All @@ -47,9 +51,9 @@ object Tag {
case namespacePredicateValue(namespace, predicate, value) if value.exists(_ != '=') =>
Tag(namespace.trim, predicate.trim, Some(value.trim), None, colour)
case namespacePredicate(namespace, predicate) => Tag(namespace.trim, predicate.trim, None, None, colour)
case PredicateValue(predicate, value) => Tag(defaultNamespace, predicate.trim, Some(value.trim), None, colour)
case predicate(predicate) => Tag(defaultNamespace, predicate.trim, None, None, colour)
case _ => Tag(defaultNamespace, name, None, None, colour)
// case PredicateValue(predicate, value) => Tag(defaultNamespace, predicate.trim, Some(value.trim), None, colour)
case predicate(predicate) => Tag(defaultNamespace, predicate.trim, None, None, colour)
case _ => Tag(defaultNamespace, name, None, None, colour)
}
}
}
102 changes: 57 additions & 45 deletions thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
package org.thp.thehive.models

import org.apache.tinkerpop.gremlin.process.traversal.P
import org.apache.tinkerpop.gremlin.process.traversal.{Order, P, TextP}
import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality
import org.janusgraph.core.schema.ConsistencyModifier
import org.janusgraph.graphdb.types.TypeDefinitionCategory
import org.reflections.Reflections
import org.reflections.scanners.SubTypesScanner
import org.reflections.util.ConfigurationBuilder
import org.thp.scalligraph.EntityId
import org.thp.scalligraph.auth.AuthContext
import org.thp.scalligraph.janus.JanusDatabase
import org.thp.scalligraph.models._
import org.thp.scalligraph.traversal.TraversalOps._
import org.thp.scalligraph.traversal.{Converter, Graph, Traversal}
import org.thp.scalligraph.{EntityId, RichSeq}
import org.thp.thehive.services.LocalUserSrv
import play.api.Logger

Expand Down Expand Up @@ -94,59 +94,70 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
Success(())
}
//=====[release 4.0.3]=====
//=====[release 4.0.4]=====
// Taxonomies
.addVertexModel[String]("Taxonomy", Seq("namespace"))
.addVertexModel[String]("Taxonomy")
.addProperty[String]("Taxonomy", "namespace")
.addProperty[String]("Taxonomy", "description")
.addProperty[Int]("Taxonomy", "version")
.dbOperation[Database]("Add Custom taxonomy vertex for each Organisation") { db =>
db.tryTransaction { implicit g =>
// For each organisation, if there is no custom taxonomy, create it
db.labelFilter("Organisation", Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o =>
Traversal.V(EntityId(o.id)).out[OrganisationTaxonomy].v[Taxonomy].unsafeHas("namespace", s"_freetags_${o.id()}").headOption match {
case None =>
val taxoVertex = g.addVertex("Taxonomy")
taxoVertex.property("_label", "Taxonomy")
taxoVertex.property("_createdBy", "[email protected]")
taxoVertex.property("_createdAt", new Date())
taxoVertex.property("namespace", s"_freetags_${o.id()}")
taxoVertex.property("description", "Custom taxonomy")
taxoVertex.property("version", 1)
o.addEdge("OrganisationTaxonomy", taxoVertex)
Success(())
case _ => Success(())
db.labelFilter("Organisation", Traversal.V()).unsafeHas("name", P.neq("admin")).foreach { o =>
val hasFreetagsTaxonomy = Traversal
.V(EntityId(o.id))
.out[OrganisationTaxonomy]
.v[Taxonomy]
.unsafeHas("namespace", s"_freetags_${o.id()}")
.exists
if (!hasFreetagsTaxonomy) {
val taxoVertex = g.addVertex("Taxonomy")
taxoVertex.property("_label", "Taxonomy")
taxoVertex.property("_createdBy", "[email protected]")
taxoVertex.property("_createdAt", new Date())
taxoVertex.property("namespace", s"_freetags_${o.id()}")
taxoVertex.property("description", "Custom taxonomy")
taxoVertex.property("version", 1)
o.addEdge("OrganisationTaxonomy", taxoVertex)
}
}
}.map(_ => ())
Success(())
}
}
.dbOperation[Database]("Add each tag to its Organisation's Custom taxonomy") { db =>
db.tryTransaction { implicit g =>
db.labelFilter("Organisation", Traversal.V()).unsafeHas("name", P.neq("admin")).toIterator.toTry { o =>
val customTaxo = Traversal.V(EntityId(o.id())).out("OrganisationTaxonomy").unsafeHas("namespace", s"_freetags_${o.id()}").head
Traversal
.V(EntityId(o.id()))
.unionFlat(
_.out("OrganisationShare").out("ShareCase").out("CaseTag"),
_.out("OrganisationShare").out("ShareObservable").out("ObservableTag"),
_.in("AlertOrganisation").out("AlertTag"),
_.in("CaseTemplateOrganisation").out("CaseTemplateTag")
.updateGraph("Add each tag to its Organisation's FreeTags taxonomy", "Tag") { tags =>
tags
.project(
_.by.by(
_.unionFlat(
_.in("CaseTag").in("ShareCase").in("OrganisationShare"),
_.in("ObservableTag").unionFlat(_.in("ShareObservable").in("OrganisationShare"), _.in("AlertObservable").out("AlertOrganisation")),
_.in("AlertTag").out("AlertOrganisation"),
_.in("CaseTemplateTag").out("CaseTemplateOrganisation")
)
.toSeq
.foreach { tag =>
// Create a freetext tag and store it into predicate
val tagStr = tagString(
tag.property("namespace").value().toString,
tag.property("predicate").value().toString,
tag.property("value").orElse("")
)
tag.property("namespace", s"_freetags_${o.id()}")
tag.property("predicate", tagStr)
tag.property("value").remove()
customTaxo.addEdge("TaxonomyTag", tag)
}
Success(())
.dedup
.sort(_.by("_createdAt", Order.desc))
.limit(1)
.out("OrganisationTaxonomy")
.unsafeHas("namespace", TextP.startingWith("_freetags_"))
.option
)
)
.foreach {
case (tag, Some(freeTagsTaxo)) =>
val tagStr = tagString(
tag.property[String]("namespace").value(),
tag.property[String]("predicate").value(),
tag.property[String]("value").orElse("")
)
tag.property("namespace", freeTagsTaxo.property[String]("namespace").value)
tag.property("predicate", tagStr)
tag.property("value").remove()
freeTagsTaxo.addEdge("TaxonomyTag", tag)
}
}.map(_ => ())
Success(())
}
.updateGraph("Add manageTaxonomy to admin profile", "Profile") { traversal =>
Try(traversal.unsafeHas("name", "admin").raw.property("permissions", "manageTaxonomy").iterate())
traversal.unsafeHas("name", "admin").raw.property("permissions", "manageTaxonomy").iterate()
Success(())
}
.updateGraph("Remove colour property for Tags", "Tag") { traversal =>
Expand All @@ -159,6 +170,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
traversal.raw.property("colour", "#000000").iterate()
Success(())
}
// Patterns
.updateGraph("Add managePattern permission to admin profile", "Profile") { traversal =>
traversal.unsafeHas("name", "admin").raw.property("permissions", "managePattern").iterate()
Success(())
Expand All @@ -171,7 +183,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
.iterate()
Success(())
}
//=====[release 4.0.3]=====
// Index backend
/* Alert index */
.addProperty[Seq[String]]("Alert", "tags")
.addProperty[EntityId]("Alert", "organisationId")
Expand Down
Loading

0 comments on commit f048c7a

Please sign in to comment.