From 834c74b3a8b40e5d5f1256adf32f03a731efdb12 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 29 Jul 2020 10:06:49 +0200 Subject: [PATCH 001/237] #1454 Add tasks in case template output --- dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala | 3 ++- thehive/app/org/thp/thehive/controllers/v1/Conversion.scala | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala index f57a63af9e..c43c0496ac 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala @@ -42,7 +42,8 @@ case class OutputCaseTemplate( tlp: Option[Int], pap: Option[Int], summary: Option[String], - customFields: Set[OutputCustomFieldValue] = Set.empty + customFields: Set[OutputCustomFieldValue] = Set.empty, + tasks: Seq[OutputTask] ) object OutputCaseTemplate { diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 00a48d3ae6..4ec7257c4d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -145,6 +145,7 @@ object Conversion { .withFieldConst(_._type, "CaseTemplate") .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).toSet) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) + .withFieldComputed(_.tasks, _.tasks.map(_.toOutput)) .transform ) From 88f43fe17d84dea524ae9591993d4853580cfe86 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 5 Aug 2020 08:06:04 +0200 Subject: [PATCH 002/237] #1461 Fix localfs location setting in configuration sample --- conf/application.sample.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/application.sample.conf b/conf/application.sample.conf index aa0950aab4..902228ccce 100644 --- a/conf/application.sample.conf +++ b/conf/application.sample.conf @@ -33,7 +33,7 @@ db.janusgraph { storage { ## Local filesystem // provider: localfs - // localfs.directory: /path/to/files + // localfs.location: /path/to/files ## Hadoop filesystem (HDFS) // provider: hdfs @@ -96,4 +96,4 @@ storage { //} # Define maximum size of attachments (default 10MB) -//play.http.parser.maxDiskBuffer: 1GB \ No newline at end of file +//play.http.parser.maxDiskBuffer: 1GB From 11e92c18a8b3fefe8af2401e27e981ae9c21608b Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 7 Aug 2020 17:45:42 +0200 Subject: [PATCH 003/237] #1454 Improve customfields parser --- .../thp/thehive/dto/v1/CustomFieldValue.scala | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index 7f0112345e..7be924aa0d 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -24,24 +24,37 @@ case class InputCustomFieldValue(name: String, value: Option[Any], order: Option object InputCustomFieldValue { - val parser: FieldsParser[Seq[InputCustomFieldValue]] = FieldsParser("customFieldValue") { + val valueParser: FieldsParser[Option[Any]] = FieldsParser("customFieldValue") { + case (_, FString(value)) => Good(Some(value)) + case (_, FNumber(value)) => Good(Some(value)) + case (_, FBoolean(value)) => Good(Some(value)) + case (_, FAny(value :: _)) => Good(Some(value)) + case (_, FUndefined | FNull) => Good(None) + } + + val parser: FieldsParser[Seq[InputCustomFieldValue]] = FieldsParser("customFieldValues") { case (_, FObject(fields)) => fields .toSeq .validatedBy { - case (name, FString(value)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FNumber(value)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FBoolean(value)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FAny(value :: _)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FNull) => Good(InputCustomFieldValue(name, None, None)) - case (name, other) => - Bad( - One( - InvalidFormatAttributeError(name, "CustomFieldValue", Set("field: string", "field: number", "field: boolean", "field: date"), other) - ) - ) + case (name, valueField) => valueParser(valueField).map(v => InputCustomFieldValue(name, v, None)) } .map(_.toSeq) + case (_, FSeq(list)) => + list.zipWithIndex.validatedBy { + case (cf: FObject, i) => + val order = FieldsParser.int(cf.get("order")).getOrElse(i) + for { + name <- FieldsParser.string(cf.get("name")) + value <- valueParser(cf.get("value")) + } yield InputCustomFieldValue(name, value, Some(order)) + case (other, i) => + Bad( + One( + InvalidFormatAttributeError(s"customFild[$i]", "CustomFieldValue", Set.empty, other) + ) + ) + } case _ => Good(Nil) } implicit val writes: Writes[Seq[InputCustomFieldValue]] = Writes[Seq[InputCustomFieldValue]] { icfv => From b3e4185c9ff026706aae8452e8b7dc4de06d5fcb Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 7 Aug 2020 18:12:38 +0200 Subject: [PATCH 004/237] Change drone secrets --- .drone.yml | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/.drone.yml b/.drone.yml index e75677b412..876ad17c27 100644 --- a/.drone.yml +++ b/.drone.yml @@ -48,8 +48,7 @@ steps: - name: build-packages image: thehiveproject/drone-scala-node settings: - pgp_key: - from_secret: pgp_key + pgp_key: {from_secret: pgp_key} commands: - | V=$(sbt -no-colors --error "print thehive/version" | tail -1) @@ -76,14 +75,10 @@ steps: - name: send packages image: appleboy/drone-scp settings: - host: - from_secret: scp_host - username: - from_secret: scp_user - key: - from_secret: scp_key - target: - from_secret: incoming_path + host: {from_secret: package_host} + username: {from_secret: package_user} + key: {from_secret: package_key} + target: {from_secret: incoming_path} source: - target/thehive*.deb - target/thehive*.rpm @@ -96,14 +91,10 @@ steps: - name: publish packages image: appleboy/drone-ssh settings: - host: - from_secret: scp_host - user: - from_secret: scp_user - key: - from_secret: scp_key - publish_script: - from_secret: publish_script + host: {from_secret: package_host} + user: {from_secret: package_user} + key: {from_secret: package_key} + publish_script: {from_secret: publish_script} commands: - PLUGIN_SCRIPT="bash $PLUGIN_PUBLISH_SCRIPT thehive4 $(cat thehive-version.txt)" /bin/drone-ssh when: @@ -116,10 +107,8 @@ steps: context: target/docker/stage dockerfile: target/docker/stage/Dockerfile repo: thehiveproject/thehive4 - username: - from_secret: docker_username - password: - from_secret: docker_password + username: {from_secret: docker_username} + password: {from_secret: docker_password} when: event: [tag] @@ -129,14 +118,10 @@ steps: settings: context: target/docker/stage dockerfile: target/docker/stage/Dockerfile - registry: - from_secret: harbor_server - repo: - from_secret: harbor_repo - username: - from_secret: harbor_username - password: - from_secret: harbor_password + registry: {from_secret: harbor_registry} + repo: {from_secret: harbor_repo} + username: {from_secret: harbor_username} + password: {from_secret: harbor_password} when: event: [tag] From 9d29e9646877d5191f4190792d117d986f7442f0 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 10 Aug 2020 11:56:47 +0200 Subject: [PATCH 005/237] #1469 Fix elasticsearch count queries --- .../org/thp/thehive/migration/th3/Input.scala | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala index 79651d1ced..f786a63bf8 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala @@ -96,10 +96,9 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[InputCase] override def countCases(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query(bool(caseFilter(filter) :+ termQuery("relations", "case"), Nil, Nil)) - .limit(0) )._2 override def listCaseObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed] = @@ -118,7 +117,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputObservable](json => Try((json \ "_parent").as[String])) override def countCaseObservables(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -130,7 +129,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listCaseObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed] = @@ -149,7 +147,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputObservable](json => Try((json \ "_parent").as[String])) override def countCaseObservables(caseId: String): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -161,7 +159,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listCaseTasks(filter: Filter): Source[Try[(String, InputTask)], NotUsed] = @@ -180,7 +177,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputTask](json => Try((json \ "_parent").as[String])) override def countCaseTasks(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -192,7 +189,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listCaseTasks(caseId: String): Source[Try[(String, InputTask)], NotUsed] = @@ -211,7 +207,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputTask](json => Try((json \ "_parent").as[String])) override def countCaseTasks(caseId: String): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -223,7 +219,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listCaseTaskLogs(filter: Filter): Source[Try[(String, InputLog)], NotUsed] = @@ -246,7 +241,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputLog](json => Try((json \ "_parent").as[String])) override def countCaseTaskLogs(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -262,7 +257,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listCaseTaskLogs(caseId: String): Source[Try[(String, InputLog)], NotUsed] = @@ -285,7 +279,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputLog](json => Try((json \ "_parent").as[String])) override def countCaseTaskLogs(caseId: String): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -301,7 +295,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 def alertFilter(filter: Filter): Seq[RangeQuery] = @@ -318,7 +311,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[InputAlert] override def countAlerts(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(alertFilter(filter) :+ termQuery("relations", "alert"), Nil, Nil)).limit(0))._2 + dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(bool(alertFilter(filter) :+ termQuery("relations", "alert"), Nil, Nil)))._2 override def listAlertObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed] = dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(alertFilter(filter) :+ termQuery("relations", "alert"), Nil, Nil))) @@ -382,7 +375,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[InputUser] override def countUsers(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(termQuery("relations", "user")).limit(0))._2 + dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(termQuery("relations", "user")))._2 override def listCustomFields(filter: Filter): Source[Try[InputCustomField], NotUsed] = dbFind(Some("all"), Nil)(indexName => @@ -396,7 +389,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe )._1.read[InputCustomField] override def countCustomFields(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -405,7 +398,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listObservableTypes(filter: Filter): Source[Try[InputObservableType], NotUsed] = @@ -415,8 +407,8 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[InputObservableType] override def countObservableTypes(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => - search(indexName).query(bool(Seq(termQuery("relations", "dblist"), termQuery("dblist", "list_artifactDataType")), Nil, Nil)).limit(0) + dbFind(Some("0-0"), Nil)(indexName => + search(indexName).query(bool(Seq(termQuery("relations", "dblist"), termQuery("dblist", "list_artifactDataType")), Nil, Nil)) )._2 override def listProfiles(filter: Filter): Source[Try[InputProfile], NotUsed] = @@ -444,7 +436,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[InputCaseTemplate] override def countCaseTemplate(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(termQuery("relations", "caseTemplate")).limit(0))._2 + dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(termQuery("relations", "caseTemplate")))._2 override def listCaseTemplateTask(filter: Filter): Source[Try[(String, InputTask)], NotUsed] = dbFind(Some("all"), Nil)(indexName => search(indexName).query(termQuery("relations", "caseTemplate"))) @@ -507,7 +499,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputJob](json => Try((json \ "_parent").as[String]))(jobReads, classTag[InputJob]) override def countJobs(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -523,7 +515,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listJobs(caseId: String): Source[Try[(String, InputJob)], NotUsed] = @@ -546,7 +537,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .readWithParent[InputJob](json => Try((json \ "_parent").as[String]))(jobReads, classTag[InputJob]) override def countJobs(caseId: String): Future[Long] = - dbFind(Some("all"), Nil)(indexName => + dbFind(Some("0-0"), Nil)(indexName => search(indexName) .query( bool( @@ -562,7 +553,6 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe Nil ) ) - .limit(0) )._2 override def listJobObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed] = @@ -631,7 +621,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[(String, InputAction)] override def countAction(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(termQuery("relations", "action")).limit(0))._2 + dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(termQuery("relations", "action")))._2 override def listAction(entityId: String): Source[Try[(String, InputAction)], NotUsed] = dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(Seq(termQuery("relations", "action"), idsQuery(entityId)), Nil, Nil))) @@ -639,7 +629,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[(String, InputAction)] override def countAction(entityId: String): Future[Long] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(Seq(termQuery("relations", "action"), idsQuery(entityId)), Nil, Nil)).limit(0))._2 + dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(bool(Seq(termQuery("relations", "action"), idsQuery(entityId)), Nil, Nil)))._2 def auditFilter(filter: Filter): Seq[RangeQuery] = if (filter.auditDateRange._1.isDefined || filter.auditDateRange._2.isDefined) { @@ -654,7 +644,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe .read[(String, InputAudit)] override def countAudit(filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit"), Nil, Nil)).limit(0))._2 + dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit"), Nil, Nil)))._2 override def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed] = dbFind(Some("all"), Nil)(indexName => @@ -662,7 +652,7 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe )._1.read[(String, InputAudit)] def countAudit(entityId: String, filter: Filter): Future[Long] = - dbFind(Some("all"), Nil)(indexName => - search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), Nil, Nil)).limit(0) + dbFind(Some("0-0"), Nil)(indexName => + search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), Nil, Nil)) )._2 } From a1a358df990a00aae84b20ed3f145dd2c41ec41f Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 10 Aug 2020 11:59:10 +0200 Subject: [PATCH 006/237] #1469 Don't check alert duplication --- .../scala/org/thp/thehive/migration/th4/Output.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 2a31b34304..6c22d7c27a 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -670,9 +670,14 @@ class Output @Inject() ( } ) tags = inputAlert.tags.filterNot(_.isEmpty).flatMap(getTag(_).toOption).toSeq - alert <- alertSrv.create(inputAlert.alert, organisation, tags, inputAlert.customFields, caseTemplate) - _ = updateMetaData(alert.alert, inputAlert.metaData) - _ = inputAlert.caseId.flatMap(getCase(_).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert.alert, _)) +// alert <- alertSrv.create(inputAlert.alert, organisation, tags, inputAlert.customFields, caseTemplate) // FIXME don't check duplicate + alert <- alertSrv.createEntity(inputAlert.alert) + _ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), alert, organisation) + _ <- caseTemplate.map(ct => alertSrv.alertCaseTemplateSrv.create(AlertCaseTemplate(), alert, ct)).flip + _ <- tags.toTry(t => alertSrv.alertTagSrv.create(AlertTag(), alert, t)) + _ <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, name, value) } + _ = updateMetaData(alert, inputAlert.metaData) + _ = inputAlert.caseId.flatMap(getCase(_).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert, _)) } yield IdMapping(inputAlert.metaData.id, alert._id) } From 0d11679ff844442b00dd50affa79965a4e9660bb Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 10 Aug 2020 11:59:34 +0200 Subject: [PATCH 007/237] #1469 Don't search profile --- .../src/main/scala/org/thp/thehive/migration/th4/Output.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 6c22d7c27a..3ce399c454 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -526,7 +526,7 @@ class Output @Inject() ( } inputCase.organisations.foldLeft(false) { case (ownerSet, (organisationName, profileName)) => - val owner = profileName == profileSrv.orgAdmin.name && !ownerSet + val owner = profileName == Profile.orgAdmin.name && !ownerSet val shared = for { organisation <- getOrganisation(organisationName) profile <- getProfile(profileName) From 2db770b7d06bf6b51cdb1c7602ebdb5e6851e30b Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 10 Aug 2020 12:03:43 +0200 Subject: [PATCH 008/237] Update version --- build.sbt | 2 +- frontend/bower.json | 2 +- frontend/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 68872fba7f..22aed1a0a3 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.0.0-1" +val thehiveVersion = "4.0.1-1-SNAPSHOT" val scala212 = "2.12.11" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/frontend/bower.json b/frontend/bower.json index 205e3d1659..dc76d44f6e 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.0", + "version": "4.0.1-1-SNAPSHOT", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index b4abacdea8..eaa2a3099b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.0", + "version": "4.0.1-1-SNAPSHOT", "license": "AGPL-3.0", "repository": { "type": "git", From 028b52e58d476a06a60b9a24677d440a3c4ff98d Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 10 Aug 2020 16:02:27 +0200 Subject: [PATCH 009/237] #1469 Add information on already migrated entities --- .../thp/thehive/migration/MigrationOps.scala | 2 +- .../thp/thehive/migration/th4/Output.scala | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala index 5038c17858..2f329a67e1 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala @@ -25,7 +25,7 @@ class MigrationStats() { count = 0 sum = 0 } - def isEmpty: Boolean = count == 0 + def isEmpty: Boolean = count == 0L override def toString: String = if (isEmpty) "0" else (sum / count).toString } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 3ce399c454..a7d1c93d61 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -228,6 +228,27 @@ class Output @Inject() ( caseTemplates = caseTemplatesBuilder.result() caseNumbers = caseNumbersBuilder.result() alerts = alertsBuilder.result() + if (profiles.nonEmpty || + organisations.nonEmpty || + users.nonEmpty || + impactStatuses.nonEmpty || + resolutionStatuses.nonEmpty || + observableTypes.nonEmpty || + customFields.nonEmpty || + caseTemplates.nonEmpty || + caseNumbers.nonEmpty || + alerts.nonEmpty) + logger.info(s"""Already migrated: + | ${profiles.size} profiles\n + | ${organisations.size} organisations\n + | ${users.size} users\n + | ${impactStatuses.size} impactStatuses\n + | ${resolutionStatuses.size} resolutionStatuses\n + | ${observableTypes.size} observableTypes\n + | ${customFields.size} customFields\n + | ${caseTemplates.size} caseTemplates\n + | ${caseNumbers.size} caseNumbers\n + | ${alerts.size} alerts""".stripMargin) } def startMigration(): Try[Unit] = { From cf775dcdf8196ab12cd9eccd92f27d1f3edfd527 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 11 Aug 2020 08:54:00 +0200 Subject: [PATCH 010/237] #1469 Don't check observable type existence before create new one --- .../src/main/scala/org/thp/thehive/migration/th4/Output.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index a7d1c93d61..d233e2db5f 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -388,7 +388,7 @@ class Output @Inject() ( observableTypes .get(typeName) .fold[Try[ObservableType with Entity]] { - observableTypeSrv.create(ObservableType(typeName, isAttachment = false)).map { ot => + observableTypeSrv.createEntity(ObservableType(typeName, isAttachment = false)).map { ot => observableTypes += (typeName -> ot) ot } From 3d58beecfa48d814ee9cba91065f5fd34b249a06 Mon Sep 17 00:00:00 2001 From: Explie Date: Wed, 26 Aug 2020 11:06:47 +0200 Subject: [PATCH 011/237] Moved nested /case/.../ calls to the top of the routing table. Fix for: https://github.com/TheHive-Project/TheHive/issues/1492 --- .../thp/thehive/controllers/v0/Router.scala | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Router.scala b/thehive/app/org/thp/thehive/controllers/v0/Router.scala index 957d6e9831..e1b5772d59 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Router.scala @@ -58,6 +58,34 @@ class Router @Inject() ( case DELETE(p"/case/share/$shareId") => shareCtrl.removeShare(shareId) case PATCH(p"/case/share/$shareId") => shareCtrl.updateShare(shareId) + case GET(p"/case/task") => queryExecutor.task.search + case POST(p"/case/$caseId/task") => taskCtrl.create(caseId) // Audit ok + case GET(p"/case/task/$taskId") => taskCtrl.get(taskId) + case PATCH(p"/case/task/$taskId") => taskCtrl.update(taskId) // Audit ok + case POST(p"/case/task/_search") => queryExecutor.task.search + //case POST(p"/case/$caseId/task/_search") => taskCtrl.search + case POST(p"/case/task/_stats") => queryExecutor.task.stats + + //case GET(p"/case/task/$taskId/log") => logCtrl.findInTask(taskId) + //case POST(p"/case/task/$taskId/log/_search") => logCtrl.findInTask(taskId) + case POST(p"/case/task/log/_search") => queryExecutor.log.search + case POST(p"/case/task/log/_stats") => queryExecutor.log.stats + case POST(p"/case/task/$taskId/log") => logCtrl.create(taskId) // Audit ok + case PATCH(p"/case/task/log/$logId") => logCtrl.update(logId) // Audit ok + case DELETE(p"/case/task/log/$logId") => logCtrl.delete(logId) // Audit ok, weird logs/silent errors though (stream related) + // case GET(p"/case/task/log/$logId") => logCtrl.get(logId) + + case POST(p"/case/artifact/_search") => queryExecutor.observable.search + // case POST(p"/case/:caseId/artifact/_search") ⇒ observableCtrl.findInCase(caseId) + case POST(p"/case/artifact/_stats") => queryExecutor.observable.stats + case POST(p"/case/$caseId/artifact") => observableCtrl.create(caseId) // Audit ok + case GET(p"/case/artifact/$observableId") => observableCtrl.get(observableId) + case DELETE(p"/case/artifact/$observableId") => observableCtrl.delete(observableId) // Audit ok + case PATCH(p"/case/artifact/_bulk") => observableCtrl.bulkUpdate // Audit ok + case PATCH(p"/case/artifact/$observableId") => observableCtrl.update(observableId) // Audit ok + case GET(p"/case/artifact/$observableId/similar") => observableCtrl.findSimilar(observableId) + case POST(p"/case/artifact/$observableId/shares") => shareCtrl.shareObservable(observableId) + case GET(p"/case") => queryExecutor.`case`.search case POST(p"/case") => caseCtrl.create // Audit ok case GET(p"/case/$caseId") => caseCtrl.get(caseId) @@ -107,34 +135,6 @@ class Router @Inject() ( case PUT(p"/organisation/$organisationId1/links") => organisationCtrl.bulkLink(organisationId1) case DELETE(p"/organisation/$organisationId1/link/$organisationId2") => organisationCtrl.unlink(organisationId1, organisationId2) - case GET(p"/case/task") => queryExecutor.task.search - case POST(p"/case/$caseId/task") => taskCtrl.create(caseId) // Audit ok - case GET(p"/case/task/$taskId") => taskCtrl.get(taskId) - case PATCH(p"/case/task/$taskId") => taskCtrl.update(taskId) // Audit ok - case POST(p"/case/task/_search") => queryExecutor.task.search - //case POST(p"/case/$caseId/task/_search") => taskCtrl.search - case POST(p"/case/task/_stats") => queryExecutor.task.stats - -//case GET(p"/case/task/$taskId/log") => logCtrl.findInTask(taskId) -//case POST(p"/case/task/$taskId/log/_search") => logCtrl.findInTask(taskId) - case POST(p"/case/task/log/_search") => queryExecutor.log.search - case POST(p"/case/task/log/_stats") => queryExecutor.log.stats - case POST(p"/case/task/$taskId/log") => logCtrl.create(taskId) // Audit ok - case PATCH(p"/case/task/log/$logId") => logCtrl.update(logId) // Audit ok - case DELETE(p"/case/task/log/$logId") => logCtrl.delete(logId) // Audit ok, weird logs/silent errors though (stream related) -// case GET(p"/case/task/log/$logId") => logCtrl.get(logId) - - case POST(p"/case/artifact/_search") => queryExecutor.observable.search -// case POST(p"/case/:caseId/artifact/_search") ⇒ observableCtrl.findInCase(caseId) - case POST(p"/case/artifact/_stats") => queryExecutor.observable.stats - case POST(p"/case/$caseId/artifact") => observableCtrl.create(caseId) // Audit ok - case GET(p"/case/artifact/$observableId") => observableCtrl.get(observableId) - case DELETE(p"/case/artifact/$observableId") => observableCtrl.delete(observableId) // Audit ok - case PATCH(p"/case/artifact/_bulk") => observableCtrl.bulkUpdate // Audit ok - case PATCH(p"/case/artifact/$observableId") => observableCtrl.update(observableId) // Audit ok - case GET(p"/case/artifact/$observableId/similar") => observableCtrl.findSimilar(observableId) - case POST(p"/case/artifact/$observableId/shares") => shareCtrl.shareObservable(observableId) - case GET(p"/customField") => customFieldCtrl.list case POST(p"/customField") => customFieldCtrl.create case GET(p"/customField/$id") => customFieldCtrl.get(id) From c7f29276c746d7bb19c2d0da0b04f84badfc3881 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 1 Sep 2020 09:30:18 +0200 Subject: [PATCH 012/237] #1501 Traversal operations refactoring --- .idea/runConfigurations/Cortex_tests.xml | 50 -- .idea/runConfigurations/Misp_tests.xml | 34 -- .idea/runConfigurations/Scalligraph_tests.xml | 60 --- .idea/runConfigurations/TheHive.xml | 30 -- .idea/runConfigurations/TheHive_tests.xml | 116 ----- .idea/runConfigurations/cortex_ActionSrv.xml | 22 - .../runConfigurations/cortex_AnalyzerSrv.xml | 22 - .idea/runConfigurations/cortex_Client.xml | 22 - .../runConfigurations/cortex_EntityHelper.xml | 22 - .idea/runConfigurations/cortex_JobSrv.xml | 22 - .../runConfigurations/cortex_ResponderSrv.xml | 22 - .../cortex_ServiceHelper.xml | 22 - .../cortex_v0_AnalyzerCtrl.xml | 22 - .idea/runConfigurations/cortex_v0_JobCtrl.xml | 22 - .../cortex_v0_ReportCtrl.xml | 22 - .idea/runConfigurations/misp_Import.xml | 22 - .../scalligraph_Application.xml | 22 - .../scalligraph_Callback.xml | 22 - .../scalligraph_Cardinality.xml | 22 - .../scalligraph_Controller.xml | 22 - .idea/runConfigurations/scalligraph_FPath.xml | 22 - .../runConfigurations/scalligraph_Fields.xml | 22 - .../scalligraph_FieldsParserMacro.xml | 22 - .idea/runConfigurations/scalligraph_Index.xml | 22 - .../runConfigurations/scalligraph_Modern.xml | 22 - .idea/runConfigurations/scalligraph_Query.xml | 22 - .idea/runConfigurations/scalligraph_Retry.xml | 22 - .../scalligraph_SimpleEntity.xml | 22 - .../scalligraph_StorageSrv.xml | 22 - .../scalligraph_UpdateFieldsParserMacro.xml | 22 - .idea/runConfigurations/thehive_AlertSrv.xml | 22 - .idea/runConfigurations/thehive_CaseSrv.xml | 22 - .../thehive_DashboardSrv.xml | 22 - .../runConfigurations/thehive_Functional.xml | 22 - .../thehive_OrganisationSrv.xml | 22 - .idea/runConfigurations/thehive_UserSrv.xml | 22 - .../thehive_notification_template.xml | 22 - .../thehive_v0_AlertCtrl.xml | 22 - .../thehive_v0_AttachmentCtrl.xml | 22 - .../thehive_v0_AttachmentCtrl2.xml | 22 - .../runConfigurations/thehive_v0_CaseCtrl.xml | 22 - .../thehive_v0_CaseTemplateCtrl.xml | 22 - .../thehive_v0_CaseTemplateCtrl2.xml | 22 - .../thehive_v0_CustomFieldCtrl.xml | 22 - .../thehive_v0_DashboardCtrl.xml | 22 - .../runConfigurations/thehive_v0_LogCtrl.xml | 22 - .../runConfigurations/thehive_v0_LogCtrl2.xml | 22 - .../thehive_v0_ObservableCtrl.xml | 22 - .../thehive_v0_ObservableCtrl2.xml | 22 - .../thehive_v0_OrganisationCtrl.xml | 22 - .../thehive_v0_ProfileCtrl.xml | 22 - .idea/runConfigurations/thehive_v0_Query.xml | 22 - .../thehive_v0_ShareCtrl.xml | 22 - .../thehive_v0_StatusCtrl.xml | 22 - .../thehive_v0_StatusCtrl2.xml | 22 - .../runConfigurations/thehive_v0_TagCtrl.xml | 22 - .../runConfigurations/thehive_v0_TaskCtrl.xml | 22 - .../runConfigurations/thehive_v0_UserCtrl.xml | 22 - .../thehive_v0_UserCtrl2.xml | 22 - .../thehive_v1_AlertCtrl.xml | 22 - .../runConfigurations/thehive_v1_CaseCtrl.xml | 22 - .../thehive_v1_OrganisationCtrl.xml | 22 - .../runConfigurations/thehive_v1_UserCtrl.xml | 22 - .scalafmt.conf | 2 +- ScalliGraph | 2 +- conf/logback.xml | 2 +- .../cortex/controllers/v0/ActionCtrl.scala | 45 +- .../controllers/v0/AnalyzerTemplateCtrl.scala | 29 +- .../cortex/controllers/v0/Conversion.scala | 14 +- .../controllers/v0/CortexQueryExecutor.scala | 77 +-- .../cortex/controllers/v0/JobCtrl.scala | 34 +- .../cortex/controllers/v0/JobRenderer.scala | 19 +- .../cortex/controllers/v0/Properties.scala | 46 +- .../connector/cortex/models/Action.scala | 49 +- .../cortex/models/AnalyzerTemplate.scala | 4 +- .../cortex/models/CortexSchema.scala | 4 +- .../thehive/connector/cortex/models/Job.scala | 8 +- .../cortex/services/ActionOperationSrv.scala | 23 +- .../connector/cortex/services/ActionSrv.scala | 143 +++--- .../cortex/services/AnalyzerTemplateSrv.scala | 60 ++- .../cortex/services/CortexAuditSrv.scala | 2 +- .../cortex/services/EntityHelper.scala | 57 ++- .../connector/cortex/services/JobSrv.scala | 214 ++++---- .../cortex/services/ServiceHelper.scala | 12 +- .../connector/cortex/services/package.scala | 13 - .../controllers/v0/AnalyzerCtrlTest.scala | 2 +- .../v0/AnalyzerTemplateCtrlTest.scala | 2 +- .../cortex/controllers/v0/JobCtrlTest.scala | 6 +- .../cortex/services/ActionSrvTest.scala | 18 +- .../cortex/services/AnalyzerSrvTest.scala | 2 +- .../cortex/services/EntityHelperTest.scala | 9 +- .../cortex/services/JobSrvTest.scala | 17 +- .../cortex/services/ResponderSrvTest.scala | 6 +- .../cortex/services/ServiceHelperTest.scala | 8 +- .../scala/org/thp/cortex/dto/v0/Job.scala | 2 +- .../scala/org/thp/thehive/dto/v0/Audit.scala | 2 +- .../scala/org/thp/thehive/dto/v1/Audit.scala | 2 +- .../scala/org/thp/thehive/dto/v1/Case.scala | 2 +- .../thp/thehive/migration/MigrationOps.scala | 2 +- .../org/thp/thehive/migration/Terminal.scala | 4 +- .../thehive/migration/th4/NoAuditSrv.scala | 7 +- .../thp/thehive/migration/th4/Output.scala | 279 +++++------ .../thehive/connector/misp/MispModule.scala | 1 + .../misp/controllers/v0/MispCtrl.scala | 6 +- .../misp/services/MispExportSrv.scala | 10 +- .../misp/services/MispImportSrv.scala | 125 ++--- .../misp/services/TheHiveMispClient.scala | 10 +- .../misp/services/MispImportSrvTest.scala | 11 +- project/Dependencies.scala | 3 +- .../app/org/thp/thehive/ClusterSetup.scala | 4 - .../app/org/thp/thehive/TheHiveModule.scala | 18 +- .../thp/thehive/controllers/dav/Router.scala | 2 +- .../org/thp/thehive/controllers/dav/VFS.scala | 36 +- .../thehive/controllers/v0/AlertCtrl.scala | 117 +++-- .../controllers/v0/AttachmentCtrl.scala | 3 +- .../thehive/controllers/v0/AuditCtrl.scala | 17 +- .../controllers/v0/AuditRenderer.scala | 175 +++---- .../controllers/v0/AuthenticationCtrl.scala | 6 +- .../thp/thehive/controllers/v0/CaseCtrl.scala | 47 +- .../thehive/controllers/v0/CaseRenderer.scala | 108 ++-- .../controllers/v0/CaseTemplateCtrl.scala | 31 +- .../thehive/controllers/v0/Conversion.scala | 81 +-- .../controllers/v0/CustomFieldCtrl.scala | 6 +- .../controllers/v0/DashboardCtrl.scala | 41 +- .../thehive/controllers/v0/DescribeCtrl.scala | 6 +- .../thp/thehive/controllers/v0/ListCtrl.scala | 9 +- .../thp/thehive/controllers/v0/LogCtrl.scala | 26 +- .../controllers/v0/ObservableCtrl.scala | 32 +- .../controllers/v0/ObservableRenderer.scala | 35 +- .../controllers/v0/ObservableTypeCtrl.scala | 22 +- .../controllers/v0/OrganisationCtrl.scala | 34 +- .../thp/thehive/controllers/v0/PageCtrl.scala | 22 +- .../controllers/v0/PermissionCtrl.scala | 1 - .../thehive/controllers/v0/ProfileCtrl.scala | 23 +- .../thehive/controllers/v0/Properties.scala | 443 +++++++++-------- .../thehive/controllers/v0/QueryCtrl.scala | 71 ++- .../thehive/controllers/v0/ShareCtrl.scala | 31 +- .../thehive/controllers/v0/StatsCtrl.scala | 26 +- .../thehive/controllers/v0/StreamCtrl.scala | 3 +- .../thp/thehive/controllers/v0/TagCtrl.scala | 28 +- .../thp/thehive/controllers/v0/TaskCtrl.scala | 33 +- .../controllers/v0/TheHiveQueryExecutor.scala | 133 +++-- .../thp/thehive/controllers/v0/UserCtrl.scala | 78 +-- .../thehive/controllers/v1/AlertCtrl.scala | 48 +- .../controllers/v1/AlertRenderer.scala | 56 ++- .../thehive/controllers/v1/AuditCtrl.scala | 23 +- .../controllers/v1/AuthenticationCtrl.scala | 13 +- .../thp/thehive/controllers/v1/CaseCtrl.scala | 46 +- .../thehive/controllers/v1/CaseRenderer.scala | 126 ++--- .../controllers/v1/CaseTemplateCtrl.scala | 29 +- .../thehive/controllers/v1/Conversion.scala | 70 +-- .../controllers/v1/CustomFieldCtrl.scala | 7 +- .../thehive/controllers/v1/DescribeCtrl.scala | 10 +- .../thp/thehive/controllers/v1/LogCtrl.scala | 30 +- .../thehive/controllers/v1/LogRenderer.scala | 79 +-- .../controllers/v1/ObservableCtrl.scala | 40 +- .../controllers/v1/ObservableRenderer.scala | 113 +++-- .../controllers/v1/OrganisationCtrl.scala | 32 +- .../thehive/controllers/v1/ProfileCtrl.scala | 23 +- .../thehive/controllers/v1/Properties.scala | 245 +++++----- .../thehive/controllers/v1/QueryCtrl.scala | 14 +- .../thp/thehive/controllers/v1/TaskCtrl.scala | 40 +- .../thehive/controllers/v1/TaskRenderer.scala | 92 ++-- .../controllers/v1/TheHiveQueryExecutor.scala | 12 +- .../thp/thehive/controllers/v1/UserCtrl.scala | 82 ++-- .../app/org/thp/thehive/models/Alert.scala | 14 +- .../org/thp/thehive/models/Attachment.scala | 4 +- .../app/org/thp/thehive/models/Audit.scala | 68 +-- thehive/app/org/thp/thehive/models/Case.scala | 24 +- .../org/thp/thehive/models/CaseTemplate.scala | 12 +- .../app/org/thp/thehive/models/Config.scala | 8 +- .../org/thp/thehive/models/CustomField.scala | 45 +- .../org/thp/thehive/models/Dashboard.scala | 8 +- .../app/org/thp/thehive/models/KeyValue.scala | 4 +- thehive/app/org/thp/thehive/models/Log.scala | 6 +- .../org/thp/thehive/models/Observable.scala | 14 +- .../thp/thehive/models/ObservableType.scala | 6 +- .../org/thp/thehive/models/Organisation.scala | 8 +- thehive/app/org/thp/thehive/models/Page.scala | 6 +- .../org/thp/thehive/models/ReportTag.scala | 6 +- thehive/app/org/thp/thehive/models/Role.scala | 10 +- .../thehive/models/SchemaUpdaterActor.scala | 2 +- .../app/org/thp/thehive/models/Share.scala | 12 +- thehive/app/org/thp/thehive/models/Tag.scala | 4 +- thehive/app/org/thp/thehive/models/Task.scala | 6 +- .../models/TheHiveSchemaDefinition.scala | 10 +- thehive/app/org/thp/thehive/models/User.scala | 8 +- .../org/thp/thehive/services/AlertSrv.scala | 397 ++++++++------- .../thp/thehive/services/AttachmentSrv.scala | 28 +- .../org/thp/thehive/services/AuditSrv.scala | 183 ++++--- .../org/thp/thehive/services/CaseSrv.scala | 462 +++++++++--------- .../thehive/services/CaseTemplateSrv.scala | 173 ++++--- .../org/thp/thehive/services/ConfigSrv.scala | 149 +++--- .../thp/thehive/services/CustomFieldSrv.scala | 217 ++++---- .../thp/thehive/services/DashboardSrv.scala | 93 ++-- .../org/thp/thehive/services/DataSrv.scala | 54 +- .../thehive/services/DatabaseWrapper.scala | 214 ++++---- .../org/thp/thehive/services/FlowActor.scala | 29 +- .../thehive/services/ImpactStatusSrv.scala | 38 +- .../thp/thehive/services/KeyValueSrv.scala | 11 +- .../thehive/services/LocalKeyAuthSrv.scala | 17 +- .../services/LocalPasswordAuthSrv.scala | 5 +- .../thp/thehive/services/LocalUserSrv.scala | 13 +- .../app/org/thp/thehive/services/LogSrv.scala | 156 +++--- .../thp/thehive/services/ObservableSrv.scala | 313 ++++++------ .../thehive/services/ObservableTypeSrv.scala | 40 +- .../thehive/services/OrganisationSrv.scala | 153 +++--- .../org/thp/thehive/services/PageSrv.scala | 42 +- .../org/thp/thehive/services/ProfileSrv.scala | 56 +-- .../thp/thehive/services/ReportTagSrv.scala | 26 +- .../services/ResolutionStatusSrv.scala | 40 +- .../org/thp/thehive/services/RoleSrv.scala | 33 +- .../org/thp/thehive/services/ShareSrv.scala | 132 ++--- .../org/thp/thehive/services/StreamSrv.scala | 7 +- .../thp/thehive/services/TOTPAuthSrv.scala | 13 +- .../app/org/thp/thehive/services/TagSrv.scala | 49 +- .../org/thp/thehive/services/TaskSrv.scala | 166 +++---- .../org/thp/thehive/services/UserSrv.scala | 359 +++++++------- .../notification/NotificationActor.scala | 31 +- .../notification/notifiers/AppendToFile.scala | 10 +- .../notification/notifiers/Emailer.scala | 20 +- .../notification/notifiers/Mattermost.scala | 16 +- .../notification/notifiers/Notifier.scala | 2 +- .../notification/notifiers/Template.scala | 36 +- .../notification/notifiers/Webhook.scala | 136 ++++-- .../notification/triggers/CaseCreated.scala | 2 +- .../notification/triggers/FilteredEvent.scala | 2 +- .../notification/triggers/GlobalTrigger.scala | 2 +- .../notification/triggers/LogInMyTask.scala | 8 +- .../notification/triggers/TaskAssigned.scala | 7 +- .../notification/triggers/Trigger.scala | 2 +- .../org/thp/thehive/services/package.scala | 58 +-- .../thehive/services/th3/Aggregation.scala | 398 +++++++++++++++ .../org/thp/thehive/DatabaseBuilder.scala | 8 +- .../test/org/thp/thehive/TestAppBuilder.scala | 20 +- .../controllers/v0/AlertCtrlTest.scala | 4 +- .../controllers/v0/AuditCtrlTest.scala | 8 +- .../thehive/controllers/v0/CaseCtrlTest.scala | 91 ++-- .../controllers/v0/CaseTemplateCtrlTest.scala | 5 +- .../controllers/v0/CustomFieldCtrlTest.scala | 10 +- .../controllers/v0/DashboardCtrlTest.scala | 10 +- .../thehive/controllers/v0/LogCtrlTest.scala | 11 +- .../controllers/v0/ObservableCtrlTest.scala | 5 +- .../controllers/v0/OrganisationCtrlTest.scala | 4 +- .../thehive/controllers/v0/PageCtrlTest.scala | 4 +- .../controllers/v0/ProfileCtrlTest.scala | 4 +- .../thehive/controllers/v0/QueryTest.scala | 16 +- .../controllers/v0/ShareCtrlTest.scala | 7 +- .../thehive/controllers/v0/TagCtrlTest.scala | 4 +- .../thehive/controllers/v0/TaskCtrlTest.scala | 17 +- .../thehive/controllers/v0/UserCtrlTest.scala | 4 +- .../controllers/v1/AlertCtrlTest.scala | 4 +- .../controllers/v1/OrganisationCtrlTest.scala | 4 +- .../thp/thehive/services/AlertSrvTest.scala | 30 +- .../thehive/services/AttachmentSrvTest.scala | 6 +- .../thp/thehive/services/AuditSrvTest.scala | 10 +- .../thp/thehive/services/CaseSrvTest.scala | 65 +-- .../services/CaseTemplateSrvTest.scala | 16 +- .../thehive/services/DashboardSrvTest.scala | 13 +- .../thp/thehive/services/DataSrvTest.scala | 9 +- .../services/ImpactStatusSrvTest.scala | 7 +- .../thp/thehive/services/UserSrvTest.scala | 12 +- .../notifiers/NotificationTemplateTest.scala | 20 +- .../triggers/AlertCreatedTest.scala | 8 +- .../triggers/TaskAssignedTest.scala | 13 +- 265 files changed, 5072 insertions(+), 5999 deletions(-) delete mode 100644 .idea/runConfigurations/Cortex_tests.xml delete mode 100644 .idea/runConfigurations/Misp_tests.xml delete mode 100644 .idea/runConfigurations/Scalligraph_tests.xml delete mode 100644 .idea/runConfigurations/TheHive.xml delete mode 100644 .idea/runConfigurations/TheHive_tests.xml delete mode 100644 .idea/runConfigurations/cortex_ActionSrv.xml delete mode 100644 .idea/runConfigurations/cortex_AnalyzerSrv.xml delete mode 100644 .idea/runConfigurations/cortex_Client.xml delete mode 100644 .idea/runConfigurations/cortex_EntityHelper.xml delete mode 100644 .idea/runConfigurations/cortex_JobSrv.xml delete mode 100644 .idea/runConfigurations/cortex_ResponderSrv.xml delete mode 100644 .idea/runConfigurations/cortex_ServiceHelper.xml delete mode 100644 .idea/runConfigurations/cortex_v0_AnalyzerCtrl.xml delete mode 100644 .idea/runConfigurations/cortex_v0_JobCtrl.xml delete mode 100644 .idea/runConfigurations/cortex_v0_ReportCtrl.xml delete mode 100644 .idea/runConfigurations/misp_Import.xml delete mode 100644 .idea/runConfigurations/scalligraph_Application.xml delete mode 100644 .idea/runConfigurations/scalligraph_Callback.xml delete mode 100644 .idea/runConfigurations/scalligraph_Cardinality.xml delete mode 100644 .idea/runConfigurations/scalligraph_Controller.xml delete mode 100644 .idea/runConfigurations/scalligraph_FPath.xml delete mode 100644 .idea/runConfigurations/scalligraph_Fields.xml delete mode 100644 .idea/runConfigurations/scalligraph_FieldsParserMacro.xml delete mode 100644 .idea/runConfigurations/scalligraph_Index.xml delete mode 100644 .idea/runConfigurations/scalligraph_Modern.xml delete mode 100644 .idea/runConfigurations/scalligraph_Query.xml delete mode 100644 .idea/runConfigurations/scalligraph_Retry.xml delete mode 100644 .idea/runConfigurations/scalligraph_SimpleEntity.xml delete mode 100644 .idea/runConfigurations/scalligraph_StorageSrv.xml delete mode 100644 .idea/runConfigurations/scalligraph_UpdateFieldsParserMacro.xml delete mode 100644 .idea/runConfigurations/thehive_AlertSrv.xml delete mode 100644 .idea/runConfigurations/thehive_CaseSrv.xml delete mode 100644 .idea/runConfigurations/thehive_DashboardSrv.xml delete mode 100644 .idea/runConfigurations/thehive_Functional.xml delete mode 100644 .idea/runConfigurations/thehive_OrganisationSrv.xml delete mode 100644 .idea/runConfigurations/thehive_UserSrv.xml delete mode 100644 .idea/runConfigurations/thehive_notification_template.xml delete mode 100644 .idea/runConfigurations/thehive_v0_AlertCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_AttachmentCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_AttachmentCtrl2.xml delete mode 100644 .idea/runConfigurations/thehive_v0_CaseCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_CaseTemplateCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_CaseTemplateCtrl2.xml delete mode 100644 .idea/runConfigurations/thehive_v0_CustomFieldCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_DashboardCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_LogCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_LogCtrl2.xml delete mode 100644 .idea/runConfigurations/thehive_v0_ObservableCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_ObservableCtrl2.xml delete mode 100644 .idea/runConfigurations/thehive_v0_OrganisationCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_ProfileCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_Query.xml delete mode 100644 .idea/runConfigurations/thehive_v0_ShareCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_StatusCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_StatusCtrl2.xml delete mode 100644 .idea/runConfigurations/thehive_v0_TagCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_TaskCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_UserCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v0_UserCtrl2.xml delete mode 100644 .idea/runConfigurations/thehive_v1_AlertCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v1_CaseCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v1_OrganisationCtrl.xml delete mode 100644 .idea/runConfigurations/thehive_v1_UserCtrl.xml delete mode 100644 cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/package.scala create mode 100644 thehive/app/org/thp/thehive/services/th3/Aggregation.scala diff --git a/.idea/runConfigurations/Cortex_tests.xml b/.idea/runConfigurations/Cortex_tests.xml deleted file mode 100644 index 806a16db24..0000000000 --- a/.idea/runConfigurations/Cortex_tests.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/Misp_tests.xml b/.idea/runConfigurations/Misp_tests.xml deleted file mode 100644 index d068ce0111..0000000000 --- a/.idea/runConfigurations/Misp_tests.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/Scalligraph_tests.xml b/.idea/runConfigurations/Scalligraph_tests.xml deleted file mode 100644 index 30ca06a8da..0000000000 --- a/.idea/runConfigurations/Scalligraph_tests.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/TheHive.xml b/.idea/runConfigurations/TheHive.xml deleted file mode 100644 index d9b3e83524..0000000000 --- a/.idea/runConfigurations/TheHive.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/TheHive_tests.xml b/.idea/runConfigurations/TheHive_tests.xml deleted file mode 100644 index c5796cdc2b..0000000000 --- a/.idea/runConfigurations/TheHive_tests.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_ActionSrv.xml b/.idea/runConfigurations/cortex_ActionSrv.xml deleted file mode 100644 index 6c6c97726e..0000000000 --- a/.idea/runConfigurations/cortex_ActionSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_AnalyzerSrv.xml b/.idea/runConfigurations/cortex_AnalyzerSrv.xml deleted file mode 100644 index 07f9b8036c..0000000000 --- a/.idea/runConfigurations/cortex_AnalyzerSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_Client.xml b/.idea/runConfigurations/cortex_Client.xml deleted file mode 100644 index b4034f8316..0000000000 --- a/.idea/runConfigurations/cortex_Client.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_EntityHelper.xml b/.idea/runConfigurations/cortex_EntityHelper.xml deleted file mode 100644 index e38521624e..0000000000 --- a/.idea/runConfigurations/cortex_EntityHelper.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_JobSrv.xml b/.idea/runConfigurations/cortex_JobSrv.xml deleted file mode 100644 index 43ba5d8ffb..0000000000 --- a/.idea/runConfigurations/cortex_JobSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_ResponderSrv.xml b/.idea/runConfigurations/cortex_ResponderSrv.xml deleted file mode 100644 index 2c0a3f1760..0000000000 --- a/.idea/runConfigurations/cortex_ResponderSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_ServiceHelper.xml b/.idea/runConfigurations/cortex_ServiceHelper.xml deleted file mode 100644 index 0ce6cc7d84..0000000000 --- a/.idea/runConfigurations/cortex_ServiceHelper.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_v0_AnalyzerCtrl.xml b/.idea/runConfigurations/cortex_v0_AnalyzerCtrl.xml deleted file mode 100644 index 62052f98ab..0000000000 --- a/.idea/runConfigurations/cortex_v0_AnalyzerCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_v0_JobCtrl.xml b/.idea/runConfigurations/cortex_v0_JobCtrl.xml deleted file mode 100644 index 8a4cf4076d..0000000000 --- a/.idea/runConfigurations/cortex_v0_JobCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/cortex_v0_ReportCtrl.xml b/.idea/runConfigurations/cortex_v0_ReportCtrl.xml deleted file mode 100644 index 1dccd1353c..0000000000 --- a/.idea/runConfigurations/cortex_v0_ReportCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/misp_Import.xml b/.idea/runConfigurations/misp_Import.xml deleted file mode 100644 index 84518e23d8..0000000000 --- a/.idea/runConfigurations/misp_Import.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Application.xml b/.idea/runConfigurations/scalligraph_Application.xml deleted file mode 100644 index 6d2cc18120..0000000000 --- a/.idea/runConfigurations/scalligraph_Application.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Callback.xml b/.idea/runConfigurations/scalligraph_Callback.xml deleted file mode 100644 index ea3579a3c6..0000000000 --- a/.idea/runConfigurations/scalligraph_Callback.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Cardinality.xml b/.idea/runConfigurations/scalligraph_Cardinality.xml deleted file mode 100644 index cdc076327f..0000000000 --- a/.idea/runConfigurations/scalligraph_Cardinality.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Controller.xml b/.idea/runConfigurations/scalligraph_Controller.xml deleted file mode 100644 index 98f53366da..0000000000 --- a/.idea/runConfigurations/scalligraph_Controller.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_FPath.xml b/.idea/runConfigurations/scalligraph_FPath.xml deleted file mode 100644 index da23a95012..0000000000 --- a/.idea/runConfigurations/scalligraph_FPath.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Fields.xml b/.idea/runConfigurations/scalligraph_Fields.xml deleted file mode 100644 index 4fd0933557..0000000000 --- a/.idea/runConfigurations/scalligraph_Fields.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_FieldsParserMacro.xml b/.idea/runConfigurations/scalligraph_FieldsParserMacro.xml deleted file mode 100644 index 8ca5bb5846..0000000000 --- a/.idea/runConfigurations/scalligraph_FieldsParserMacro.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Index.xml b/.idea/runConfigurations/scalligraph_Index.xml deleted file mode 100644 index ba49456ac1..0000000000 --- a/.idea/runConfigurations/scalligraph_Index.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Modern.xml b/.idea/runConfigurations/scalligraph_Modern.xml deleted file mode 100644 index 0c983e8481..0000000000 --- a/.idea/runConfigurations/scalligraph_Modern.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Query.xml b/.idea/runConfigurations/scalligraph_Query.xml deleted file mode 100644 index 94954378dc..0000000000 --- a/.idea/runConfigurations/scalligraph_Query.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_Retry.xml b/.idea/runConfigurations/scalligraph_Retry.xml deleted file mode 100644 index 24fb112a83..0000000000 --- a/.idea/runConfigurations/scalligraph_Retry.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_SimpleEntity.xml b/.idea/runConfigurations/scalligraph_SimpleEntity.xml deleted file mode 100644 index 484b111da9..0000000000 --- a/.idea/runConfigurations/scalligraph_SimpleEntity.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_StorageSrv.xml b/.idea/runConfigurations/scalligraph_StorageSrv.xml deleted file mode 100644 index cee02ae686..0000000000 --- a/.idea/runConfigurations/scalligraph_StorageSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/scalligraph_UpdateFieldsParserMacro.xml b/.idea/runConfigurations/scalligraph_UpdateFieldsParserMacro.xml deleted file mode 100644 index 460ba8cc45..0000000000 --- a/.idea/runConfigurations/scalligraph_UpdateFieldsParserMacro.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_AlertSrv.xml b/.idea/runConfigurations/thehive_AlertSrv.xml deleted file mode 100644 index f9ce3e0601..0000000000 --- a/.idea/runConfigurations/thehive_AlertSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_CaseSrv.xml b/.idea/runConfigurations/thehive_CaseSrv.xml deleted file mode 100644 index 9a368c5986..0000000000 --- a/.idea/runConfigurations/thehive_CaseSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_DashboardSrv.xml b/.idea/runConfigurations/thehive_DashboardSrv.xml deleted file mode 100644 index d9c5acf444..0000000000 --- a/.idea/runConfigurations/thehive_DashboardSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_Functional.xml b/.idea/runConfigurations/thehive_Functional.xml deleted file mode 100644 index f641683454..0000000000 --- a/.idea/runConfigurations/thehive_Functional.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_OrganisationSrv.xml b/.idea/runConfigurations/thehive_OrganisationSrv.xml deleted file mode 100644 index 523c373a1b..0000000000 --- a/.idea/runConfigurations/thehive_OrganisationSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_UserSrv.xml b/.idea/runConfigurations/thehive_UserSrv.xml deleted file mode 100644 index 6cf8cfa4ef..0000000000 --- a/.idea/runConfigurations/thehive_UserSrv.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_notification_template.xml b/.idea/runConfigurations/thehive_notification_template.xml deleted file mode 100644 index cb4798d853..0000000000 --- a/.idea/runConfigurations/thehive_notification_template.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_AlertCtrl.xml b/.idea/runConfigurations/thehive_v0_AlertCtrl.xml deleted file mode 100644 index 05b84b8bb3..0000000000 --- a/.idea/runConfigurations/thehive_v0_AlertCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_AttachmentCtrl.xml b/.idea/runConfigurations/thehive_v0_AttachmentCtrl.xml deleted file mode 100644 index 9f399f4a57..0000000000 --- a/.idea/runConfigurations/thehive_v0_AttachmentCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_AttachmentCtrl2.xml b/.idea/runConfigurations/thehive_v0_AttachmentCtrl2.xml deleted file mode 100644 index 9f399f4a57..0000000000 --- a/.idea/runConfigurations/thehive_v0_AttachmentCtrl2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_CaseCtrl.xml b/.idea/runConfigurations/thehive_v0_CaseCtrl.xml deleted file mode 100644 index 39b18c0567..0000000000 --- a/.idea/runConfigurations/thehive_v0_CaseCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_CaseTemplateCtrl.xml b/.idea/runConfigurations/thehive_v0_CaseTemplateCtrl.xml deleted file mode 100644 index 0939f2b322..0000000000 --- a/.idea/runConfigurations/thehive_v0_CaseTemplateCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_CaseTemplateCtrl2.xml b/.idea/runConfigurations/thehive_v0_CaseTemplateCtrl2.xml deleted file mode 100644 index 0939f2b322..0000000000 --- a/.idea/runConfigurations/thehive_v0_CaseTemplateCtrl2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_CustomFieldCtrl.xml b/.idea/runConfigurations/thehive_v0_CustomFieldCtrl.xml deleted file mode 100644 index d72ff5659f..0000000000 --- a/.idea/runConfigurations/thehive_v0_CustomFieldCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_DashboardCtrl.xml b/.idea/runConfigurations/thehive_v0_DashboardCtrl.xml deleted file mode 100644 index 2bc75a1d0d..0000000000 --- a/.idea/runConfigurations/thehive_v0_DashboardCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_LogCtrl.xml b/.idea/runConfigurations/thehive_v0_LogCtrl.xml deleted file mode 100644 index 3f09c665bf..0000000000 --- a/.idea/runConfigurations/thehive_v0_LogCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_LogCtrl2.xml b/.idea/runConfigurations/thehive_v0_LogCtrl2.xml deleted file mode 100644 index 3f09c665bf..0000000000 --- a/.idea/runConfigurations/thehive_v0_LogCtrl2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_ObservableCtrl.xml b/.idea/runConfigurations/thehive_v0_ObservableCtrl.xml deleted file mode 100644 index af9b51346f..0000000000 --- a/.idea/runConfigurations/thehive_v0_ObservableCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_ObservableCtrl2.xml b/.idea/runConfigurations/thehive_v0_ObservableCtrl2.xml deleted file mode 100644 index af9b51346f..0000000000 --- a/.idea/runConfigurations/thehive_v0_ObservableCtrl2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_OrganisationCtrl.xml b/.idea/runConfigurations/thehive_v0_OrganisationCtrl.xml deleted file mode 100644 index c083f0190f..0000000000 --- a/.idea/runConfigurations/thehive_v0_OrganisationCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_ProfileCtrl.xml b/.idea/runConfigurations/thehive_v0_ProfileCtrl.xml deleted file mode 100644 index f9fa89e9c9..0000000000 --- a/.idea/runConfigurations/thehive_v0_ProfileCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_Query.xml b/.idea/runConfigurations/thehive_v0_Query.xml deleted file mode 100644 index 6dd28e31a5..0000000000 --- a/.idea/runConfigurations/thehive_v0_Query.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_ShareCtrl.xml b/.idea/runConfigurations/thehive_v0_ShareCtrl.xml deleted file mode 100644 index 897700d845..0000000000 --- a/.idea/runConfigurations/thehive_v0_ShareCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_StatusCtrl.xml b/.idea/runConfigurations/thehive_v0_StatusCtrl.xml deleted file mode 100644 index 18a2d43738..0000000000 --- a/.idea/runConfigurations/thehive_v0_StatusCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_StatusCtrl2.xml b/.idea/runConfigurations/thehive_v0_StatusCtrl2.xml deleted file mode 100644 index 18a2d43738..0000000000 --- a/.idea/runConfigurations/thehive_v0_StatusCtrl2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_TagCtrl.xml b/.idea/runConfigurations/thehive_v0_TagCtrl.xml deleted file mode 100644 index 38138bd877..0000000000 --- a/.idea/runConfigurations/thehive_v0_TagCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_TaskCtrl.xml b/.idea/runConfigurations/thehive_v0_TaskCtrl.xml deleted file mode 100644 index c608f7613c..0000000000 --- a/.idea/runConfigurations/thehive_v0_TaskCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_UserCtrl.xml b/.idea/runConfigurations/thehive_v0_UserCtrl.xml deleted file mode 100644 index a5089e7665..0000000000 --- a/.idea/runConfigurations/thehive_v0_UserCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v0_UserCtrl2.xml b/.idea/runConfigurations/thehive_v0_UserCtrl2.xml deleted file mode 100644 index a5089e7665..0000000000 --- a/.idea/runConfigurations/thehive_v0_UserCtrl2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v1_AlertCtrl.xml b/.idea/runConfigurations/thehive_v1_AlertCtrl.xml deleted file mode 100644 index 60cae05e76..0000000000 --- a/.idea/runConfigurations/thehive_v1_AlertCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v1_CaseCtrl.xml b/.idea/runConfigurations/thehive_v1_CaseCtrl.xml deleted file mode 100644 index 56bbac7ba4..0000000000 --- a/.idea/runConfigurations/thehive_v1_CaseCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v1_OrganisationCtrl.xml b/.idea/runConfigurations/thehive_v1_OrganisationCtrl.xml deleted file mode 100644 index f6aab95437..0000000000 --- a/.idea/runConfigurations/thehive_v1_OrganisationCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/thehive_v1_UserCtrl.xml b/.idea/runConfigurations/thehive_v1_UserCtrl.xml deleted file mode 100644 index bab9c7c972..0000000000 --- a/.idea/runConfigurations/thehive_v1_UserCtrl.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.scalafmt.conf b/.scalafmt.conf index 3843d59dbf..97286da352 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = 2.3.2 +version = 2.6.4 project.git = true align = more # For pretty alignment. assumeStandardLibraryStripMargin = true diff --git a/ScalliGraph b/ScalliGraph index 36b47d6920..7eafad1cee 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 36b47d6920d09ceeb721b36e3c6e907b3a293578 +Subproject commit 7eafad1cee2f24fd9ad8402a307ccbaff8f36d11 diff --git a/conf/logback.xml b/conf/logback.xml index 6a19ad03ed..52ba1c47b9 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -35,7 +35,7 @@ - +
- #{{item.case.caseId}} - {{item.case.title}} + #{{item.case.number}} - {{item.case.title}}
From a92077329fef3168c0565cb53ac8e044b14b4238 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 8 Sep 2020 17:46:02 +0200 Subject: [PATCH 031/237] #1517 Don't split multiline observable --- dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala | 2 +- dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala index bc23cc2687..c63f0b01de 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala @@ -24,7 +24,7 @@ object InputObservable { implicit val writes: Writes[InputObservable] = Json.writes[InputObservable] val fp: FieldsParser[Seq[String]] = FieldsParser[Seq[String]]("data") { - case (_, FString(s)) => Good(s.split("\\R+").toSeq) + case (_, FString(s)) => Good(Seq(s)) case (_, FAny(s)) => Good(s) case (_, FSeq(a)) => a.validatedBy(FieldsParser.string(_)) case (_, FUndefined) => Good(Nil) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala index cac1608986..e94bcf09c8 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala @@ -24,7 +24,7 @@ object InputObservable { implicit val writes: Writes[InputObservable] = Json.writes[InputObservable] val fp: FieldsParser[Seq[String]] = FieldsParser[Seq[String]]("data") { - case (_, FString(s)) => Good(s.split("\\R+").toSeq) + case (_, FString(s)) => Good(Seq(s)) case (_, FAny(s)) => Good(s) case (_, FSeq(a)) => a.validatedBy(FieldsParser.string(_)) case (_, FUndefined) => Good(Nil) From de9b134a02e2171bcfca6a27ad40b4702fe4ec56 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 8 Sep 2020 18:00:39 +0200 Subject: [PATCH 032/237] #1517 Fix tests --- .../org/thp/thehive/controllers/v0/ObservableCtrlTest.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala index 9e09bce672..308f9e65cd 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala @@ -54,7 +54,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "tlp":2, "message":"love exciting and new", "tags":["tagfile"], - "data":"multi\nline\ntest" + "data":["multi","line","test"] } """.stripMargin)) val result = app[ObservableCtrl].create("#1")(request) @@ -151,7 +151,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "tlp":2, "message":"localhost", "tags":["local", "host"], - "data":"127.0.0.1\n127.0.0.2" + "data":["127.0.0.1","127.0.0.2"] } """)) val request = FakeRequest( From 4a3af442757e5c7954ddd475d153ca8118a9ca6d Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 16 Sep 2020 20:22:09 +0200 Subject: [PATCH 033/237] initial commit sample configuration for a test envifonment --- docker/README.md | 51 ++++++++ docker/cortex/application.conf | 217 ++++++++++++++++++++++++++++++++ docker/docker-compose.yml | 103 +++++++++++++++ docker/thehive/application.conf | 78 ++++++++++++ 4 files changed, 449 insertions(+) create mode 100644 docker/README.md create mode 100644 docker/cortex/application.conf create mode 100644 docker/docker-compose.yml create mode 100644 docker/thehive/application.conf diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..d1a1f88ed8 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,51 @@ +## Example of docker-compose (not for production) +With this docker-compose.yml you will be able to run the following images: +- The Hive 4 +- Cassandra 3.11 +- Cortex 3.1.0-0.2RC1 +- Elasticsearch 7.9.0 +- Kibana 7.9.0 +- MISP 2.4.131 +- Mysql 8.0.21 +- Redis 6.0.8 + +## Some Hint +### Cortex-Analyzers +- In order to use Analyzers in docker version, set the application.conf of Cortex the online json url instead absolute path of analyzers: + https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json +- In order to use Analyzers in docker version set the application.conf thejob: + job { + runner = [docker] +} +- The analyzer in docker version will need to download from internet images, so have to add in "/etc/default/docker" + DOCKER_OPTS="--dns 8.8.8.8 --dns 1.1.1.1" +- When Cortex launches an analyzer need to pass the object to being analyzed, so need share /tmp folder +- When Cortex launches an analyzer it uses docker.sock, have to map in compose + /var/run/docker.sock:/var/run/docker.sock +- Have to change permission on /var/run/docker.sock in order to let use socket by cortex docker and cortex-analyzers docker +- First time an analyzer/responder is executed, it will take a while because docker image is being downloaded on the fly, from second run of analyzer/responder it runs normally + sudo chmod 666 /var/run/docker.sock + +### The Hive +- In order to let The Hive reads the external application.conf and configure Cortex had to pass in command of docker compose the following option: + --no-config +- In order to let The Hive reads the external application.conf and configure MISP for receive alerts had to pass in command of docker compose the following option: + --no-config-secret +- Default credentials: admin@thehive.local // secret +- The cortex key in application.conf must be generated in Cortex +- MISP connection is https, in order to skip the verify of self signed certificate have do add this setting in the hive application.conf under MISP section: + wsConfig { ssl { loose { acceptAnyCertificate: true } } } + + +### MISP + +- login with default credentials: admin@admin.test // admin +- request change password +- go in Automation page and grab the api key to use in the hive application.conf to receive alerts from MISP or to use in MISP analyzers inside Cortex. + +### Cortex +- login page on 9001 port, then click "update database" and create superadmin +- as superadmin create org and other user (remember to set password) and create apikey to use in the hive application.conf + + + diff --git a/docker/cortex/application.conf b/docker/cortex/application.conf new file mode 100644 index 0000000000..0fe0c01b63 --- /dev/null +++ b/docker/cortex/application.conf @@ -0,0 +1,217 @@ +# Sample Cortex application.conf file + +## SECRET KEY +# +# The secret key is used to secure cryptographic functions. +# +# IMPORTANT: If you deploy your application to several instances, make +# sure to use the same key. +play.http.secret.key="msd3232fdn3ofgfbki83ihtzHSD" + +## ElasticSearch +search { + # Name of the index + index = cortex + # ElasticSearch instance address. + # For cluster, join address:port with ',': "http://ip1:9200,ip2:9200,ip3:9200" + uri = "http://elasticsearch:9200" + + ## Advanced configuration + # Scroll keepalive. + #keepalive = 1m + # Scroll page size. + #pagesize = 50 + # Number of shards + #nbshards = 5 + # Number of replicas + #nbreplicas = 1 + # Arbitrary settings + #settings { + # # Maximum number of nested fields + # mapping.nested_fields.limit = 100 + #} + + ## Authentication configuration + #search.username = "" + #search.password = "" + + ## SSL configuration + #search.keyStore { + # path = "/path/to/keystore" + # type = "JKS" # or PKCS12 + # password = "keystore-password" + #} + #search.trustStore { + # path = "/path/to/trustStore" + # type = "JKS" # or PKCS12 + # password = "trustStore-password" + #} +} + +## Cache +# +# If an analyzer is executed against the same observable, the previous report can be returned without re-executing the +# analyzer. The cache is used only if the second job occurs within cache.job (the default is 10 minutes). +cache.job = 10 minutes + +## Authentication +auth { + # "provider" parameter contains the authentication provider(s). It can be multi-valued, which is useful + # for migration. + # The available auth types are: + # - services.LocalAuthSrv : passwords are stored in the user entity within ElasticSearch). No + # configuration are required. + # - ad : use ActiveDirectory to authenticate users. The associated configuration shall be done in + # the "ad" section below. + # - ldap : use LDAP to authenticate users. The associated configuration shall be done in the + # "ldap" section below. + # - oauth2 : use OAuth/OIDC to authenticate users. Configuration is under "auth.oauth2" and "auth.sso" keys + provider = [local] + + ad { + # The Windows domain name in DNS format. This parameter is required if you do not use + # 'serverNames' below. + #domainFQDN = "mydomain.local" + + # Optionally you can specify the host names of the domain controllers instead of using 'domainFQDN + # above. If this parameter is not set, TheHive uses 'domainFQDN'. + #serverNames = [ad1.mydomain.local, ad2.mydomain.local] + + # The Windows domain name using short format. This parameter is required. + #domainName = "MYDOMAIN" + + # If 'true', use SSL to connect to the domain controller. + #useSSL = true + } + + ldap { + # The LDAP server name or address. The port can be specified using the 'host:port' + # syntax. This parameter is required if you don't use 'serverNames' below. + #serverName = "ldap.mydomain.local:389" + + # If you have multiple LDAP servers, use the multi-valued setting 'serverNames' instead. + #serverNames = [ldap1.mydomain.local, ldap2.mydomain.local] + + # Account to use to bind to the LDAP server. This parameter is required. + #bindDN = "cn=thehive,ou=services,dc=mydomain,dc=local" + + # Password of the binding account. This parameter is required. + #bindPW = "***secret*password***" + + # Base DN to search users. This parameter is required. + #baseDN = "ou=users,dc=mydomain,dc=local" + + # Filter to search user in the directory server. Please note that {0} is replaced + # by the actual user name. This parameter is required. + #filter = "(cn={0})" + + # If 'true', use SSL to connect to the LDAP directory server. + #useSSL = true + } + + oauth2 { + # URL of the authorization server + #clientId = "client-id" + #clientSecret = "client-secret" + #redirectUri = "https://my-thehive-instance.example/index.html#!/login" + #responseType = "code" + #grantType = "authorization_code" + + # URL from where to get the access token + #authorizationUrl = "https://auth-site.com/OAuth/Authorize" + #tokenUrl = "https://auth-site.com/OAuth/Token" + + # The endpoint from which to obtain user details using the OAuth token, after successful login + #userUrl = "https://auth-site.com/api/User" + #scope = "openid profile" + # Type of authorization header + #authorizationHeader = "Bearer" # or token + } + + # Single-Sign On + sso { + # Autocreate user in database? + #autocreate = false + + # Autoupdate its profile and roles? + #autoupdate = false + + # Autologin user using SSO? + #autologin = false + + # Attributes mappings + #attributes { + # login = "login" + # name = "name" + # groups = "groups" + # roles = "roles" # list of roles, separated with comma + # organisation = "org" + #} + + # Name of mapping class from user resource to backend user ('simple' or 'group') + #mapper = group + # Default roles for users with no groups mapped ("read", "analyze", "orgadmin") + #defaultRoles = [] + # Default organization + #defaultOrganization = "MyOrga" + + #groups { + # # URL to retreive groups (leave empty if you are using OIDC) + # #url = "https://auth-site.com/api/Groups" + # # Group mappings, you can have multiple roles for each group: they are merged + # mappings { + # admin-profile-name = ["admin"] + # editor-profile-name = ["write"] + # reader-profile-name = ["read"] + # } + #} + } +} + +job { + runner = [docker] +} +## ANALYZERS +# +analyzer { + # analyzer location + # url can be point to: + # - directory where analyzers are installed + # - json file containing the list of analyzer descriptions + urls = [ + "https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json" + #"/absolute/path/of/analyzers" + ] + + # Sane defaults. Do not change unless you know what you are doing. + fork-join-executor { + # Min number of threads available for analysis. + parallelism-min = 2 + # Parallelism (threads) ... ceil(available processors * factor). + parallelism-factor = 2.0 + # Max number of threads available for analysis. + parallelism-max = 4 + } +} + +# RESPONDERS +# +responder { + # responder location (same format as analyzer.urls) + urls = [ + "https://dl.bintray.com/thehive-project/cortexneurons/responders.json" + #"/absolute/path/of/responders" + ] + + # Sane defaults. Do not change unless you know what you are doing. + fork-join-executor { + # Min number of threads available for analysis. + parallelism-min = 2 + # Parallelism (threads) ... ceil(available processors * factor). + parallelism-factor = 2.0 + # Max number of threads available for analysis. + parallelism-max = 4 + } +} + +# It's the end my friend. Happy hunting! diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000000..6f1553042a --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,103 @@ +version: "3.8" +services: + elasticsearch: + image: 'elasticsearch:7.9.0' + container_name: elasticsearch + restart: unless-stopped + ports: + - '0.0.0.0:9200:9200' + environment: + - http.host=0.0.0.0 + - discovery.type=single-node + - cluster.name=hive + - script.allowed_types= inline + - thread_pool.search.queue_size=100000 + - thread_pool.write.queue_size=10000 + - gateway.recover_after_nodes=1 + - xpack.security.enabled=false + ulimits: + nofile: + soft: 65536 + hard: 65536 + volumes: + - ./elasticsearch/data:/usr/share/elasticsearch/data + - ./elasticsearch/logs:/usr/share/elasticsearch/logs + kibana: + image: 'docker.elastic.co/kibana/kibana:7.9.0' + container_name: kibana + restart: unless-stopped + depends_on: + - elasticsearch + ports: + - '5601:5601' + cortex: + image: 'thehiveproject/cortex:3.1.0-0.2RC1' + container_name: cortex + restart: unless-stopped + volumes: + - ./cortex/application.conf:/etc/cortex/application.conf + - /var/run/docker.sock:/var/run/docker.sock + - /tmp:/tmp + depends_on: + - elasticsearch + ports: + - '0.0.0.0:9001:9001' + + cassandra: + image: cassandra:3.11 + container_name: cassandra + restart: unless-stopped + hostname: cassandra + environment: + - CASSANDRA_CLUSTER_NAME=thp + volumes: + - ./cassandra-data:/var/lib/cassandra/data + + thehive: + image: 'thehiveproject/thehive4:latest' + container_name: thehive + restart: unless-stopped + depends_on: + - cassandra + ports: + - '0.0.0.0:9000:9000' + volumes: + - ./thehive/application.conf:/etc/thehive/application.conf + - ./data:/data + command: '--no-config --no-config-secret' + + redis: + image: redis:latest + container_name: redis + restart: unless-stopped + + mysql: + image: mysql:latest + container_name: mysql + restart: unless-stopped + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + - "MYSQL_USER=misp" + - "MYSQL_PASSWORD=example" + - "MYSQL_ROOT_PASSWORD=password" + - "MYSQL_DATABASE=misp" + volumes: + - ./mysql:/var/lib/mysql + misp: + image: coolacid/misp-docker:core-latest + container_name: misp + restart: unless-stopped + depends_on: + - redis + - mysql + ports: + - "80:80" + - "443:443" + environment: + - "HOSTNAME=http://misp" + - "REDIS_FQDN=redis" + - "INIT=true" # Initialze MISP, things includes, attempting to import SQL and the Files DIR + - "CRON_USER_ID=1" # The MISP user ID to run cron jobs as + - "DISIPV6=true" # Disable IPV6 in nginx + \ No newline at end of file diff --git a/docker/thehive/application.conf b/docker/thehive/application.conf new file mode 100644 index 0000000000..793d661be0 --- /dev/null +++ b/docker/thehive/application.conf @@ -0,0 +1,78 @@ +play.http.secret.key="t5EeDXh2dEtJxohh" + +# JanusGraph +db { + provider: janusgraph + janusgraph { + storage { + backend: cql + hostname: ["cassandra"] + + cql { + cluster-name: thp # cluster name + keyspace: thehive # name of the keyspace + read-consistency-level: ONE + write-consistency-level: ONE + } + } + } +} + +storage { + provider: localfs + localfs.directory: /opt/data +} + +play.modules.enabled += org.thp.thehive.connector.cortex.CortexModule +cortex { + servers = [ + { + name = local + url = "http://cortex:9001" + auth { + type = "bearer" + key = "JmjjnBDuLL2WgJBsF00vmxTdWTqMj0Jw" + } + # HTTP client configuration (SSL and proxy) + # wsConfig {} + # List TheHive organisation which can use this Cortex server. All ("*") by default + # includedTheHiveOrganisations = ["*"] + # List TheHive organisation which cannot use this Cortex server. None by default + # excludedTheHiveOrganisations = [] + } + ] + # Check job update time intervalcortex + refreshDelay = 5 seconds + # Maximum number of successive errors before give up + maxRetryOnError = 3 + # Check remote Cortex status time interval + statusCheckInterval = 1 minute +} +# MISP configuration +play.modules.enabled += org.thp.thehive.connector.misp.MispModule +misp { + interval: 5 min + servers: [ + { + name = "MISP THP" # MISP name + url = "https://misp/" # URL or MISP + auth { + type = key + key = "s7wSDr0I78WD8ImMpS2P8sX9Iy9N4Jiboz3pdWtm" # MISP API key + } + wsConfig { ssl { loose { acceptAnyCertificate: true } } } + } + ] +} + + +notification.webhook.endpoints = [ + { + name: local + url: "http://thehive:5000/" + version: 0 + wsConfig: {} + includedTheHiveOrganisations: [] + excludedTheHiveOrganisations: [] + } +] From f598a4bfc567068fb6e4ba5a5aa97dab74e400f9 Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 16 Sep 2020 20:57:02 +0200 Subject: [PATCH 034/237] Update docker-compose.yml --- docker/docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 6f1553042a..d2775b6241 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -71,9 +71,8 @@ services: container_name: redis restart: unless-stopped - mysql: + db: image: mysql:latest - container_name: mysql restart: unless-stopped command: --default-authentication-plugin=mysql_native_password restart: always @@ -90,7 +89,7 @@ services: restart: unless-stopped depends_on: - redis - - mysql + - db ports: - "80:80" - "443:443" @@ -100,4 +99,4 @@ services: - "INIT=true" # Initialze MISP, things includes, attempting to import SQL and the Files DIR - "CRON_USER_ID=1" # The MISP user ID to run cron jobs as - "DISIPV6=true" # Disable IPV6 in nginx - \ No newline at end of file + From 41e265472ca6838a1b4c795a838d832caf3598d8 Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 16 Sep 2020 21:14:23 +0200 Subject: [PATCH 035/237] Update README.md --- docker/README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docker/README.md b/docker/README.md index d1a1f88ed8..825470b4dc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -13,28 +13,31 @@ With this docker-compose.yml you will be able to run the following images: ### Cortex-Analyzers - In order to use Analyzers in docker version, set the application.conf of Cortex the online json url instead absolute path of analyzers: https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json -- In order to use Analyzers in docker version set the application.conf thejob: +- In order to use Analyzers in docker version set the application.conf thejob: ``` job { runner = [docker] -} +} ``` - The analyzer in docker version will need to download from internet images, so have to add in "/etc/default/docker" - DOCKER_OPTS="--dns 8.8.8.8 --dns 1.1.1.1" + ``` DOCKER_OPTS="--dns 8.8.8.8 --dns 1.1.1.1" ``` - When Cortex launches an analyzer need to pass the object to being analyzed, so need share /tmp folder - When Cortex launches an analyzer it uses docker.sock, have to map in compose - /var/run/docker.sock:/var/run/docker.sock + ``` /var/run/docker.sock:/var/run/docker.sock ``` - Have to change permission on /var/run/docker.sock in order to let use socket by cortex docker and cortex-analyzers docker + ```sudo chmod 666 /var/run/docker.sock``` - First time an analyzer/responder is executed, it will take a while because docker image is being downloaded on the fly, from second run of analyzer/responder it runs normally - sudo chmod 666 /var/run/docker.sock +- In order to let use socket both cortex image and analyzers images had to do + + ### The Hive - In order to let The Hive reads the external application.conf and configure Cortex had to pass in command of docker compose the following option: --no-config - In order to let The Hive reads the external application.conf and configure MISP for receive alerts had to pass in command of docker compose the following option: - --no-config-secret + ``` --no-config-secret ``` - Default credentials: admin@thehive.local // secret - The cortex key in application.conf must be generated in Cortex - MISP connection is https, in order to skip the verify of self signed certificate have do add this setting in the hive application.conf under MISP section: - wsConfig { ssl { loose { acceptAnyCertificate: true } } } + ``` wsConfig { ssl { loose { acceptAnyCertificate: true } } } ``` ### MISP From 636541de746a3f8f6d13a0944e94849753c5338a Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 17 Sep 2020 09:44:56 +0200 Subject: [PATCH 036/237] Update README.md --- docker/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker/README.md b/docker/README.md index 825470b4dc..360a2698ac 100644 --- a/docker/README.md +++ b/docker/README.md @@ -10,6 +10,16 @@ With this docker-compose.yml you will be able to run the following images: - Redis 6.0.8 ## Some Hint + +### Mapping volumes +If you take a look of docker-compose.yml you will see you need some local folder that needs to be mapped, so before do docker-compose up, ensure folders (and config files) exist: +- ./elasticsearch/data:/usr/share/elasticsearch/data +- ./elasticsearch/logs:/usr/share/elasticsearch/logs +- ./cortex/application.conf:/etc/cortex/application.conf +- ./thehive/application.conf:/etc/thehive/application.conf +- ./mysql:/var/lib/mysql + + ### Cortex-Analyzers - In order to use Analyzers in docker version, set the application.conf of Cortex the online json url instead absolute path of analyzers: https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json From 138139a55796f26386a1bf16e108f2da366ebd0f Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 17 Sep 2020 09:55:07 +0200 Subject: [PATCH 037/237] Update README.md --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 360a2698ac..b3c842cf8c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -21,7 +21,7 @@ If you take a look of docker-compose.yml you will see you need some local folder ### Cortex-Analyzers -- In order to use Analyzers in docker version, set the application.conf of Cortex the online json url instead absolute path of analyzers: +- In order to use Analyzers in docker version, set the online json url instead absolute path of analyzers in the application.conf of Cortex: https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json - In order to use Analyzers in docker version set the application.conf thejob: ``` job { From 9bd45e4ce06962edbe43d9b30f45ea8e9ac91514 Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 17 Sep 2020 09:56:53 +0200 Subject: [PATCH 038/237] Update README.md --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index b3c842cf8c..11d47431d1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -21,9 +21,9 @@ If you take a look of docker-compose.yml you will see you need some local folder ### Cortex-Analyzers -- In order to use Analyzers in docker version, set the online json url instead absolute path of analyzers in the application.conf of Cortex: +- In order to use Analyzers in docker version, it is set the online json url instead absolute path of analyzers in the application.conf of Cortex: https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json -- In order to use Analyzers in docker version set the application.conf thejob: ``` +- In order to use Analyzers in docker version it is set the application.conf thejob: ``` job { runner = [docker] } ``` From 41cfed80148cd4a82c9f90d17a6da105d1ffd4d5 Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 17 Sep 2020 10:04:33 +0200 Subject: [PATCH 039/237] Update README.md --- docker/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index 11d47431d1..f7f55349b8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -37,7 +37,9 @@ If you take a look of docker-compose.yml you will see you need some local folder - First time an analyzer/responder is executed, it will take a while because docker image is being downloaded on the fly, from second run of analyzer/responder it runs normally - In order to let use socket both cortex image and analyzers images had to do - +### Cortex +- login page on 9001 port, then click "update database" and create superadmin +- as superadmin create org and other user (remember to set password) and create apikey to use for connect with the hive ### The Hive - In order to let The Hive reads the external application.conf and configure Cortex had to pass in command of docker compose the following option: @@ -45,7 +47,7 @@ If you take a look of docker-compose.yml you will see you need some local folder - In order to let The Hive reads the external application.conf and configure MISP for receive alerts had to pass in command of docker compose the following option: ``` --no-config-secret ``` - Default credentials: admin@thehive.local // secret -- The cortex key in application.conf must be generated in Cortex +- In order to connect The Hive with cortex take the cortex key generated in Cortex and set it in thehive/application.conf - MISP connection is https, in order to skip the verify of self signed certificate have do add this setting in the hive application.conf under MISP section: ``` wsConfig { ssl { loose { acceptAnyCertificate: true } } } ``` @@ -56,9 +58,6 @@ If you take a look of docker-compose.yml you will see you need some local folder - request change password - go in Automation page and grab the api key to use in the hive application.conf to receive alerts from MISP or to use in MISP analyzers inside Cortex. -### Cortex -- login page on 9001 port, then click "update database" and create superadmin -- as superadmin create org and other user (remember to set password) and create apikey to use in the hive application.conf From 1cc1909e7e29c81005993dc52740a48abb250a01 Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 17 Sep 2020 11:35:04 +0200 Subject: [PATCH 040/237] Update README.md --- docker/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index f7f55349b8..fd8c418d9d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -35,7 +35,6 @@ If you take a look of docker-compose.yml you will see you need some local folder - Have to change permission on /var/run/docker.sock in order to let use socket by cortex docker and cortex-analyzers docker ```sudo chmod 666 /var/run/docker.sock``` - First time an analyzer/responder is executed, it will take a while because docker image is being downloaded on the fly, from second run of analyzer/responder it runs normally -- In order to let use socket both cortex image and analyzers images had to do ### Cortex - login page on 9001 port, then click "update database" and create superadmin From c310be81d1ba2efd64dbb4ce3df5cd28865b8f4b Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 18 Sep 2020 14:44:47 +0200 Subject: [PATCH 041/237] Update README.md --- docker/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker/README.md b/docker/README.md index fd8c418d9d..3540ff439c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,8 +17,22 @@ If you take a look of docker-compose.yml you will see you need some local folder - ./elasticsearch/logs:/usr/share/elasticsearch/logs - ./cortex/application.conf:/etc/cortex/application.conf - ./thehive/application.conf:/etc/thehive/application.conf +- ./data:/data - ./mysql:/var/lib/mysql +Structure would look like: +``` +├── docker-compose.yml +├── elasticsearch +│ └── data +│ └── logs +├── cortex +│ └── application.conf +└── thehive + └── application.conf +└── data +└── mysql +``` ### Cortex-Analyzers - In order to use Analyzers in docker version, it is set the online json url instead absolute path of analyzers in the application.conf of Cortex: From 7bed78e4be3a85586be1c763cdb9db3642bbefc4 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 18 Sep 2020 15:37:19 +0200 Subject: [PATCH 042/237] added memory caps, docker info, folder structure --- docker/README.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 3540ff439c..8fcfb04a6c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,6 +11,19 @@ With this docker-compose.yml you will be able to run the following images: ## Some Hint +### docker-compose version +In docker-compose version is set 3.8, to run this version you need at least Docker Engine 19.03.0+ (check widh docker --version) and at least Docker Compose 1.25.5 (check with docker-compose --version) +``` +Compose file format Docker Engine release +3.8 19.03.0+ +3.7 18.06.0+ +3.6 18.02.0+ +3.5 17.12.0+ +3.4 17.09.0+ +``` +If for some reason you have a previous version of Docker Engine or a previous version of Docker Compose and can't upgrade those, you can use 3.7 or 3.6 in docker-compose.yml + + ### Mapping volumes If you take a look of docker-compose.yml you will see you need some local folder that needs to be mapped, so before do docker-compose up, ensure folders (and config files) exist: - ./elasticsearch/data:/usr/share/elasticsearch/data @@ -27,13 +40,31 @@ Structure would look like: │ └── data │ └── logs ├── cortex -│ └── application.conf +│ └── application.conf └── thehive └── application.conf └── data └── mysql ``` +### ElasticSearch +ElasticSearch container likes big mmap count (https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) so from shell you can cgange with +```sysctl -w vm.max_map_count=262144``` +Due you would run all on same system and maybe you don't have a limited amount of RAM, better to set some size, for ElasticSearch, in docker-compose.yml I added those: + +```- bootstrap.memory_lock=true``` +```- "ES_JAVA_OPTS=-Xms512m -Xmx512m"``` + +Adjust depending on your needs and your env. Without these settings in my environment ElasticSearch was using 1.5GB + +### Cassandra +Like for ElasticSearch maybe you would run all on same system and maybe you don't have a limited amount of RAM, better to set some size, here for Cassandra, in docker-compose.yml I added those: + +```- MAX_HEAP_SIZE=1G``` +```- HEAP_NEWSIZE=1G``` + +Adjust depending on your needs and your env. Without these settings in my environment Cassandra was using 4GB. + ### Cortex-Analyzers - In order to use Analyzers in docker version, it is set the online json url instead absolute path of analyzers in the application.conf of Cortex: https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json From f8a4ce331e2a42e2441a3b8c7df6b12a5a81ed84 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 18 Sep 2020 15:38:20 +0200 Subject: [PATCH 043/237] Update docker-compose.yml --- docker/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d2775b6241..a77e333d0b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -15,6 +15,8 @@ services: - thread_pool.write.queue_size=10000 - gateway.recover_after_nodes=1 - xpack.security.enabled=false + - bootstrap.memory_lock=true + - 'ES_JAVA_OPTS=-Xms256m -Xmx256m' ulimits: nofile: soft: 65536 @@ -49,6 +51,8 @@ services: restart: unless-stopped hostname: cassandra environment: + - MAX_HEAP_SIZE=1G + - HEAP_NEWSIZE=1G - CASSANDRA_CLUSTER_NAME=thp volumes: - ./cassandra-data:/var/lib/cassandra/data From d899705f2641336fc66d70d7bb4f8cd8c527f9a5 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 18 Sep 2020 15:38:44 +0200 Subject: [PATCH 044/237] Update README.md --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 8fcfb04a6c..ac7536274d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -53,7 +53,7 @@ ElasticSearch container likes big mmap count (https://www.elastic.co/guide/en/el Due you would run all on same system and maybe you don't have a limited amount of RAM, better to set some size, for ElasticSearch, in docker-compose.yml I added those: ```- bootstrap.memory_lock=true``` -```- "ES_JAVA_OPTS=-Xms512m -Xmx512m"``` +```- "ES_JAVA_OPTS=-Xms256m -Xmx256m"``` Adjust depending on your needs and your env. Without these settings in my environment ElasticSearch was using 1.5GB From 7d8e80c07e5c3687d174c2a5b5de85797839d4f2 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 18 Sep 2020 17:21:29 +0200 Subject: [PATCH 045/237] added Shuffle FOSS SOAR as example --- docker/docker-compose.yml | 64 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a77e333d0b..5d16e2b6ec 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -104,3 +104,67 @@ services: - "CRON_USER_ID=1" # The MISP user ID to run cron jobs as - "DISIPV6=true" # Disable IPV6 in nginx + +#READY FOR AUTOMATION ? + frontend: + image: frikky/shuffle:frontend + container_name: shuffle-frontend + hostname: shuffle-frontend + ports: + - "3001:80" + - "3443:443" + environment: + - BACKEND_HOSTNAME=shuffle-backend + restart: unless-stopped + depends_on: + - backend + backend: + image: frikky/shuffle:backend + container_name: shuffle-backend + hostname: shuffle-backend + ports: + - "5001:5001" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./shuffle-apps:/shuffle-apps + environment: + - DATASTORE_EMULATOR_HOST=shuffle-database:8000 + - SHUFFLE_APP_HOTLOAD_FOLDER=./shuffle-apps + - ORG_ID=Shuffle + - SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps + - SHUFFLE_DEFAULT_USERNAME=admin + - SHUFFLE_DEFAULT_PASSWORD=password + - SHUFFLE_DEFAULT_APIKEY=mysecretkey + - HTTP_PROXY= + - HTTPS_PROXY= + restart: unless-stopped + depends_on: + - database + orborus: + image: frikky/shuffle:orborus + container_name: shuffle-orborus + hostname: shuffle-orborus + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - SHUFFLE_APP_SDK_VERSION=0.6.0 + - SHUFFLE_WORKER_VERSION=0.6.0 + - ORG_ID=Shuffle + - ENVIRONMENT_NAME=Shuffle + - BASE_URL=http://shuffle-backend:5001 + - DOCKER_API_VERSION=1.40 + - HTTP_PROXY= + - HTTPS_PROXY= + - SHUFFLE_PASS_WORKER_PROXY=False + restart: unless-stopped + database: + image: frikky/shuffle:database + container_name: shuffle-database + hostname: shuffle-database + ports: + - "8000:8000" + restart: unless-stopped + volumes: + - ./shuffle-database:/etc/shuffle + + From b887e3da85bb663a91c580b82ebfaca8b56de470 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 18 Sep 2020 17:36:02 +0200 Subject: [PATCH 046/237] added Shuffle --- docker/README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docker/README.md b/docker/README.md index ac7536274d..057765a297 100644 --- a/docker/README.md +++ b/docker/README.md @@ -103,5 +103,41 @@ Adjust depending on your needs and your env. Without these settings in my enviro - go in Automation page and grab the api key to use in the hive application.conf to receive alerts from MISP or to use in MISP analyzers inside Cortex. +### SHUFFLE +To test automation I choose SHUFFLE (https://shuffler.io/) +In docker-compose.yml , after the comment "#READY FOR AUTOMATION ? " there is part dedicated to Shuffle (you can remove as the others if not needed) +Here will not document how to use it, there is already documentation (https://shuffler.io/docs/about). + +Here just describe how to connect the things together. + +- After SHUFFLE starts, go at login page (the frontend port by default is 3001), put credentials choosen in docker-compose.yml , for your convenience I set admin // password , create your first workflow, can be anything you have in mind, then go in Triggers, place Webhook node on dashboard, select it and grab the Webhook URI. it will be something like http://192.168.29.1:3001/api/v1/hooks/webhook_0982214b-3b92-4a85-b6fa-771982c2b449 +- Go in applicaiton.conf of The Hive and modify the url under webhook notification part: +``` +notification.webhook.endpoints = [ + { + name: local + url: "http://192.168.29.1:3001/api/v1/hooks/webhook_0982214b-3b92-4a85-b6fa-771982c2b449" + version: 0 + wsConfig: {} + includedTheHiveOrganisations: [] + excludedTheHiveOrganisations: [] + } +] +``` +- In The Hive webhooks are not enabled by default, you should enable it, there is a guide to do it: https://github.com/TheHive-Project/TheHiveDocs/blob/master/TheHive4/Administration/Webhook.md +In my case I had to call this: +``` +curl -XPUT -uuser@thehive.local:user@thehive.local -H 'Content-type: application/json' 127.0.0.1:9000/api/config/organisation/notification -d ' +{ + "value": [ + { + "delegate": false, + "trigger": { "name": "AnyEvent"}, + "notifier": { "name": "webhook", "endpoint": "local" } + } + ] +}' +``` +- Now are able to play automation with The Hive, Cortex-Analyzers, MISP thanks to SHUFFLE! From 33aa6a191b8e5a7fe51dd9c11c5e57b181be0457 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 18 Sep 2020 17:45:54 +0200 Subject: [PATCH 047/237] Update README.md --- docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/README.md b/docker/README.md index 057765a297..f317715d24 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,6 +8,7 @@ With this docker-compose.yml you will be able to run the following images: - MISP 2.4.131 - Mysql 8.0.21 - Redis 6.0.8 +- Shuffle 0.7.1 ## Some Hint From dfeea8833f75d0fe7a387b9a23f0e7d84616f6fe Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 21 Sep 2020 13:44:41 +0200 Subject: [PATCH 048/237] #1501 Fix coalesce converter problem --- ScalliGraph | 2 +- .../controllers/v0/AuditRenderer.scala | 4 +- .../thp/thehive/controllers/v0/CaseCtrl.scala | 2 +- .../thehive/controllers/v0/CaseRenderer.scala | 6 +-- .../controllers/v0/ObservableRenderer.scala | 8 ++-- .../controllers/v1/ObservableRenderer.scala | 2 +- .../org/thp/thehive/services/AuditSrv.scala | 40 +++++++++---------- .../org/thp/thehive/services/CaseSrv.scala | 17 ++++---- .../notification/notifiers/Webhook.scala | 39 +++++++++--------- .../thehive/services/th3/Aggregation.scala | 8 ++-- 10 files changed, 65 insertions(+), 63 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 812a43e5c8..183ce24956 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 812a43e5c8053ba05bfbf313e241077133b4ead0 +Subproject commit 183ce2495694013a277cef83ab7f30280be7e137 diff --git a/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala index 21bbe67c58..218047efbc 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala @@ -42,7 +42,7 @@ trait AuditRenderer { def observableToJson: Traversal.V[Observable] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = _.project( _.by(_.richObservable.domainMap(_.toJson)) - .by(_.coalesce(o => caseToJson(o.`case`), o => alertToJson(o.alert))) + .by(_.coalesceMulti(o => caseToJson(o.`case`), o => alertToJson(o.alert))) ).domainMap { case (obs, caseOrAlert) => obs.as[JsObject] + ((caseOrAlert \ "_type").asOpt[String].getOrElse("") -> caseOrAlert) } @@ -83,7 +83,7 @@ trait AuditRenderer { def auditRenderer: Traversal.V[Audit] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = (_: Traversal.V[Audit]) - .coalesce(_.`object`, _.identity.setConverter[Vertex, IdentityConverter[Vertex]](Converter.identity[Vertex])) + .coalesceIdent[Vertex](_.`object`, _.identity) .choose( _.on(_.label) .option("Case", t => caseToJson(t.v[Case])) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index 40c288c4ea..dcd27bd6ba 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -345,7 +345,7 @@ class PublicCase @Inject() ( .sack((_: Long) - (_: JLong), _.by(_.value(_.startDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long))) .sack((_: Long) / (_: Long), _.by(_.constant(3600000L))) .sack[Long], - _.constant(0L) + 0L ) ).readonly ) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala index 5b56c2f48e..631b7ce686 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala @@ -58,8 +58,8 @@ trait CaseRenderer { def isOwnerStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[Boolean, String, Converter[Boolean, String]] = _.origin.value(_.name).domainMap(_ == authContext.organisation) - def caseStatsRenderer( - implicit authContext: AuthContext + def caseStatsRenderer(implicit + authContext: AuthContext ): Traversal.V[Case] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = _.project( _.by( @@ -68,7 +68,7 @@ trait CaseRenderer { _.by(taskStats) .by(observableStats) ), - _.constant2[(JsObject, JsObject), JMap[String, Any]](JsObject.empty -> JsObject.empty) + JsObject.empty -> JsObject.empty ) ).by(alertStats) .by(mergeFromStats) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala index ddbc076830..6589aa9d9b 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala @@ -16,8 +16,8 @@ import play.api.libs.json.{JsObject, Json} trait ObservableRenderer { - def observableStatsRenderer( - implicit authContext: AuthContext + def observableStatsRenderer(implicit + authContext: AuthContext ): Traversal.V[Observable] => Traversal[JsObject, JMap[JBoolean, JLong], Converter[JsObject, JMap[JBoolean, JLong]]] = _.similar .visible @@ -32,8 +32,8 @@ trait ObservableRenderer { } def observableLinkRenderer: V[Observable] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = - _.coalesce( - _.alert.richAlert.domainMap(a => Json.obj("alert" -> a.toJson)), + _.coalesceMulti( + _.alert.richAlert.domainMap(a => Json.obj("alert" -> a.toJson)), _.`case`.richCaseWithoutPerms.domainMap(c => Json.obj("case" -> c.toJson)) ) } diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala index ae88526ba2..9f69cf6185 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala @@ -43,7 +43,7 @@ trait ObservableRenderer { _.origin.has("name", authContext.organisation).fold.domainMap(l => JsBoolean(l.nonEmpty)) def observableLinks: Traversal.V[Observable] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = - _.coalesce( + _.coalesceMulti( _.alert.richAlert.domainMap(a => Json.obj("alert" -> a.toJson)), _.`case`.richCaseWithoutPerms.domainMap(c => Json.obj("case" -> c.toJson)) ) diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index 46c7bb43f6..ba3256f75e 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -107,8 +107,8 @@ class AuditSrv @Inject() ( } } - private def createFromPending(tx: AnyRef, audit: Audit, context: Option[Product with Entity], `object`: Option[Product with Entity])( - implicit graph: Graph, + private def createFromPending(tx: AnyRef, audit: Audit, context: Option[Product with Entity], `object`: Option[Product with Entity])(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = { logger.debug(s"Store audit entity: $audit") @@ -123,8 +123,8 @@ class AuditSrv @Inject() ( } } - def create(audit: Audit, context: Option[Product with Entity], `object`: Option[Product with Entity])( - implicit graph: Graph, + def create(audit: Audit, context: Option[Product with Entity], `object`: Option[Product with Entity])(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = { def setupCallbacks(tx: AnyRef): Try[Unit] = { @@ -190,8 +190,8 @@ class AuditSrv @Inject() ( class UserAudit extends SelfContextObjectAudit[User] { - def changeProfile(user: User with Entity, organisation: Organisation, profile: Profile)( - implicit graph: Graph, + def changeProfile(user: User with Entity, organisation: Organisation, profile: Profile)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = auditSrv.create( @@ -200,8 +200,8 @@ class AuditSrv @Inject() ( Some(user) ) - def delete(user: User with Entity, organisation: Organisation with Entity)( - implicit graph: Graph, + def delete(user: User with Entity, organisation: Organisation with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = auditSrv.create( @@ -213,8 +213,8 @@ class AuditSrv @Inject() ( class ShareAudit { - def shareCase(`case`: Case with Entity, organisation: Organisation with Entity, profile: Profile with Entity)( - implicit graph: Graph, + def shareCase(`case`: Case with Entity, organisation: Organisation with Entity, profile: Profile with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = auditSrv.create( @@ -223,8 +223,8 @@ class AuditSrv @Inject() ( Some(`case`) ) - def shareTask(task: Task with Entity, `case`: Case with Entity, organisation: Organisation with Entity)( - implicit graph: Graph, + def shareTask(task: Task with Entity, `case`: Case with Entity, organisation: Organisation with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = auditSrv.create( @@ -233,8 +233,8 @@ class AuditSrv @Inject() ( Some(`case`) ) - def shareObservable(observable: Observable with Entity, `case`: Case with Entity, organisation: Organisation with Entity)( - implicit graph: Graph, + def shareObservable(observable: Observable with Entity, `case`: Case with Entity, organisation: Organisation with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = auditSrv.create( @@ -250,8 +250,8 @@ class AuditSrv @Inject() ( Some(`case`) ) - def unshareTask(task: Task with Entity, `case`: Case with Entity, organisation: Organisation with Entity)( - implicit graph: Graph, + def unshareTask(task: Task with Entity, `case`: Case with Entity, organisation: Organisation with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = auditSrv.create( @@ -260,8 +260,8 @@ class AuditSrv @Inject() ( Some(`case`) ) - def unshareObservable(observable: Observable with Entity, `case`: Case with Entity, organisation: Organisation with Entity)( - implicit graph: Graph, + def unshareObservable(observable: Observable with Entity, `case`: Case with Entity, organisation: Organisation with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = auditSrv.create( @@ -329,14 +329,14 @@ object AuditOps { def `case`: Traversal.V[Case] = traversal .out[AuditContext] - .coalesce(_.in().hasLabel("Share"), _.hasLabel("Share")) + .coalesceIdent[Vertex](_.in().hasLabel("Share"), _.hasLabel("Share")) .out[ShareCase] .v[Case] def organisation: Traversal.V[Organisation] = traversal .out[AuditContext] - .coalesce( + .coalesceIdent[Vertex]( _.hasLabel("Organisation"), _.in().hasLabel("Share").in[OrganisationShare], _.both().hasLabel("Organisation") diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 8cb6192f2d..a37a10bcbc 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -75,9 +75,10 @@ class CaseSrv @Inject() ( case (task, owner) => taskSrv.create(task, owner) } _ <- createdTasks.toTry(t => shareSrv.shareTask(t, createdCase, organisation)) - caseTemplateCustomFields = caseTemplate - .fold[Seq[RichCustomField]](Nil)(_.customFields) - .map(cf => (cf.name, cf.value, cf.order)) + caseTemplateCustomFields = + caseTemplate + .fold[Seq[RichCustomField]](Nil)(_.customFields) + .map(cf => (cf.name, cf.value, cf.order)) cfs <- (caseTemplateCustomFields ++ customFields).toTry { case (name, value, order) => createCustomField(createdCase, name, value, order) } caseTemplateTags = caseTemplate.fold[Seq[Tag with Entity]](Nil)(_.tags) allTags = tags ++ caseTemplateTags @@ -152,8 +153,8 @@ class CaseSrv @Inject() ( } yield () } - def addObservable(`case`: Case with Entity, richObservable: RichObservable)( - implicit graph: Graph, + def addObservable(`case`: Case with Entity, richObservable: RichObservable)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = { val alreadyExistInThatCase = observableSrv @@ -206,8 +207,8 @@ class CaseSrv @Inject() ( .map(_ => ()) } - def setOrCreateCustomField(`case`: Case with Entity, customFieldName: String, value: Option[Any], order: Option[Int])( - implicit graph: Graph, + def setOrCreateCustomField(`case`: Case with Entity, customFieldName: String, value: Option[Any], order: Option[Int])(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = { val cfv = get(`case`).customFields(customFieldName) @@ -373,7 +374,7 @@ object CaseOps { def richCaseWithCustomRenderer[D, G, C <: Converter[D, G]]( entityRenderer: Traversal.V[Case] => Traversal[D, G, C] - )(implicit authContext: AuthContext) = + )(implicit authContext: AuthContext): Traversal[(RichCase, D), JMap[String, Any], Converter[(RichCase, D), JMap[String, Any]]] = traversal .project( _.by diff --git a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala index d004645668..ad21bf9974 100644 --- a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala +++ b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala @@ -64,10 +64,11 @@ class WebhookProvider @Inject() ( override def apply(config: Configuration): Try[Notifier] = for { name <- config.getOrFail[String]("endpoint") - config <- webhookConfigs - .get - .find(_.name == name) - .fold[Try[WebhookNotification]](Failure(BadConfigurationError(s"Webhook configuration `$name` not found`")))(Success.apply) + config <- + webhookConfigs + .get + .find(_.name == name) + .fold[Try[WebhookNotification]](Failure(BadConfigurationError(s"Webhook configuration `$name` not found`")))(Success.apply) } yield new Webhook(config, auditSrv, customFieldSrv, mat, ec) } @@ -111,7 +112,7 @@ class Webhook( def observableToJson: Traversal.V[Observable] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = _.project( _.by(_.richObservable.domainMap(_.toJson)) - .by(_.coalesce(o => caseToJson(o.`case`), o => alertToJson(o.alert))) + .by(_.coalesceMulti(o => caseToJson(o.`case`), o => alertToJson(o.alert))) ).domainMap { case (obs, caseOrAlert) => obs.as[JsObject] + ((caseOrAlert \ "_type").asOpt[String].getOrElse("") -> caseOrAlert) } @@ -134,20 +135,20 @@ class Webhook( ).domainMap { case (vertex, _) => JsObject( - UMapping.string.optional.getProperty(vertex, "workerId").map(v => "analyzerId" -> JsString(v)).toList ::: - UMapping.string.optional.getProperty(vertex, "workerName").map(v => "analyzerName" -> JsString(v)).toList ::: + UMapping.string.optional.getProperty(vertex, "workerId").map(v => "analyzerId" -> JsString(v)).toList ::: + UMapping.string.optional.getProperty(vertex, "workerName").map(v => "analyzerName" -> JsString(v)).toList ::: UMapping.string.optional.getProperty(vertex, "workerDefinition").map(v => "analyzerDefinition" -> JsString(v)).toList ::: - UMapping.string.optional.getProperty(vertex, "status").map(v => "status" -> JsString(v)).toList ::: - UMapping.date.optional.getProperty(vertex, "startDate").map(v => "startDate" -> JsNumber(v.getTime)).toList ::: - UMapping.date.optional.getProperty(vertex, "endDate").map(v => "endDate" -> JsNumber(v.getTime)).toList ::: - UMapping.string.optional.getProperty(vertex, "cortexId").map(v => "cortexId" -> JsString(v)).toList ::: - UMapping.string.optional.getProperty(vertex, "cortexJobId").map(v => "cortexJobId" -> JsString(v)).toList ::: - UMapping.string.optional.getProperty(vertex, "_createdBy").map(v => "_createdBy" -> JsString(v)).toList ::: - UMapping.date.optional.getProperty(vertex, "_createdAt").map(v => "_createdAt" -> JsNumber(v.getTime)).toList ::: - UMapping.string.optional.getProperty(vertex, "_updatedBy").map(v => "_updatedBy" -> JsString(v)).toList ::: - UMapping.date.optional.getProperty(vertex, "_updatedAt").map(v => "_updatedAt" -> JsNumber(v.getTime)).toList ::: - UMapping.string.optional.getProperty(vertex, "_type").map(v => "_type" -> JsString(v)).toList ::: - UMapping.string.optional.getProperty(vertex, "_id").map(v => "_id" -> JsString(v)).toList + UMapping.string.optional.getProperty(vertex, "status").map(v => "status" -> JsString(v)).toList ::: + UMapping.date.optional.getProperty(vertex, "startDate").map(v => "startDate" -> JsNumber(v.getTime)).toList ::: + UMapping.date.optional.getProperty(vertex, "endDate").map(v => "endDate" -> JsNumber(v.getTime)).toList ::: + UMapping.string.optional.getProperty(vertex, "cortexId").map(v => "cortexId" -> JsString(v)).toList ::: + UMapping.string.optional.getProperty(vertex, "cortexJobId").map(v => "cortexJobId" -> JsString(v)).toList ::: + UMapping.string.optional.getProperty(vertex, "_createdBy").map(v => "_createdBy" -> JsString(v)).toList ::: + UMapping.date.optional.getProperty(vertex, "_createdAt").map(v => "_createdAt" -> JsNumber(v.getTime)).toList ::: + UMapping.string.optional.getProperty(vertex, "_updatedBy").map(v => "_updatedBy" -> JsString(v)).toList ::: + UMapping.date.optional.getProperty(vertex, "_updatedAt").map(v => "_updatedAt" -> JsNumber(v.getTime)).toList ::: + UMapping.string.optional.getProperty(vertex, "_type").map(v => "_type" -> JsString(v)).toList ::: + UMapping.string.optional.getProperty(vertex, "_id").map(v => "_id" -> JsString(v)).toList ) } @@ -165,7 +166,7 @@ class Webhook( .option("Job", jobToJson) .none(_.constant2[JsObject, JMap[String, Any]](JsObject.empty)) ), - _.constant2[JsObject, JMap[String, Any]](JsObject.empty) + JsObject.empty ) } diff --git a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala index 9842199ed2..00209b1289 100644 --- a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala +++ b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala @@ -165,7 +165,7 @@ case class AggSum(aggName: Option[String], fieldName: String) extends Aggregatio .sum .domainMap(sum => Output(Json.obj(name -> JsNumber(BigDecimal(sum.toString))))) .castDomain[Output[_]], - _.constant2(Output(Json.obj(name -> JsNull))) + Output(Json.obj(name -> JsNull)) ) } } @@ -187,7 +187,7 @@ case class AggAvg(aggName: Option[String], fieldName: String) extends Aggregatio .select(fieldPath, t) .mean .domainMap(avg => Output(Json.obj(name -> avg.asInstanceOf[Double]))), - _.constant2(Output(Json.obj(name -> JsNull))) + Output(Json.obj(name -> JsNull)) ) } } @@ -210,7 +210,7 @@ case class AggMin(aggName: Option[String], fieldName: String) extends Aggregatio .select(fieldPath, t) .min .domainMap(min => Output(Json.obj(name -> property.mapping.selectRenderer.toJson(min)))), - _.constant2(Output(Json.obj(name -> JsNull))) + Output(Json.obj(name -> JsNull)) ) } } @@ -233,7 +233,7 @@ case class AggMax(aggName: Option[String], fieldName: String) extends Aggregatio .select(fieldPath, t) .max .domainMap(max => Output(Json.obj(name -> property.mapping.selectRenderer.toJson(max)))), - _.constant2(Output(Json.obj(name -> JsNull))) + Output(Json.obj(name -> JsNull)) ) } } From f59c49ff7337991e55f1fc5ec5fc88283b611374 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 21 Sep 2020 17:07:27 +0200 Subject: [PATCH 049/237] Distinguish local and global aggregation --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index 183ce24956..7264448db2 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 183ce2495694013a277cef83ab7f30280be7e137 +Subproject commit 7264448db2963a43d7d024d83a6bd27c52ec8ced From 30b963378896da37a522dc58b95b4f2171874dc7 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 21 Sep 2020 17:18:49 +0200 Subject: [PATCH 050/237] #1544 Fix related cases query to return all similar observables --- .../org/thp/thehive/services/CaseSrv.scala | 40 +++++++------------ .../thp/thehive/services/ObservableSrv.scala | 2 +- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index a37a10bcbc..97eb3c1542 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -468,33 +468,21 @@ object CaseOps { JMap[String, Any] ]]] traversal - .`match`( - _.as(originCaseLabel)( - _.in[ShareCase] - .filter( - _.in[OrganisationShare] - .has("name", authContext.organisation) - ) - .out[ShareObservable] - .v[Observable] - ).as(observableLabel), - _.as(observableLabel)( - _.out[ObservableData] - .in[ObservableData] - .in[ShareObservable] - .filter( - _.in[OrganisationShare] - .has("name", authContext.organisation) - ) - .out("ShareCase") - .where(P.neq(originCaseLabel.name)) - .v[Case] - ).as(linkedCaseLabel), - _.as(linkedCaseLabel)(_.richCase).as(richCaseLabel), - _.as(observableLabel)(_.richObservable.fold).as(richObservablesLabel) + .as(originCaseLabel) + .observables + .as(observableLabel) + .out[ObservableData] + .in[ObservableData] + .in[ShareObservable] + .filter( + _.in[OrganisationShare] + .has("name", authContext.organisation) ) - .dedup(richCaseLabel) - .select((richCaseLabel, richObservablesLabel)) + .out[ShareCase] + .where(P.neq(originCaseLabel.name)) + .group(_.by, _.by(_.select(observableLabel).richObservable.fold)) + .unfold + .project(_.by(_.selectKeys.v[Case].richCase).by(_.selectValues)) .toSeq } diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 1460605c75..779393f97b 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -331,7 +331,7 @@ object ObservableOps { def similar: Traversal.V[Observable] = { val originLabel = StepLabel.v[Observable] traversal - .aggregate(originLabel) + .aggregateLocal(originLabel) .unionFlat( _.out[ObservableData] .in[ObservableData], From 6fe7a129769c59751f0ab0bc514a2ce0fbe00b06 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 21 Sep 2020 17:53:18 +0200 Subject: [PATCH 051/237] #1545 Fix permission check in task creation --- .../app/org/thp/thehive/controllers/v0/TaskCtrl.scala | 2 +- .../app/org/thp/thehive/controllers/v1/TaskCtrl.scala | 3 ++- thehive/app/org/thp/thehive/services/CaseSrv.scala | 10 +--------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala index a54cd71bfe..bdc3ab160c 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala @@ -37,7 +37,7 @@ class TaskCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val inputTask: InputTask = request.body("task") for { - case0 <- caseSrv.getOrFail(caseId) + case0 <- caseSrv.get(caseId).can(Permissions.manageTask).getOrFail("Case") owner <- inputTask.owner.map(userSrv.getOrFail).flip createdTask <- taskSrv.create(inputTask.toTask, owner) organisation <- organisationSrv.getOrFail(request.organisation) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala index 32470eb1d1..2f132f8109 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala @@ -10,6 +10,7 @@ import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputTask import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ import org.thp.thehive.services.{CaseSrv, OrganisationSrv, ShareSrv, TaskSrv} @@ -66,7 +67,7 @@ class TaskCtrl @Inject() ( val inputTask: InputTask = request.body("task") val caseId: String = request.body("caseId") for { - case0 <- caseSrv.getOrFail(caseId) + case0 <- caseSrv.get(caseId).can(Permissions.manageTask).getOrFail("Case") createdTask <- taskSrv.create(inputTask.toTask, None) organisation <- organisationSrv.getOrFail(request.organisation) _ <- shareSrv.shareTask(createdTask, case0, organisation) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 97eb3c1542..856d2aff3d 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -1,6 +1,6 @@ package org.thp.thehive.services -import java.util.{List => JList, Map => JMap} +import java.util.{Map => JMap} import akka.actor.ActorRef import javax.inject.{Inject, Named, Singleton} @@ -459,14 +459,6 @@ object CaseOps { def linkedCases(implicit authContext: AuthContext): Seq[(RichCase, Seq[RichObservable])] = { val originCaseLabel = StepLabel.v[Case] val observableLabel = StepLabel.v[Observable] - val linkedCaseLabel = StepLabel.v[Case] - - val richCaseLabel = StepLabel[RichCase, JMap[String, Any], Converter[RichCase, JMap[String, Any]]] - val richObservablesLabel = - StepLabel[Seq[RichObservable], JList[JMap[String, Any]], Converter.CList[RichObservable, JMap[String, Any], Converter[ - RichObservable, - JMap[String, Any] - ]]] traversal .as(originCaseLabel) .observables From df8f8e268c6913f77493d59c53b30e1f96f8a136 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 21 Sep 2020 18:44:22 +0200 Subject: [PATCH 052/237] #1546 Fix audit and user url in describe API --- thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala | 4 ++-- thehive/app/org/thp/thehive/services/th3/Aggregation.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala index 1aec31a7a1..77db1bb1c9 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala @@ -80,9 +80,9 @@ class DescribeCtrl @Inject() ( EntityDescription("case_task", "/case/task", taskCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_task", _))), EntityDescription("alert", "/alert", alertCtrl.publicData.publicProperties.list.flatMap(propertyToJson("alert", _))), EntityDescription("case_artifact", "/case/artifact", observableCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_artifact", _))), - EntityDescription("user", "user", userCtrl.publicData.publicProperties.list.flatMap(propertyToJson("user", _))), + EntityDescription("user", "/user", userCtrl.publicData.publicProperties.list.flatMap(propertyToJson("user", _))), EntityDescription("case_task_log", "/case/task/log", logCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), - EntityDescription("audit", "audit", auditCtrl.publicData.publicProperties.list.flatMap(propertyToJson("audit", _))) + EntityDescription("audit", "/audit", auditCtrl.publicData.publicProperties.list.flatMap(propertyToJson("audit", _))) ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") diff --git a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala index 00209b1289..0613df53a5 100644 --- a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala +++ b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala @@ -283,8 +283,8 @@ case class FieldAggregation( case order => order -> Order.asc } .foldLeft(groupedVertices) { - case (acc, (field, order)) if field == fieldName => acc.sort(_.by(_.selectKeys, order)) - case (acc, (field, order)) if field == "count" => acc.sort(_.by(_.selectValues.localCount, order)) + case (acc, (field, order)) if field == fieldName => acc.sort(_.by(_.selectKeys, order)) + case (acc, (field, order)) if field == "count" || field == "_count" => acc.sort(_.by(_.selectValues.localCount, order)) case (acc, (field, _)) => logger.warn(s"In field aggregation you can only sort by the field ($fieldName) or by count, not by $field") acc From 30573d82d8c85daed61c973a9a2076bf4b8c2538 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 18 Sep 2020 07:17:07 +0200 Subject: [PATCH 053/237] #1548 #1519 Update docker entrypoint to include default configuration file and remove prevent cassandra wait if it is not configured --- package/docker/entrypoint | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/package/docker/entrypoint b/package/docker/entrypoint index cdbf897d2a..cbf58c58ff 100755 --- a/package/docker/entrypoint +++ b/package/docker/entrypoint @@ -76,9 +76,7 @@ done if test "${CONFIG}" = 1 then - echo "Waiting until Cassandra DB is up" - sleep 30 # Sleep until cassandra Db is up - CONFIG_FILE=$(mktemp).conf + CONFIG_FILE=$(mktemp --tmpdir thehive-XXXXXX.conf) if test "${CONFIG_SECRET}" = 1 then if test -z "${SECRET}" @@ -106,18 +104,21 @@ then echo "storage.directory = \"${BDB_DIRECTORY}\"" >> ${CONFIG_FILE} echo "berkeleyje.freeDisk = 1" >> ${CONFIG_FILE} else - echo "Using cassanra address = ${CQL[@]}" + echo "Using cassandra address = ${CQL[@]}" echo "storage.backend = cql" >> ${CONFIG_FILE} - if [[ -n $CQL_USERNAME && -n $CQL_PASSWORD ]];then - echo "storage.username = \"${CQL_USERNAME}\"" >> ${CONFIG_FILE} - echo "storage.password = \"${CQL_PASSWORD}\"" >> ${CONFIG_FILE} - printf "Using ${CQL_USERNAME} as cassandra username and ${CQL_PASSWORD} as its password\n" + if [[ -n $CQL_USERNAME && -n $CQL_PASSWORD ]] + then + echo "storage.username = \"${CQL_USERNAME}\"" >> ${CONFIG_FILE} + echo "storage.password = \"${CQL_PASSWORD}\"" >> ${CONFIG_FILE} + printf "Using ${CQL_USERNAME} as cassandra username and ${CQL_PASSWORD} as its password\n" fi echo "storage.cql.cluster-name = thp" >> ${CONFIG_FILE} echo "storage.cql.keyspace = thehive" >> ${CONFIG_FILE} echo "storage.hostname = [" >> ${CONFIG_FILE} printf '%s\n' "${CQL_HOSTS[@]}" >> ${CONFIG_FILE} echo "]" >> ${CONFIG_FILE} + echo "Waiting until Cassandra DB is up" + sleep 30 # Sleep until cassandra Db is up fi echo "}" >> ${CONFIG_FILE} fi @@ -174,7 +175,7 @@ then fi fi - echo "include file(\"secret.conf\")" >> ${CONFIG_FILE} + echo "include file(\"/etc/thehive/application.conf\")" >> ${CONFIG_FILE} fi bin/thehive \ From e0a27868051fecd85d109479cc8f0700d0341817 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 25 Sep 2020 08:57:08 +0200 Subject: [PATCH 054/237] #1550 Fix case use in describe API --- .../thehive/controllers/v0/DescribeCtrl.scala | 33 +++++++++++-------- .../thehive/controllers/v1/DescribeCtrl.scala | 29 ++++++++-------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala index 77db1bb1c9..8acaf7294b 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala @@ -75,16 +75,23 @@ class DescribeCtrl @Inject() ( ) ).toOption - val entityDescriptions: Seq[EntityDescription] = Seq( - EntityDescription("case", "/case", caseCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case", _))), - EntityDescription("case_task", "/case/task", taskCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_task", _))), - EntityDescription("alert", "/alert", alertCtrl.publicData.publicProperties.list.flatMap(propertyToJson("alert", _))), - EntityDescription("case_artifact", "/case/artifact", observableCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_artifact", _))), - EntityDescription("user", "/user", userCtrl.publicData.publicProperties.list.flatMap(propertyToJson("user", _))), - EntityDescription("case_task_log", "/case/task/log", logCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), - EntityDescription("audit", "/audit", auditCtrl.publicData.publicProperties.list.flatMap(propertyToJson("audit", _))) - ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ - describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") + def entityDescriptions: Seq[EntityDescription] = + cacheApi.getOrElseUpdate(s"describe.v0", cacheExpire) { + Seq( + EntityDescription("case", "/case", caseCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case", _))), + EntityDescription("case_task", "/case/task", taskCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_task", _))), + EntityDescription("alert", "/alert", alertCtrl.publicData.publicProperties.list.flatMap(propertyToJson("alert", _))), + EntityDescription( + "case_artifact", + "/case/artifact", + observableCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_artifact", _)) + ), + EntityDescription("user", "/user", userCtrl.publicData.publicProperties.list.flatMap(propertyToJson("user", _))), + EntityDescription("case_task_log", "/case/task/log", logCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), + EntityDescription("audit", "/audit", auditCtrl.publicData.publicProperties.list.flatMap(propertyToJson("audit", _))) + ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ + describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") + } implicit val propertyDescriptionWrites: Writes[PropertyDescription] = Json.writes[PropertyDescription].transform((_: JsObject) + ("description" -> JsString(""))) @@ -179,7 +186,7 @@ class DescribeCtrl @Inject() ( .auth { _ => entityDescriptions .collectFirst { - case desc if desc.label == modelName => Success(Results.Ok(cacheApi.getOrElseUpdate(s"describe.v0.$modelName", cacheExpire)(desc.toJson))) + case desc if desc.label == modelName => Success(Results.Ok(desc.toJson)) } .getOrElse(Failure(NotFoundError(s"Model $modelName not found"))) } @@ -187,9 +194,7 @@ class DescribeCtrl @Inject() ( def describeAll: Action[AnyContent] = entrypoint("describe all models") .auth { _ => - val descriptors = entityDescriptions.map { desc => - desc.label -> cacheApi.getOrElseUpdate(s"describe.v0.${desc.label}", cacheExpire)(desc.toJson) - } + val descriptors = entityDescriptions.map(desc => desc.label -> desc.toJson) Success(Results.Ok(JsObject(descriptors))) } } diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index 1c5c2b6ec4..24c63afeea 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -73,16 +73,19 @@ class DescribeCtrl @Inject() ( ) ).toOption - val entityDescriptions: Seq[EntityDescription] = Seq( - EntityDescription("case", caseCtrl.publicProperties.list.flatMap(propertyToJson("case", _))), - EntityDescription("case_task", taskCtrl.publicProperties.list.flatMap(propertyToJson("case_task", _))), - EntityDescription("alert", alertCtrl.publicProperties.list.flatMap(propertyToJson("alert", _))), - EntityDescription("case_artifact", observableCtrl.publicProperties.list.flatMap(propertyToJson("case_artifact", _))), - EntityDescription("user", userCtrl.publicProperties.list.flatMap(propertyToJson("user", _))), -// EntityDescription("case_task_log", logCtrl.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), - EntityDescription("audit", auditCtrl.publicProperties.list.flatMap(propertyToJson("audit", _))) - ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ - describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") + def entityDescriptions: Seq[EntityDescription] = + cacheApi.getOrElseUpdate(s"describe.v1", cacheExpire) { + Seq( + EntityDescription("case", caseCtrl.publicProperties.list.flatMap(propertyToJson("case", _))), + EntityDescription("case_task", taskCtrl.publicProperties.list.flatMap(propertyToJson("case_task", _))), + EntityDescription("alert", alertCtrl.publicProperties.list.flatMap(propertyToJson("alert", _))), + EntityDescription("case_artifact", observableCtrl.publicProperties.list.flatMap(propertyToJson("case_artifact", _))), + EntityDescription("user", userCtrl.publicProperties.list.flatMap(propertyToJson("user", _))), + // EntityDescription("case_task_log", logCtrl.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), + EntityDescription("audit", auditCtrl.publicProperties.list.flatMap(propertyToJson("audit", _))) + ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ + describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") + } implicit val propertyDescriptionWrites: Writes[PropertyDescription] = Json.writes[PropertyDescription].transform((_: JsObject) + ("description" -> JsString(""))) @@ -187,7 +190,7 @@ class DescribeCtrl @Inject() ( .auth { _ => entityDescriptions .collectFirst { - case desc if desc.label == modelName => Success(Results.Ok(cacheApi.getOrElseUpdate(s"describe.v1.$modelName", cacheExpire)(desc.toJson))) + case desc if desc.label == modelName => Success(Results.Ok(desc.toJson)) } .getOrElse(Failure(NotFoundError(s"Model $modelName not found"))) } @@ -195,9 +198,7 @@ class DescribeCtrl @Inject() ( def describeAll: Action[AnyContent] = entrypoint("describe all models") .auth { _ => - val descriptors = entityDescriptions.map { desc => - desc.label -> cacheApi.getOrElseUpdate(s"describe.v1.${desc.label}", cacheExpire)(desc.toJson) - } + val descriptors = entityDescriptions.map(desc => desc.label -> desc.toJson) Success(Results.Ok(JsObject(descriptors))) } } From 22217882ed5209cc6f28fc58843781ed4e4ac406 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 28 Sep 2020 16:13:02 +0200 Subject: [PATCH 055/237] #1549 Fix typo --- frontend/app/views/partials/personal-settings.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/views/partials/personal-settings.html b/frontend/app/views/partials/personal-settings.html index 13793dda0e..54193a9061 100644 --- a/frontend/app/views/partials/personal-settings.html +++ b/frontend/app/views/partials/personal-settings.html @@ -162,7 +162,7 @@

- Need a two-step authanticator app? Download on of the folliwing + Need a two-step authanticator app? Download one of the folliwing
iOS devices: Authy
From a076dcfad2799cade35ee93869d11682a03c4d9a Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 28 Sep 2020 16:14:54 +0200 Subject: [PATCH 056/237] #1549 Fix typo --- frontend/app/views/partials/personal-settings.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/views/partials/personal-settings.html b/frontend/app/views/partials/personal-settings.html index 54193a9061..8adb4364c6 100644 --- a/frontend/app/views/partials/personal-settings.html +++ b/frontend/app/views/partials/personal-settings.html @@ -162,7 +162,7 @@

- Need a two-step authanticator app? Download one of the folliwing + Need a two-step authenticator app? Download one of the following:
iOS devices: Authy
From 7b560468c81758e809d0d0eda21f1669aeafd6db Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 28 Sep 2020 16:56:25 +0200 Subject: [PATCH 057/237] #1470 Fix the merge into case button, in the alert similar cases panel --- .../components/alert/AlertSimilarCaseListCmp.js | 10 +++++++++- .../components/alert/similar-case-list.component.html | 2 +- frontend/app/views/partials/alert/event.dialog.html | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index bad9185bd5..1762426556 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -49,12 +49,20 @@ }); }; + self.merge = function(caseId) { + this.onMergeIntoCase({ + caseId: caseId + }); + }; + + }, controllerAs: '$cmp', templateUrl: 'views/components/alert/similar-case-list.component.html', bindings: { alertId: '<', - onListLoad: '&' + onListLoad: '&', + onMergeIntoCase: '&' } }); })(); diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index 8ca43c44f1..431d99ddcf 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -78,7 +78,7 @@
- +
diff --git a/frontend/app/views/partials/alert/event.dialog.html b/frontend/app/views/partials/alert/event.dialog.html index 092219fe2d..67935fc906 100644 --- a/frontend/app/views/partials/alert/event.dialog.html +++ b/frontend/app/views/partials/alert/event.dialog.html @@ -83,7 +83,9 @@

- +
From e75112b2a44449c5d61f04f433a8807334fd5407 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 29 Sep 2020 15:09:42 +0200 Subject: [PATCH 058/237] #1554 Fix related alert search in case --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index 7264448db2..985e5e8a7f 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 7264448db2963a43d7d024d83a6bd27c52ec8ced +Subproject commit 985e5e8a7f488b1797d6bae478ddf07ed0f4c874 From 91f35270a81a5a41b77046d2b31d4c05d3c64dec Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 5 Oct 2020 19:07:23 +0200 Subject: [PATCH 059/237] #1567 Fix id fields in observable details --- .../app/scripts/controllers/case/CaseObservablesItemCtrl.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js b/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js index ef07475da0..b07205e918 100644 --- a/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js @@ -181,8 +181,8 @@ $scope.openArtifact = function (a) { $state.go('app.case.observables-item', { - caseId: a['case'].id, - itemId: a.id + caseId: a.stats['case']._id, + itemId: a._id }); }; From 84721672d4af067759544f359a0c208489cd1b50 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 5 Oct 2020 19:15:58 +0200 Subject: [PATCH 060/237] #1501 Enforce type of "has" and "choose" steps --- ScalliGraph | 2 +- .../cortex/services/ActionSrvTest.scala | 37 +++--- .../cortex/services/EntityHelperTest.scala | 9 +- .../cortex/services/JobSrvTest.scala | 8 +- .../cortex/services/ResponderSrvTest.scala | 2 +- .../misp/services/MispImportSrv.scala | 29 +++-- .../misp/services/TheHiveMispClient.scala | 37 +++--- .../org/thp/thehive/controllers/dav/VFS.scala | 102 +++++++++-------- .../controllers/v0/AuditRenderer.scala | 4 +- .../thp/thehive/controllers/v0/CaseCtrl.scala | 2 +- .../models/TheHiveSchemaDefinition.scala | 5 +- .../org/thp/thehive/services/AlertSrv.scala | 14 ++- .../org/thp/thehive/services/AuditSrv.scala | 2 +- .../org/thp/thehive/services/ConfigSrv.scala | 6 +- .../thehive/services/DatabaseWrapper.scala | 107 ------------------ .../thp/thehive/services/ReportTagSrv.scala | 6 +- .../notification/NotificationActor.scala | 4 +- .../thehive/controllers/v0/LogCtrlTest.scala | 12 +- .../thehive/controllers/v0/TaskCtrlTest.scala | 22 ++-- .../controllers/v1/AlertCtrlTest.scala | 2 +- .../notifiers/NotificationTemplateTest.scala | 4 +- .../triggers/AlertCreatedTest.scala | 2 +- 22 files changed, 159 insertions(+), 259 deletions(-) delete mode 100644 thehive/app/org/thp/thehive/services/DatabaseWrapper.scala diff --git a/ScalliGraph b/ScalliGraph index 985e5e8a7f..8a7900ffdb 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 985e5e8a7f488b1797d6bae478ddf07ed0f4c874 +Subproject commit 8a7900ffdbf11ed6e29e0dfa450e3e1ca776af35 diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ActionSrvTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ActionSrvTest.scala index 6edb69076b..66980decaa 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ActionSrvTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ActionSrvTest.scala @@ -2,10 +2,10 @@ package org.thp.thehive.connector.cortex.services import org.thp.cortex.client.{CortexClient, TestCortexClientProvider} import org.thp.cortex.dto.v0.OutputJob -import org.thp.scalligraph.AppBuilder import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{AppBuilder, EntityName} import org.thp.thehive.connector.cortex.controllers.v0.ActionCtrl import org.thp.thehive.connector.cortex.models.{JobStatus, TheHiveCortexSchemaProvider} import org.thp.thehive.models._ @@ -35,23 +35,24 @@ class ActionSrvTest extends PlaySpecification with TestAppBuilder { ) .bindNamedToProvider[Database, BasicDatabaseProvider]("with-thehive-cortex-schema") - def testAppBuilder[A](body: AppBuilder => A): A = testApp { app => - body( - app - .`override`( - _.bindActor[CortexActor]("cortex-actor") - .bindToProvider[CortexClient, TestCortexClientProvider] - .bind[Connector, TestConnector] - .bindToProvider[Schema, TheHiveCortexSchemaProvider] - ) - ) - } + def testAppBuilder[A](body: AppBuilder => A): A = + testApp { app => + body( + app + .`override`( + _.bindActor[CortexActor]("cortex-actor") + .bindToProvider[CortexClient, TestCortexClientProvider] + .bind[Connector, TestConnector] + .bindToProvider[Schema, TheHiveCortexSchemaProvider] + ) + ) + } "action service" should { "execute, create and handle finished action operations" in testApp { app => app[Database].roTransaction { implicit graph => implicit val entityWrites: OWrites[Entity] = app[ActionCtrl].entityWrites - val task1: Task with Entity = app[TaskSrv].startTraversal.has("title", "case 1 task 1").head + val task1: Task with Entity = app[TaskSrv].startTraversal.has(_.title, "case 1 task 1").head val richAction = await(app[ActionSrv].execute(task1, None, "respTest1", JsObject.empty)) richAction.workerId shouldEqual "respTest1" @@ -77,7 +78,7 @@ class ActionSrvTest extends PlaySpecification with TestAppBuilder { "handle action related to Task and Log" in testApp { app => app[Database].roTransaction { implicit graph => implicit val entityWrites: OWrites[Entity] = app[ActionCtrl].entityWrites - val log1 = app[LogSrv].startTraversal.has("message", "log for action test").head + val log1 = app[LogSrv].startTraversal.has(_.message, "log for action test").head val richAction = await(app[ActionSrv].execute(log1, None, "respTest1", JsObject.empty)) richAction.workerId shouldEqual "respTest1" @@ -101,15 +102,15 @@ class ActionSrvTest extends PlaySpecification with TestAppBuilder { } app[Database].roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "case 2 task 2").has("status", "Completed").exists must beTrue - app[TaskSrv].startTraversal.has("title", "case 2 task 2").logs.has("message", "test log from action").exists must beTrue + app[TaskSrv].startTraversal.has(_.title, "case 2 task 2").has(_.status, TaskStatus.Completed).exists must beTrue + app[TaskSrv].startTraversal.has(_.title, "case 2 task 2").logs.has(_.message, "test log from action").exists must beTrue } } "handle action related to an Alert" in testApp { app => implicit val entityWrites: OWrites[Entity] = app[ActionCtrl].entityWrites val alert = app[Database].roTransaction { implicit graph => - app[AlertSrv].get("testType;testSource;ref2").visible.head + app[AlertSrv].get(EntityName("testType;testSource;ref2")).visible.head } alert.read must beFalse val richAction = await(app[ActionSrv].execute(alert, None, "respTest1", JsObject.empty)) @@ -122,7 +123,7 @@ class ActionSrvTest extends PlaySpecification with TestAppBuilder { updatedActionTry must beSuccessfulTry app[Database].roTransaction { implicit graph => - val updatedAlert = app[AlertSrv].get("testType;testSource;ref2").visible.richAlert.head // FIXME + val updatedAlert = app[AlertSrv].get(EntityName("testType;testSource;ref2")).visible.richAlert.head // FIXME updatedAlert.read must beTrue updatedAlert.tags.map(_.toString) must contain("test tag from action") // TODO } diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/EntityHelperTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/EntityHelperTest.scala index 5c78531e68..0aa8bef360 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/EntityHelperTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/EntityHelperTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.connector.cortex.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.scalligraph.traversal.TraversalOps._ @@ -17,7 +18,7 @@ class EntityHelperTest extends PlaySpecification with TestAppBuilder { "return task info" in testApp { app => app[Database].roTransaction { implicit graph => for { - task <- app[TaskSrv].startTraversal.has("title", "case 1 task 1").getOrFail("Task") + task <- app[TaskSrv].startTraversal.has(_.title, "case 1 task 1").getOrFail("Task") (title, tlp, pap) <- app[EntityHelper].entityInfo(task) } yield (title, tlp, pap) } must beASuccessfulTry.which { @@ -31,7 +32,7 @@ class EntityHelperTest extends PlaySpecification with TestAppBuilder { "return observable info" in testApp { app => app[Database].roTransaction { implicit graph => for { - observable <- app[ObservableSrv].startTraversal.has("message", "Some weird domain").getOrFail("Observable") + observable <- app[ObservableSrv].startTraversal.has(_.message, "Some weird domain").getOrFail("Observable") (title, tlp, pap) <- app[EntityHelper].entityInfo(observable) } yield (title, tlp, pap) } must beASuccessfulTry.which { @@ -45,7 +46,7 @@ class EntityHelperTest extends PlaySpecification with TestAppBuilder { "find a manageable entity only (task)" in testApp { app => app[Database].roTransaction { implicit graph => for { - task <- app[TaskSrv].startTraversal.has("title", "case 1 task 1").getOrFail("Task") + task <- app[TaskSrv].startTraversal.has(_.title, "case 1 task 1").getOrFail("Task") t <- app[EntityHelper].get("Task", task._id, Permissions.manageAction) } yield t } must beSuccessfulTry @@ -54,7 +55,7 @@ class EntityHelperTest extends PlaySpecification with TestAppBuilder { "find a manageable entity only (alert)" in testApp { app => app[Database].roTransaction { implicit graph => for { - alert <- app[AlertSrv].get("testType;testSource;ref2").visible.getOrFail("Alert") + alert <- app[AlertSrv].get(EntityName("testType;testSource;ref2")).visible.getOrFail("Alert") t <- app[EntityHelper].get("Alert", alert._id, Permissions.manageAction) } yield t } must beSuccessfulTry diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala index 45e886592f..c6aba2221d 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala @@ -57,7 +57,7 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder { val createdJobTry = app[Database].tryTransaction { implicit graph => for { - observable <- app[ObservableSrv].startTraversal.has("message", "hello world").getOrFail("Observable") + observable <- app[ObservableSrv].startTraversal.has(_.message, "hello world").getOrFail("Observable") createdJob <- app[JobSrv].create(job, observable) } yield createdJob } @@ -69,14 +69,14 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder { (updatedJob.report.get \ "data").as[String] shouldEqual "imageedit_2_3904987689.jpg" app[Database].roTransaction { implicit graph => - app[JobSrv].get(updatedJob).observable.has("message", "hello world").exists must beTrue + app[JobSrv].get(updatedJob).observable.has(_.message, "hello world").exists must beTrue app[JobSrv].get(updatedJob).reportObservables.toList.length must equalTo(2).updateMessage { s => s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}" } for { - audit <- app[AuditSrv].startTraversal.has("objectId", updatedJob._id).getOrFail("Audit") - organisation <- app[OrganisationSrv].get("cert").getOrFail("Organisation") + audit <- app[AuditSrv].startTraversal.has(_.objectId, updatedJob._id.toString).getOrFail("Audit") + organisation <- app[OrganisationSrv].getByName("cert").getOrFail("Organisation") user <- app[UserSrv].startTraversal.getByName("certuser@thehive.local").getOrFail("User") } yield new JobFinished().filter(audit, Some(updatedJob), organisation, Some(user)) } must beASuccessfulTry(true) diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ResponderSrvTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ResponderSrvTest.scala index e2f280cca5..528b0c8b2c 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ResponderSrvTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ResponderSrvTest.scala @@ -32,7 +32,7 @@ class ResponderSrvTest extends PlaySpecification with TestAppBuilder { "responder service" should { "fetch responders by type" in testApp { app => val task = app[Database].roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "case 1 task 1").head + app[TaskSrv].startTraversal.has(_.title, "case 1 task 1").head } val r = await(app[ResponderSrv].getRespondersByType("case_task", task._id)) diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala index 8e0f6d77e6..216272bfb9 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala @@ -308,22 +308,21 @@ class MispImportSrv @Inject() ( Future.fromTry { logger.info("Removing old observables") db.tryTransaction { implicit graph => - alertSrv - .get(alert) - .observables - .filter( - _.or( - _.has("_updatedAt", P.lt(startSyncDate)), - _.and(_.hasNot("_updatedAt"), _.has("_createdAt", P.lt(startSyncDate))) - ) + alertSrv + .get(alert) + .observables + .filter( + _.or( + _.has(_._updatedAt, P.lt(startSyncDate)), + _.and(_.hasNot(_._updatedAt), _.has(_._createdAt, P.lt(startSyncDate))) ) - .toIterator - .toTry { obs => - logger.info(s"Remove $obs") - observableSrv.remove(obs) - } - } - .map(_ => ()) + ) + .toIterator + .toTry { obs => + logger.info(s"Remove $obs") + observableSrv.remove(obs) + } + }.map(_ => ()) } } } diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala index 690098fc46..4841cda933 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala @@ -114,22 +114,23 @@ class TheHiveMispClient( whitelistTags ) { - @Inject() def this(config: TheHiveMispClientConfig, mat: Materializer) = this( - config.name, - config.url, - config.auth, - new ProxyWS(config.wsConfig, mat), - config.maxAge, - config.excludedOrganisations, - config.excludedTags, - config.whitelistTags, - config.purpose, - config.caseTemplate, - config.artifactTags, - config.exportCaseTags, - config.includedTheHiveOrganisations, - config.excludedTheHiveOrganisations - ) + @Inject() def this(config: TheHiveMispClientConfig, mat: Materializer) = + this( + config.name, + config.url, + config.auth, + new ProxyWS(config.wsConfig, mat), + config.maxAge, + config.excludedOrganisations, + config.excludedTags, + config.whitelistTags, + config.purpose, + config.caseTemplate, + config.artifactTags, + config.exportCaseTags, + config.includedTheHiveOrganisations, + config.excludedTheHiveOrganisations + ) val (canImport, canExport) = purpose match { case MispPurpose.ImportAndExport => (true, true) @@ -140,9 +141,9 @@ class TheHiveMispClient( def organisationFilter(organisationSteps: Traversal.V[Organisation]): Traversal.V[Organisation] = { val includedOrgs = if (includedTheHiveOrganisations.contains("*") || includedTheHiveOrganisations.isEmpty) organisationSteps - else organisationSteps.has("name", P.within(includedTheHiveOrganisations)) + else organisationSteps.has(_.name, P.within(includedTheHiveOrganisations: _*)) if (excludedTheHiveOrganisations.isEmpty) includedOrgs - else includedOrgs.has("name", P.without(excludedTheHiveOrganisations)) + else includedOrgs.has(_.name, P.without(excludedTheHiveOrganisations: _*)) } override def getStatus(implicit ec: ExecutionContext): Future[JsObject] = diff --git a/thehive/app/org/thp/thehive/controllers/dav/VFS.scala b/thehive/app/org/thp/thehive/controllers/dav/VFS.scala index d13b2606b6..ac2c7b8c16 100644 --- a/thehive/app/org/thp/thehive/controllers/dav/VFS.scala +++ b/thehive/app/org/thp/thehive/controllers/dav/VFS.scala @@ -13,55 +13,57 @@ import org.thp.thehive.services.TaskOps._ @Singleton class VFS @Inject() (caseSrv: CaseSrv) { - def get(path: List[String])(implicit graph: Graph, authContext: AuthContext): Seq[Resource] = path match { - case Nil | "" :: Nil => List(StaticResource("")) - case "cases" :: Nil => List(StaticResource("")) - case "cases" :: cid :: Nil => caseSrv.startTraversal.getByNumber(cid.toInt).toSeq.map(EntityResource(_, "")) - case "cases" :: cid :: "observables" :: Nil => List(StaticResource("")) - case "cases" :: cid :: "tasks" :: Nil => List(StaticResource("")) - case "cases" :: cid :: "observables" :: aid :: Nil => - caseSrv - .startTraversal - .getByNumber(cid.toInt) - .observables - .attachments - .has("attachmentId", aid) - .toSeq - .map(AttachmentResource(_, emptyId = true)) - case "cases" :: cid :: "tasks" :: aid :: Nil => - caseSrv - .startTraversal - .getByNumber(cid.toInt) - .tasks - .logs - .attachments - .has("attachmentId", aid) - .toSeq - .map(AttachmentResource(_, emptyId = true)) - case _ => Nil - } + def get(path: List[String])(implicit graph: Graph, authContext: AuthContext): Seq[Resource] = + path match { + case Nil | "" :: Nil => List(StaticResource("")) + case "cases" :: Nil => List(StaticResource("")) + case "cases" :: cid :: Nil => caseSrv.startTraversal.getByNumber(cid.toInt).toSeq.map(EntityResource(_, "")) + case "cases" :: cid :: "observables" :: Nil => List(StaticResource("")) + case "cases" :: cid :: "tasks" :: Nil => List(StaticResource("")) + case "cases" :: cid :: "observables" :: aid :: Nil => + caseSrv + .startTraversal + .getByNumber(cid.toInt) + .observables + .attachments + .has(_.attachmentId, aid) + .toSeq + .map(AttachmentResource(_, emptyId = true)) + case "cases" :: cid :: "tasks" :: aid :: Nil => + caseSrv + .startTraversal + .getByNumber(cid.toInt) + .tasks + .logs + .attachments + .has(_.attachmentId, aid) + .toSeq + .map(AttachmentResource(_, emptyId = true)) + case _ => Nil + } - def list(path: List[String])(implicit graph: Graph, authContext: AuthContext): Seq[Resource] = path match { - case Nil | "" :: Nil => List(StaticResource("cases")) - case "cases" :: Nil => caseSrv.startTraversal.visible.toSeq.map(c => EntityResource(c, c.number.toString)) - case "cases" :: cid :: Nil => List(StaticResource("observables"), StaticResource("tasks")) - case "cases" :: cid :: "observables" :: Nil => - caseSrv - .startTraversal - .getByNumber(cid.toInt) - .observables - .attachments - .domainMap(AttachmentResource(_, emptyId = false)) - .toSeq - case "cases" :: cid :: "tasks" :: Nil => - caseSrv - .startTraversal - .getByNumber(cid.toInt) - .tasks - .logs - .attachments - .domainMap(AttachmentResource(_, emptyId = false)) - .toSeq - case _ => Nil - } + def list(path: List[String])(implicit graph: Graph, authContext: AuthContext): Seq[Resource] = + path match { + case Nil | "" :: Nil => List(StaticResource("cases")) + case "cases" :: Nil => caseSrv.startTraversal.visible.toSeq.map(c => EntityResource(c, c.number.toString)) + case "cases" :: cid :: Nil => List(StaticResource("observables"), StaticResource("tasks")) + case "cases" :: cid :: "observables" :: Nil => + caseSrv + .startTraversal + .getByNumber(cid.toInt) + .observables + .attachments + .domainMap(AttachmentResource(_, emptyId = false)) + .toSeq + case "cases" :: cid :: "tasks" :: Nil => + caseSrv + .startTraversal + .getByNumber(cid.toInt) + .tasks + .logs + .attachments + .domainMap(AttachmentResource(_, emptyId = false)) + .toSeq + case _ => Nil + } } diff --git a/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala index 218047efbc..4b93313587 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AuditRenderer.scala @@ -98,8 +98,8 @@ trait AuditRenderer { def jsonSummary(auditSrv: AuditSrv, requestId: String)(implicit graph: Graph): JsObject = auditSrv .startTraversal - .has("requestId", requestId) - .has("mainAction", false) + .has(_.requestId, requestId) + .has(_.mainAction, false) .group( _.byValue(_.objectType), _.by(_.groupCount(_.byValue(_.action))) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index dcd27bd6ba..e8f7d3cb4b 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -337,7 +337,7 @@ class PublicCase @Inject() ( .property("computed.handlingDurationInHours", UMapping.long)( _.select( _.coalesce( - _.has("endDate") + _.has(_.endDate) .sack( (_: JLong, endDate: JLong) => endDate, _.by(_.value(_.endDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long)) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 926fe35def..41b8efcaf6 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -14,14 +14,13 @@ import org.thp.scalligraph.janus.JanusDatabase import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ import play.api.Logger -import play.api.inject.Injector import scala.collection.JavaConverters._ import scala.reflect.runtime.{universe => ru} import scala.util.{Success, Try} @Singleton -class TheHiveSchemaDefinition @Inject() (injector: Injector) extends Schema with UpdatableSchema { +class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { // Make sure TypeDefinitionCategory has been initialised before ModifierType to prevent ExceptionInInitializerError TypeDefinitionCategory.BACKING_INDEX @@ -30,7 +29,7 @@ class TheHiveSchemaDefinition @Inject() (injector: Injector) extends Schema with val operations: Operations = Operations(name) .addProperty[Option[Boolean]]("Observable", "seen") .updateGraph("Add manageConfig permission to org-admin profile", "Profile") { traversal => - Try(traversal.has("name", "org-admin").raw.property("permissions", "manageConfig").iterate()) + Try(traversal.unsafeHas("name", "org-admin").raw.property("permissions", "manageConfig").iterate()) Success(()) } .updateGraph("Remove duplicate custom fields", "CustomField") { traversal => diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index c35ae92c49..2a330e7e86 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -150,7 +150,7 @@ class AlertSrv @Inject() ( ): Try[Unit] = { val maybeExistingObservable = richObservable.dataOrAttachment match { case Left(data) => get(alert).observables.filterOnData(data.data) - case Right(attachment) => get(alert).observables.has("attachmentId", attachment.attachmentId) + case Right(attachment) => get(alert).observables.filterOnAttachmentId(attachment.attachmentId) } maybeExistingObservable .richObservable @@ -334,9 +334,13 @@ object AlertOps { def getBySourceId(`type`: String, source: String, sourceRef: String): Traversal.V[Alert] = traversal - .has("type", `type`) - .has("source", source) - .has("sourceRef", sourceRef) + .has(_.`type`, `type`) + .has(_.source, source) + .has(_.sourceRef, sourceRef) + + def filterByType(`type`: String): Traversal.V[Alert] = traversal.has(_.`type`, `type`) + + def filterBySource(source: String): Traversal.V[Alert] = traversal.has(_.source, source) def organisation: Traversal.V[Organisation] = traversal.out[AlertOrganisation].v[Organisation] @@ -454,7 +458,7 @@ object AlertOps { } def customFields(name: String): Traversal.E[AlertCustomField] = - traversal.outE[AlertCustomField].filter(_.inV.has("name", name)).e[AlertCustomField] + traversal.outE[AlertCustomField].filter(_.inV.v[CustomField].has(_.name, name)) def customFields: Traversal.E[AlertCustomField] = traversal.outE[AlertCustomField].e[AlertCustomField] diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index ba3256f75e..ae4a9d8765 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -65,7 +65,7 @@ class AuditSrv @Inject() ( */ def getMainByIds(order: Order, ids: String*)(implicit graph: Graph): Traversal.V[Audit] = getByIds(ids: _*) - .has("mainAction", true) + .has(_.mainAction, true) .sort(_.by("_createdAt", order)) def mergeAudits[R](body: => Try[R])(auditCreator: R => Try[Unit])(implicit graph: Graph): Try[R] = { diff --git a/thehive/app/org/thp/thehive/services/ConfigSrv.scala b/thehive/app/org/thp/thehive/services/ConfigSrv.scala index cbbb3372f0..7a9a394d64 100644 --- a/thehive/app/org/thp/thehive/services/ConfigSrv.scala +++ b/thehive/app/org/thp/thehive/services/ConfigSrv.scala @@ -68,7 +68,7 @@ class ConfigSrv @Inject() ( userSrv .get(userName) .config - .has("name", name) + .has(_.name, name) .headOption } } @@ -82,7 +82,7 @@ object ConfigOps { def notificationRaw: Traversal[Config with Entity, Vertex, Converter[Config with Entity, Vertex]] = traversal .clone() - .has("name", P.eq[String]("notification")) + .has(_.name, "notification") // Retrieve triggers configured for each organisation val organisationTriggers: Iterator[(String, Trigger, Option[String])] = { @@ -137,6 +137,6 @@ object ConfigOps { } } - def getValue[A: Reads](name: String): Traversal[JsValue, String, Converter[JsValue, String]] = traversal.has("name", name).value(_.value) + def getValue[A: Reads](name: String): Traversal[JsValue, String, Converter[JsValue, String]] = traversal.has(_.name, name).value(_.value) } } diff --git a/thehive/app/org/thp/thehive/services/DatabaseWrapper.scala b/thehive/app/org/thp/thehive/services/DatabaseWrapper.scala deleted file mode 100644 index ae762f23ba..0000000000 --- a/thehive/app/org/thp/thehive/services/DatabaseWrapper.scala +++ /dev/null @@ -1,107 +0,0 @@ -//package org.thp.thehive.services -// -//import java.util.Date -//import java.util.function.Consumer -// -//import akka.NotUsed -//import akka.stream.scaladsl.Source -// -//import javax.inject.Provider -//import org.apache.tinkerpop.gremlin.structure.{Graph, Transaction} -//import org.thp.scalligraph.auth.AuthContext -//import org.thp.scalligraph.models.Model.Base -//import org.thp.scalligraph.models._ -// -//import scala.reflect.runtime.{universe => ru} -//import scala.util.Try -// -//class DatabaseWrapper(dbProvider: Provider[Database]) extends Database { -// lazy val db: Database = dbProvider.get() -// override lazy val createdAtMapping: SingleMapping[Date, _] = db.createdAtMapping -// override lazy val createdByMapping: SingleMapping[String, String] = db.createdByMapping -// override lazy val updatedAtMapping: OptionMapping[Date, _] = db.updatedAtMapping -// override lazy val updatedByMapping: OptionMapping[String, String] = db.updatedByMapping -// override lazy val binaryMapping: SingleMapping[Array[Byte], String] = db.binaryMapping -// -// override def close(): Unit = db.close() -// -// override def isValidId(id: String): Boolean = db.isValidId(id) -// -// override def createVertex[V <: Product](graph: Graph, authContext: AuthContext, model: Model.Vertex[V], v: V): V with Entity = -// db.createVertex(graph, authContext, model, v) -// -// override def createEdge[E <: Product, FROM <: Product, TO <: Product]( -// graph: Graph, -// authContext: AuthContext, -// model: Model.Edge[E], -// e: E, -// from: FROM with Entity, -// to: TO with Entity -// ): E with Entity = db.createEdge(graph, authContext, model, e, from, to) -// -//// override def update[E <: Product]( -//// elementTraversal: GremlinScala[_ <: Element], -//// fields: Seq[(String, Any)], -//// model: Base[E], -//// graph: Graph, -//// authContext: AuthContext -//// ): Try[Seq[E with Entity]] = db.update(elementTraversal, fields, model, graph, authContext) -// -// override def roTransaction[A](body: Graph => A): A = db.roTransaction(body) -// override def transaction[A](body: Graph => A): A = db.transaction(body) -// override def tryTransaction[A](body: Graph => Try[A]): Try[A] = db.tryTransaction(body) -// override def source[A](query: Graph => Iterator[A]): Source[A, NotUsed] = db.source(query) -// override def source[A, B](body: Graph => (Iterator[A], B)): (Source[A, NotUsed], B) = db.source(body) -// override def currentTransactionId(graph: Graph): AnyRef = db.currentTransactionId(graph) -// override def addCallback(callback: () => Try[Unit])(implicit graph: Graph): Unit = db.addCallback(callback) -// override def takeCallbacks(graph: Graph): List[() => Try[Unit]] = db.takeCallbacks(graph) -// override def version(module: String): Int = db.version(module) -// override def setVersion(module: String, v: Int): Try[Unit] = db.setVersion(module, v) -// override def getModel[E <: Product: ru.TypeTag]: Base[E] = db.getModel[E] -// override def getVertexModel[E <: Product: ru.TypeTag]: Model.Vertex[E] = db.getVertexModel[E] -// override def getEdgeModel[E <: Product: ru.TypeTag, FROM <: Product, TO <: Product]: Model.Edge[E, FROM, TO] = db.getEdgeModel[E, FROM, TO] -// override def createSchemaFrom(schemaObject: Schema)(implicit authContext: AuthContext): Try[Unit] = db.createSchemaFrom(schemaObject)(authContext) -// override def createSchema(model: Model, models: Model*): Try[Unit] = db.createSchema(model, models: _*) -// override def createSchema(models: Seq[Model]): Try[Unit] = db.createSchema(models) -// override def addSchemaIndexes(schemaObject: Schema): Try[Unit] = db.addSchemaIndexes(schemaObject) -// override def addSchemaIndexes(model: Model, models: Model*): Try[Unit] = db.addSchemaIndexes(model, models: _*) -// override def addSchemaIndexes(models: Seq[Model]): Try[Unit] = db.addSchemaIndexes(models) -// override def enableIndexes(): Try[Unit] = db.enableIndexes() -// override def removeAllIndexes(): Unit = db.removeAllIndexes() -// override def addProperty[T](model: String, propertyName: String, mapping: Mapping[_, _, _]): Try[Unit] = -// db.addProperty(model, propertyName, mapping) -// override def removeProperty(model: String, propertyName: String, usedOnlyByThisModel: Boolean): Try[Unit] = -// db.removeProperty(model, propertyName, usedOnlyByThisModel) -// override def addIndex(model: String, indexType: IndexType.Value, properties: Seq[String]): Try[Unit] = db.addIndex(model, indexType, properties) -// override def drop(): Unit = db.drop() -// -// override def getSingleProperty[D, G](element: Element, key: String, mapping: SingleMapping[D, G]): D = db.getSingleProperty(element, key, mapping) -// -// override def getOptionProperty[D, G](element: Element, key: String, mapping: OptionMapping[D, G]): Option[D] = -// db.getOptionProperty(element, key, mapping) -// override def getListProperty[D, G](element: Element, key: String, mapping: ListMapping[D, G]): Seq[D] = db.getListProperty(element, key, mapping) -// override def getSetProperty[D, G](element: Element, key: String, mapping: SetMapping[D, G]): Set[D] = db.getSetProperty(element, key, mapping) -// override def getProperty[D](element: Element, key: String, mapping: Mapping[D, _, _]): D = db.getProperty(element, key, mapping) -// -// override def setSingleProperty[D, G](element: Element, key: String, value: D, mapping: SingleMapping[D, _]): Unit = -// db.setSingleProperty[D, G](element, key, value, mapping) -// -// override def setOptionProperty[D, G](element: Element, key: String, value: Option[D], mapping: OptionMapping[D, _]): Unit = -// db.setOptionProperty[D, G](element, key, value, mapping) -// -// override def setListProperty[D, G](element: Element, key: String, values: Seq[D], mapping: ListMapping[D, _]): Unit = -// db.setListProperty[D, G](element, key, values, mapping) -// -// override def setSetProperty[D, G](element: Element, key: String, values: Set[D], mapping: SetMapping[D, _]): Unit = -// db.setSetProperty[D, G](element, key, values, mapping) -// override def setProperty[D](element: Element, key: String, value: D, mapping: Mapping[D, _, _]): Unit = db.setProperty(element, key, value, mapping) -// override def labelFilter[E <: Element](model: Model): GremlinScala[E] => GremlinScala[E] = db.labelFilter(model) -// override def labelFilter[E <: Element](label: String): GremlinScala[E] => GremlinScala[E] = db.labelFilter(label) -// override lazy val extraModels: Seq[Model] = db.extraModels -// override def addTransactionListener(listener: Consumer[Transaction.Status])(implicit graph: Graph): Unit = db.addTransactionListener(listener) -// override def mapPredicate[T](predicate: P[T]): P[T] = db.mapPredicate(predicate) -// override def toId(id: Any): Any = db.toId(id) -// -// override val binaryLinkModel: Model.Edge[BinaryLink, Binary, Binary] = db.binaryLinkModel -// override val binaryModel: Model.Vertex[Binary] = db.binaryModel -//} diff --git a/thehive/app/org/thp/thehive/services/ReportTagSrv.scala b/thehive/app/org/thp/thehive/services/ReportTagSrv.scala index c22e009eaf..a22d578e3a 100644 --- a/thehive/app/org/thp/thehive/services/ReportTagSrv.scala +++ b/thehive/app/org/thp/thehive/services/ReportTagSrv.scala @@ -18,8 +18,8 @@ import scala.util.Try class ReportTagSrv @Inject() (observableSrv: ObservableSrv)(implicit @Named("with-thehive-schema") db: Database) extends VertexSrv[ReportTag] { val observableReportTagSrv = new EdgeSrv[ObservableReportTag, Observable, ReportTag] - def updateTags(observable: Observable with Entity, origin: String, reportTags: Seq[ReportTag])( - implicit graph: Graph, + def updateTags(observable: Observable with Entity, origin: String, reportTags: Seq[ReportTag])(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = { observableSrv.get(observable).reportTags.fromOrigin(origin).remove() @@ -35,6 +35,6 @@ object ReportTagOps { implicit class ReportTagOpsDefs(traversal: Traversal.V[ReportTag]) { def observable: Traversal.V[Observable] = traversal.in[ObservableReportTag].v[Observable] - def fromOrigin(origin: String): Traversal.V[ReportTag] = traversal.has("origin", origin) + def fromOrigin(origin: String): Traversal.V[ReportTag] = traversal.has(_.origin, origin) } } diff --git a/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala b/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala index 382b3b8e61..dead9d9464 100644 --- a/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala +++ b/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala @@ -180,7 +180,7 @@ class NotificationActor @Inject() ( organisationSrv .get(organisation) .config - .has("name", "notification") + .has(_.name, "notification") .value(_.value) .toIterator .foreach { notificationConfig: JsValue => @@ -191,7 +191,7 @@ class NotificationActor @Inject() ( organisationSrv .get(organisation) .users - .filter(_.config.hasNot("name", "notification")) + .filter(_.config.hasNot(_.name, "notification")) .toIterator .foreach { user => executeNotification(Some(user), userConfig, audit, context, obj, organisation) diff --git a/thehive/test/org/thp/thehive/controllers/v0/LogCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/LogCtrlTest.scala index 24d2fe9dea..899c0747cb 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/LogCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/LogCtrlTest.scala @@ -14,7 +14,7 @@ class LogCtrlTest extends PlaySpecification with TestAppBuilder { "be able to create a log" in testApp { app => val task = app[Database].roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "case 1 task 1").headOption.get + app[TaskSrv].startTraversal.has(_.title, "case 1 task 1").getOrFail("Task").get } val request = FakeRequest("POST", s"/api/case/task/${task._id}/log") @@ -22,27 +22,27 @@ class LogCtrlTest extends PlaySpecification with TestAppBuilder { .withJsonBody(Json.parse(""" {"message":"log 1\n\n### yeahyeahyeahs", "deleted":false} """.stripMargin)) - val result = app[LogCtrl].create(task._id)(request) + val result = app[LogCtrl].create(task._id.toString)(request) status(result) shouldEqual 201 app[Database].roTransaction { implicit graph => - app[TaskSrv].get(task).logs.has("message", "log 1\n\n### yeahyeahyeahs").exists + app[TaskSrv].get(task).logs.has(_.message, "log 1\n\n### yeahyeahyeahs").exists } must beTrue } "be able to create and remove a log" in testApp { app => val log = app[Database].roTransaction { implicit graph => - app[LogSrv].startTraversal.has("message", "log for action test").getOrFail("Log").get + app[LogSrv].startTraversal.has(_.message, "log for action test").getOrFail("Log").get } val requestDelete = FakeRequest("DELETE", s"/api/case/task/log/${log._id}").withHeaders("user" -> "certuser@thehive.local") - val resultDelete = app[LogCtrl].delete(log._id)(requestDelete) + val resultDelete = app[LogCtrl].delete(log._id.toString)(requestDelete) status(resultDelete) shouldEqual 204 val deletedLog = app[Database].roTransaction { implicit graph => - app[LogSrv].startTraversal.has("message", "log for action test").headOption + app[LogSrv].startTraversal.has(_.message, "log for action test").headOption } deletedLog should beNone } diff --git a/thehive/test/org/thp/thehive/controllers/v0/TaskCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/TaskCtrlTest.scala index f5ce1c4dcc..42e78d7cdb 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/TaskCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/TaskCtrlTest.scala @@ -39,10 +39,10 @@ class TaskCtrlTest extends PlaySpecification with TestAppBuilder { "task controller" should { "list available tasks and get one task" in testApp { app => val taskId = app[Database].roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "case 1 task 1")._id.getOrFail("Task").get + app[TaskSrv].startTraversal.has(_.title, "case 1 task 1")._id.getOrFail("Task").get } val request = FakeRequest("GET", s"/api/case/task/$taskId").withHeaders("user" -> "certuser@thehive.local") - val result = app[TaskCtrl].get(taskId)(request) + val result = app[TaskCtrl].get(taskId.toString)(request) val resultTask = contentAsJson(result) status(result) shouldEqual 200 @@ -63,12 +63,12 @@ class TaskCtrlTest extends PlaySpecification with TestAppBuilder { "patch a task" in testApp { app => val taskId = app[Database].roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "case 1 task 1")._id.getOrFail("Task").get + app[TaskSrv].startTraversal.has(_.title, "case 1 task 1")._id.getOrFail("Task").get } val request = FakeRequest("PATCH", s"/api/case/task/$taskId") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.parse("""{"title": "new title task 1", "owner": "certuser@thehive.local", "status": "InProgress"}""")) - val result = app[TaskCtrl].update(taskId)(request) + val result = app[TaskCtrl].update(taskId.toString)(request) status(result) shouldEqual 200 @@ -85,7 +85,7 @@ class TaskCtrlTest extends PlaySpecification with TestAppBuilder { val newTask = app[Database] .roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "new title task 1").richTask.getOrFail("Task") + app[TaskSrv].startTraversal.has(_.title, "new title task 1").richTask.getOrFail("Task") } .map(TestTask.apply) .map(_.copy(startDate = None)) @@ -93,7 +93,7 @@ class TaskCtrlTest extends PlaySpecification with TestAppBuilder { } "create a new task for an existing case" in testApp { app => - val request = FakeRequest("POST", "/api/case/#1/task?flag=true") + val request = FakeRequest("POST", "/api/case/1/task?flag=true") .withJsonBody( Json .parse( @@ -107,7 +107,7 @@ class TaskCtrlTest extends PlaySpecification with TestAppBuilder { ) .withHeaders("user" -> "certuser@thehive.local") - val result = app[TaskCtrl].create("#1")(request) + val result = app[TaskCtrl].create("1")(request) val resultTask = contentAsJson(result) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") @@ -134,18 +134,18 @@ class TaskCtrlTest extends PlaySpecification with TestAppBuilder { "unset task owner" in testApp { app => val taskId = app[Database].roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "case 1 task 1")._id.getOrFail("Task").get + app[TaskSrv].startTraversal.has(_.title, "case 1 task 1")._id.getOrFail("Task").get } val request = FakeRequest("PATCH", s"/api/case/task/$taskId") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.parse("""{"owner": null}""")) - val result = app[TaskCtrl].update(taskId)(request) + val result = app[TaskCtrl].update(taskId.toString)(request) status(result) shouldEqual 200 val newTask = app[Database] .roTransaction { implicit graph => - app[TaskSrv].startTraversal.has("title", "case 1 task 1").richTask.getOrFail("Task") + app[TaskSrv].startTraversal.has(_.title, "case 1 task 1").richTask.getOrFail("Task") } .map(TestTask.apply) @@ -186,7 +186,7 @@ class TaskCtrlTest extends PlaySpecification with TestAppBuilder { } "get tasks stats" in testApp { app => - val case1 = app[Database].roTransaction(graph => app[CaseSrv].startTraversal(graph).has("title", "case#1").getOrFail("Case")) + val case1 = app[Database].roTransaction(graph => app[CaseSrv].startTraversal(graph).has(_.title, "case#1").getOrFail("Case")) case1 must beSuccessfulTry diff --git a/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala index f1c1217d82..d0f65a2559 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala @@ -136,7 +136,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { "get an alert" in testApp { app => val alertSrv = app.apply[AlertSrv] app.apply[Database].roTransaction { implicit graph => - alertSrv.startTraversal.has("sourceRef", "ref1").getOrFail("Alert") + alertSrv.startTraversal.has(_.sourceRef, "ref1").getOrFail("Alert") } must beSuccessfulTry.which { alert: Alert with Entity => val request = FakeRequest("GET", s"/api/v1/alert/${alert._id}").withHeaders("user" -> "socuser@thehive.local") val result = app[AlertCtrl].get(alert._id)(request) diff --git a/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala b/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala index 6113ad6d5f..790b847308 100644 --- a/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala +++ b/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala @@ -80,8 +80,8 @@ class NotificationTemplateTest extends PlaySpecification with TestAppBuilder { case4 <- app[CaseSrv].get("#1").getOrFail("Case") _ <- app[CaseSrv].addTags(case4, Set("emailer test")) _ <- app[CaseSrv].addTags(case4, Set("emailer test")) // this is needed to make AuditSrv write Audit in DB - audit <- app[AuditSrv].startTraversal.has("objectId", case4._id).getOrFail("Audit") - user <- app[UserSrv].get("certuser@thehive.local").getOrFail("User") + audit <- app[AuditSrv].startTraversal.has(_.objectId, case4._id.toString).getOrFail("Audit") + user <- app[UserSrv].get(EntityName("certuser@thehive.local")).getOrFail("User") msg <- templateEngine(app[Schema]).buildMessage(template, audit, Some(case4), Some(case4), Some(user), "http://localhost/") } yield msg } diff --git a/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala b/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala index c51877b37a..84eb7a9611 100644 --- a/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala +++ b/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala @@ -48,7 +48,7 @@ class AlertCreatedTest extends PlaySpecification with TestAppBuilder { alert must beSuccessfulTry - val audit = app[AuditSrv].startTraversal.has("objectId", alert.get._id).getOrFail("Audit") + val audit = app[AuditSrv].startTraversal.has(_.objectId, alert.get._id.toString).getOrFail("Audit") audit must beSuccessfulTry From 84eab8f6f66e05ac5e6b3327a79e7171962ee946 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 5 Oct 2020 19:17:27 +0200 Subject: [PATCH 061/237] #1501 Add type to safely handle entity ID --- .../cortex/controllers/v0/ActionCtrl.scala | 15 +- .../controllers/v0/AnalyzerTemplateCtrl.scala | 16 +- .../cortex/controllers/v0/Conversion.scala | 8 +- .../controllers/v0/CortexQueryExecutor.scala | 5 +- .../cortex/controllers/v0/JobCtrl.scala | 16 +- .../cortex/controllers/v0/ResponderCtrl.scala | 3 +- .../cortex/controllers/v0/Router.scala | 1 - .../connector/cortex/models/Action.scala | 8 +- ...ema.scala => CortexSchemaDefinition.scala} | 0 .../thehive/connector/cortex/models/Job.scala | 4 +- .../cortex/services/ActionOperationSrv.scala | 14 +- .../connector/cortex/services/ActionSrv.scala | 12 +- .../cortex/services/AnalyzerTemplateSrv.scala | 22 +- .../cortex/services/CortexActor.scala | 8 +- .../cortex/services/EntityHelper.scala | 15 +- .../connector/cortex/services/JobSrv.scala | 66 +++--- .../cortex/services/ResponderSrv.scala | 26 ++- .../cortex/services/ServiceHelper.scala | 37 ++-- .../cortex/controllers/v0/JobCtrlTest.scala | 2 +- .../cortex/services/JobSrvTest.scala | 6 +- .../cortex/services/ServiceHelperTest.scala | 4 +- .../scala/org/thp/thehive/dto/v0/Audit.scala | 17 +- .../scala/org/thp/thehive/dto/v1/Alert.scala | 4 +- .../scala/org/thp/thehive/dto/v1/Audit.scala | 17 +- .../scala/org/thp/thehive/dto/v1/Case.scala | 4 +- .../org/thp/thehive/dto/v1/CaseTemplate.scala | 2 +- .../thp/thehive/dto/v1/CustomFieldValue.scala | 2 +- .../org/thp/thehive/migration/IdMapping.scala | 4 +- .../thp/thehive/migration/MigrationOps.scala | 39 ++-- .../org/thp/thehive/migration/Output.scala | 19 +- .../thehive/migration/dto/InputAudit.scala | 3 +- .../thehive/migration/th3/Conversion.scala | 206 +++++++++--------- .../thp/thehive/migration/th3/DBFind.scala | 164 +++++++------- .../thp/thehive/migration/th4/Output.scala | 56 ++--- .../misp/controllers/v0/MispCtrl.scala | 5 +- .../misp/services/AttributeConverter.scala | 3 +- .../connector/misp/services/Connector.scala | 18 +- .../misp/services/MispExportSrv.scala | 15 +- .../misp/services/MispImportSrv.scala | 93 ++++---- .../misp/services/MispImportSrvTest.scala | 4 +- .../thp/thehive/controllers/dav/Router.scala | 18 +- .../thehive/controllers/v0/AlertCtrl.scala | 70 +++--- .../controllers/v0/AttachmentCtrl.scala | 6 +- .../thehive/controllers/v0/AuditCtrl.scala | 15 +- .../controllers/v0/AuthenticationCtrl.scala | 30 ++- .../thp/thehive/controllers/v0/CaseCtrl.scala | 56 ++--- .../thehive/controllers/v0/CaseRenderer.scala | 10 +- .../controllers/v0/CaseTemplateCtrl.scala | 29 ++- .../thehive/controllers/v0/ConfigCtrl.scala | 22 +- .../thehive/controllers/v0/Conversion.scala | 177 ++++++++------- .../controllers/v0/CustomFieldCtrl.scala | 15 +- .../controllers/v0/DashboardCtrl.scala | 21 +- .../thp/thehive/controllers/v0/ListCtrl.scala | 30 +-- .../thp/thehive/controllers/v0/LogCtrl.scala | 17 +- .../controllers/v0/ObservableCtrl.scala | 26 +-- .../controllers/v0/ObservableTypeCtrl.scala | 11 +- .../controllers/v0/OrganisationCtrl.scala | 32 +-- .../thp/thehive/controllers/v0/PageCtrl.scala | 13 +- .../thehive/controllers/v0/ProfileCtrl.scala | 14 +- .../thehive/controllers/v0/QueryCtrl.scala | 5 +- .../thehive/controllers/v0/ShareCtrl.scala | 96 ++++---- .../thehive/controllers/v0/StatusCtrl.scala | 13 +- .../thp/thehive/controllers/v0/TagCtrl.scala | 10 +- .../thp/thehive/controllers/v0/TaskCtrl.scala | 23 +- .../controllers/v0/TheHiveQueryExecutor.scala | 10 +- .../thp/thehive/controllers/v0/UserCtrl.scala | 79 ++++--- .../thehive/controllers/v1/AlertCtrl.scala | 37 ++-- .../controllers/v1/AlertRenderer.scala | 10 +- .../thehive/controllers/v1/AuditCtrl.scala | 7 +- .../controllers/v1/AuthenticationCtrl.scala | 25 ++- .../thp/thehive/controllers/v1/CaseCtrl.scala | 24 +- .../thehive/controllers/v1/CaseRenderer.scala | 13 +- .../controllers/v1/CaseTemplateCtrl.scala | 13 +- .../thehive/controllers/v1/Conversion.scala | 36 ++- .../thp/thehive/controllers/v1/LogCtrl.scala | 15 +- .../thehive/controllers/v1/LogRenderer.scala | 4 +- .../controllers/v1/ObservableCtrl.scala | 18 +- .../controllers/v1/ObservableRenderer.scala | 9 +- .../controllers/v1/OrganisationCtrl.scala | 15 +- .../thehive/controllers/v1/ProfileCtrl.scala | 14 +- .../thehive/controllers/v1/Properties.scala | 77 ++++++- .../{QueryCtrl.scala => QueryableCtrl.scala} | 5 +- .../thp/thehive/controllers/v1/TaskCtrl.scala | 17 +- .../thehive/controllers/v1/TaskRenderer.scala | 9 +- .../thp/thehive/controllers/v1/UserCtrl.scala | 84 +++---- .../app/org/thp/thehive/models/Alert.scala | 6 +- .../app/org/thp/thehive/models/Audit.scala | 52 +++-- thehive/app/org/thp/thehive/models/Case.scala | 6 +- .../org/thp/thehive/models/CaseTemplate.scala | 4 +- .../org/thp/thehive/models/CustomField.scala | 111 +++++----- .../org/thp/thehive/models/Dashboard.scala | 4 +- thehive/app/org/thp/thehive/models/Log.scala | 4 +- .../org/thp/thehive/models/Observable.scala | 4 +- .../org/thp/thehive/models/Organisation.scala | 4 +- .../app/org/thp/thehive/models/Share.scala | 6 +- thehive/app/org/thp/thehive/models/Task.scala | 2 +- thehive/app/org/thp/thehive/models/User.scala | 4 +- .../org/thp/thehive/services/AlertSrv.scala | 90 ++++---- .../thp/thehive/services/AttachmentSrv.scala | 19 +- .../org/thp/thehive/services/AuditSrv.scala | 60 ++--- .../org/thp/thehive/services/CaseSrv.scala | 111 +++++----- .../thehive/services/CaseTemplateSrv.scala | 71 +++--- .../thp/thehive/services/ConfigContext.scala | 7 +- .../org/thp/thehive/services/ConfigSrv.scala | 68 +++--- .../thp/thehive/services/CustomFieldSrv.scala | 29 ++- .../thp/thehive/services/DashboardSrv.scala | 60 ++--- .../org/thp/thehive/services/DataSrv.scala | 19 +- .../org/thp/thehive/services/FlowActor.scala | 11 +- .../thehive/services/ImpactStatusSrv.scala | 33 ++- .../thehive/services/LocalKeyAuthSrv.scala | 10 +- .../services/LocalPasswordAuthSrv.scala | 22 +- .../thp/thehive/services/LocalUserSrv.scala | 20 +- .../app/org/thp/thehive/services/LogSrv.scala | 43 ++-- .../thp/thehive/services/ObservableSrv.scala | 79 +++---- .../thehive/services/ObservableTypeSrv.scala | 37 ++-- .../thehive/services/OrganisationSrv.scala | 58 +++-- .../org/thp/thehive/services/PageSrv.scala | 17 +- .../org/thp/thehive/services/ProfileSrv.scala | 37 ++-- .../services/ResolutionStatusSrv.scala | 33 ++- .../org/thp/thehive/services/ShareSrv.scala | 85 ++++---- .../org/thp/thehive/services/StreamSrv.scala | 12 +- .../thp/thehive/services/TOTPAuthSrv.scala | 16 +- .../app/org/thp/thehive/services/TagSrv.scala | 18 +- .../org/thp/thehive/services/TaskSrv.scala | 34 ++- .../org/thp/thehive/services/UserSrv.scala | 143 ++++++------ .../notification/NotificationActor.scala | 15 +- .../notification/notifiers/Template.scala | 8 +- .../notification/notifiers/Webhook.scala | 7 +- .../notification/triggers/LogInMyTask.scala | 6 +- .../notification/triggers/TaskAssigned.scala | 19 +- .../org/thp/thehive/DatabaseBuilder.scala | 40 ++-- .../controllers/v0/AlertCtrlTest.scala | 11 +- .../controllers/v0/AuditCtrlTest.scala | 8 +- .../thehive/controllers/v0/CaseCtrlTest.scala | 33 +-- .../controllers/v0/CaseTemplateCtrlTest.scala | 5 +- .../controllers/v0/CustomFieldCtrlTest.scala | 6 +- .../controllers/v0/DashboardCtrlTest.scala | 16 +- .../controllers/v0/ObservableCtrlTest.scala | 26 +-- .../controllers/v0/OrganisationCtrlTest.scala | 3 +- .../thehive/controllers/v0/PageCtrlTest.scala | 3 +- .../controllers/v0/ProfileCtrlTest.scala | 3 +- .../controllers/v0/ShareCtrlTest.scala | 21 +- .../controllers/v0/StreamCtrlTest.scala | 3 +- .../thehive/controllers/v0/TagCtrlTest.scala | 2 +- .../thehive/controllers/v0/UserCtrlTest.scala | 4 +- .../controllers/v1/AlertCtrlTest.scala | 6 +- .../thehive/controllers/v1/CaseCtrlTest.scala | 32 ++- .../controllers/v1/OrganisationCtrlTest.scala | 3 +- .../thehive/controllers/v1/UserCtrlTest.scala | 6 +- .../thp/thehive/services/AlertSrvTest.scala | 73 ++++--- .../thehive/services/AttachmentSrvTest.scala | 3 +- .../thp/thehive/services/AuditSrvTest.scala | 3 +- .../thp/thehive/services/CaseSrvTest.scala | 65 +++--- .../services/CaseTemplateSrvTest.scala | 31 +-- .../thp/thehive/services/ConfigSrvTest.scala | 9 +- .../thehive/services/CustomFieldSrvTest.scala | 5 +- .../thehive/services/DashboardSrvTest.scala | 15 +- .../thp/thehive/services/DataSrvTest.scala | 5 +- .../services/LocalPasswordAuthSrvTest.scala | 3 +- .../services/OrganisationSrvTest.scala | 3 +- .../thp/thehive/services/UserSrvTest.scala | 23 +- .../notifiers/NotificationTemplateTest.scala | 28 ++- .../triggers/AlertCreatedTest.scala | 9 +- .../triggers/TaskAssignedTest.scala | 7 +- .../data/CaseTemplateCustomField.json | 6 +- 165 files changed, 2235 insertions(+), 2031 deletions(-) rename cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/{CortexSchema.scala => CortexSchemaDefinition.scala} (100%) rename thehive/app/org/thp/thehive/controllers/v1/{QueryCtrl.scala => QueryableCtrl.scala} (79%) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala index c15453ff3e..5e3da7c817 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.connector.cortex.controllers.v0 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} @@ -57,7 +58,7 @@ class ActionCtrl @Inject() ( .asyncAuth { implicit request => val action: InputAction = request.body("action") val tryEntity = db.roTransaction { implicit graph => - entityHelper.get(toObjectType(action.objectType), action.objectId, Permissions.manageAction) + entityHelper.get(toObjectType(action.objectType), EntityIdOrName(action.objectId), Permissions.manageAction) } for { entity <- Future.fromTry(tryEntity) @@ -69,21 +70,21 @@ class ActionCtrl @Inject() ( entrypoint("get by entity") .authRoTransaction(db) { implicit request => implicit graph => for { - entity <- entityHelper.get(toObjectType(objectType), objectId, Permissions.manageAction) + entity <- entityHelper.get(toObjectType(objectType), EntityIdOrName(objectId), Permissions.manageAction) } yield Results.Ok(actionSrv.listForEntity(entity._id).toJson) } } @Singleton -class PublicAction @Inject() (actionSrv: ActionSrv) extends PublicData { +class PublicAction @Inject() (actionSrv: ActionSrv, @Named("with-thehive-schema") db: Database) extends PublicData { override val entityName: String = "action" override val initialQuery: Query = Query.init[Traversal.V[Action]]("listAction", (graph, authContext) => actionSrv.startTraversal(graph).visible(authContext)) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Action]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Action]]( "getAction", - FieldsParser[IdOrName], - (param, graph, authContext) => actionSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => actionSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Action], IteratorOutput]( "page", @@ -111,7 +112,7 @@ class PublicAction @Inject() (actionSrv: ActionSrv) extends PublicData { .property("objectType", UMapping.string)(_.select(_.context.domainMap(o => fromObjectType(o._label))).readonly) .property("status", UMapping.string)(_.field.readonly) .property("startDate", UMapping.date)(_.field.readonly) - .property("objectId", UMapping.id)(_.select(_.out[ActionContext]._id).readonly) + .property("objectId", db.idMapping)(_.select(_.out[ActionContext]._id).readonly) .property("responderName", UMapping.string.optional)(_.field.readonly) .property("cortexId", UMapping.string.optional)(_.field.readonly) .property("tlp", UMapping.int.optional)(_.field.readonly) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala index 570d51a968..8dea87e972 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala @@ -4,6 +4,7 @@ import java.util.zip.ZipFile import com.google.inject.name.Named import javax.inject.{Inject, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ @@ -12,9 +13,10 @@ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} import org.thp.thehive.connector.cortex.controllers.v0.Conversion._ import org.thp.thehive.connector.cortex.dto.v0.InputAnalyzerTemplate import org.thp.thehive.connector.cortex.models.AnalyzerTemplate +import org.thp.thehive.connector.cortex.services.AnalyzerTemplateOps._ import org.thp.thehive.connector.cortex.services.AnalyzerTemplateSrv import org.thp.thehive.controllers.v0.Conversion._ -import org.thp.thehive.controllers.v0.{IdOrName, OutputParam, PublicData, QueryCtrl} +import org.thp.thehive.controllers.v0.{OutputParam, PublicData, QueryCtrl} import org.thp.thehive.models.Permissions import play.api.libs.json.{JsFalse, JsObject, JsTrue} import play.api.mvc.{Action, AnyContent, Results} @@ -34,7 +36,7 @@ class AnalyzerTemplateCtrl @Inject() ( entrypoint("get content") .authTransaction(db) { _ => implicit graph => analyzerTemplateSrv - .getOrFail(id) + .getOrFail(EntityIdOrName(id)) .map(report => Results.Ok(report.content)) } @@ -69,7 +71,7 @@ class AnalyzerTemplateCtrl @Inject() ( entrypoint("delete template") .authPermittedTransaction(db, Permissions.manageAnalyzerTemplate) { implicit request => implicit graph => analyzerTemplateSrv - .get(id) + .get(EntityIdOrName(id)) .getOrFail("AnalyzerTemplate") .map { analyzerTemplate => analyzerTemplateSrv.remove(analyzerTemplate) @@ -84,7 +86,7 @@ class AnalyzerTemplateCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("template") for { - (templateSteps, _) <- analyzerTemplateSrv.update(_.getByIds(id), propertyUpdaters) + (templateSteps, _) <- analyzerTemplateSrv.update(_.get(EntityIdOrName(id)), propertyUpdaters) template <- templateSteps.getOrFail("AnalyzerTemplate") } yield Results.Ok(template.toJson) @@ -96,10 +98,10 @@ class PublicAnalyzerTemplate @Inject() (analyzerTemplateSrv: AnalyzerTemplateSrv override val entityName: String = "analyzerTemplate" override val initialQuery: Query = Query.init[Traversal.V[AnalyzerTemplate]]("listAnalyzerTemplate", (graph, _) => analyzerTemplateSrv.startTraversal(graph)) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[AnalyzerTemplate]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[AnalyzerTemplate]]( "getReportTemplate", - FieldsParser[IdOrName], - (param, graph, _) => analyzerTemplateSrv.get(param.idOrName)(graph) + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => analyzerTemplateSrv.get(idOrName)(graph) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[AnalyzerTemplate], IteratorOutput]( diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala index 57eab42eb1..2d14ae959c 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala @@ -18,7 +18,7 @@ object Conversion { .withFieldRenamed(_.workerName, _.responderName) .withFieldRenamed(_.workerDefinition, _.responderDefinition) .withFieldComputed(_.status, _.status.toString) - .withFieldComputed(_.objectId, _.context._id) + .withFieldComputed(_.objectId, _.context._id.toString) .withFieldComputed(_.objectType, _.context._label) .withFieldComputed(_.operations, a => JsArray(a.operations).toString) .withFieldComputed(_.report, _.report.map(_.toString).getOrElse("{}")) @@ -43,7 +43,7 @@ object Conversion { case r => r + ("success" -> JsFalse) } ) - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_.id, _._id.toString) .withFieldConst(_._type, "case_artifact_job") .withFieldConst(_.case_artifact, None) .transform @@ -69,7 +69,7 @@ object Conversion { case r => r + ("success" -> JsFalse) } ) - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_.id, _._id.toString) .withFieldConst(_._type, "case_artifact_job") .withFieldConst( _.case_artifact, @@ -86,7 +86,7 @@ object Conversion { at.asInstanceOf[AnalyzerTemplate] .into[OutputAnalyzerTemplate] .withFieldComputed(_.analyzerId, _.workerId) - .withFieldConst(_.id, at._id) + .withFieldConst(_.id, at._id.toString) .withFieldComputed(_.content, _.content) .transform ) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala index 36ebfe3e10..0b28a004ef 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala @@ -3,17 +3,18 @@ package org.thp.thehive.connector.cortex.controllers.v0 import com.google.inject.name.Named import javax.inject.{Inject, Singleton} import org.scalactic.Good -import org.thp.scalligraph.BadRequestError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.FieldsParser import org.thp.scalligraph.models._ import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{BadRequestError, EntityIdOrName} import org.thp.thehive.connector.cortex.models.Job import org.thp.thehive.connector.cortex.services.JobOps._ import org.thp.thehive.controllers.v0._ import org.thp.thehive.models.Observable +import org.thp.thehive.services.ObservableOps._ import scala.reflect.runtime.{universe => ru} @@ -66,7 +67,7 @@ class CortexParentIdInputFilter(parentId: String) extends InputQuery[Traversal.U authContext: AuthContext ): Traversal.Unk = if (traversalType =:= ru.typeOf[Traversal.V[Job]]) - traversal.asInstanceOf[Traversal.V[Job]].filter(_.observable.getByIds(parentId)).asInstanceOf[Traversal.Unk] + traversal.asInstanceOf[Traversal.V[Job]].filter(_.observable.get(EntityIdOrName(parentId))).asInstanceOf[Traversal.Unk] else throw BadRequestError(s"$traversalType hasn't parent") } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala index 6546b8948c..cabe1bccc0 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala @@ -7,13 +7,13 @@ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.scalligraph.{AuthorizationError, ErrorHandler} +import org.thp.scalligraph.{AuthorizationError, EntityIdOrName, ErrorHandler} import org.thp.thehive.connector.cortex.controllers.v0.Conversion._ import org.thp.thehive.connector.cortex.models.{Job, RichJob} import org.thp.thehive.connector.cortex.services.JobOps._ import org.thp.thehive.connector.cortex.services.JobSrv import org.thp.thehive.controllers.v0.Conversion._ -import org.thp.thehive.controllers.v0.{IdOrName, OutputParam, PublicData, QueryCtrl} +import org.thp.thehive.controllers.v0.{OutputParam, PublicData, QueryCtrl} import org.thp.thehive.models.{Permissions, RichCase, RichObservable} import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.ObservableSrv @@ -36,7 +36,7 @@ class JobCtrl @Inject() ( entrypoint("get job") .authRoTransaction(db) { implicit request => implicit graph => jobSrv - .getByIds(jobId) + .get(EntityIdOrName(jobId)) .visible .richJob .getOrFail("Job") @@ -55,8 +55,8 @@ class JobCtrl @Inject() ( db.roTransaction { implicit graph => val artifactId: String = request.body("artifactId") for { - o <- observableSrv.getByIds(artifactId).richObservable.getOrFail("Observable") - c <- observableSrv.getByIds(artifactId).`case`.getOrFail("Case") + o <- observableSrv.get(EntityIdOrName(artifactId)).richObservable.getOrFail("Observable") + c <- observableSrv.get(EntityIdOrName(artifactId)).`case`.getOrFail("Case") } yield (o, c) }.fold( error => errorHandler.onServerError(request, error), @@ -76,10 +76,10 @@ class PublicJob @Inject() (jobSrv: JobSrv) extends PublicData with JobRenderer { override val entityName: String = "job" override val initialQuery: Query = Query.init[Traversal.V[Job]]("listJob", (graph, authContext) => jobSrv.startTraversal(graph).visible(authContext)) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Job]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Job]]( "getJob", - FieldsParser[IdOrName], - (param, graph, authContext) => jobSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => jobSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Job], IteratorOutput]( diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ResponderCtrl.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ResponderCtrl.scala index 70d8a82503..17ab8b76e3 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ResponderCtrl.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ResponderCtrl.scala @@ -2,6 +2,7 @@ package org.thp.thehive.connector.cortex.controllers.v0 import com.google.inject.name.Named import javax.inject.{Inject, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.thehive.connector.cortex.controllers.v0.Conversion._ @@ -24,7 +25,7 @@ class ResponderCtrl @Inject() ( entrypoint("get responders") .asyncAuth { implicit req => responderSrv - .getRespondersByType(entityType, entityId) + .getRespondersByType(entityType, EntityIdOrName(entityId)) .map(l => Results.Ok(l.toSeq.toJson)) } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Router.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Router.scala index 7968aa17d4..7d66ba446c 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Router.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Router.scala @@ -10,7 +10,6 @@ class Router @Inject() ( val jobCtrl: JobCtrl, analyzerCtrl: AnalyzerCtrl, val actionCtrl: ActionCtrl, - cortexQueryExecutor: CortexQueryExecutor, val reportCtrl: AnalyzerTemplateCtrl, responderCtrl: ResponderCtrl ) extends SimpleRouter { diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Action.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Action.scala index 75bce52a40..3fe038f67c 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Action.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Action.scala @@ -3,9 +3,9 @@ package org.thp.thehive.connector.cortex.models import java.util.Date import org.apache.tinkerpop.gremlin.structure.{Edge, Graph, Vertex} -import org.thp.scalligraph.BuildVertexEntity import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.Converter +import org.thp.scalligraph.{BuildVertexEntity, EntityId} import play.api.libs.json.JsObject @BuildVertexEntity @@ -24,7 +24,7 @@ case class Action( ) case class RichAction(action: Action with Entity, context: Product with Entity) { - def _id: String = action._id + def _id: EntityId = action._id def _createdAt: Date = action._createdAt def _createdBy: String = action._createdBy def workerId: String = action.workerId @@ -51,7 +51,7 @@ object ActionContext extends HasModel { override val fields: Map[String, Mapping[_, _, _]] = Map.empty override val converter: Converter[EEntity, Edge] = (element: Edge) => new ActionContext with Entity { - override val _id: String = element.id().toString + override val _id: EntityId = EntityId(element.id()) override val _label: String = "ActionContext" override val _createdBy: String = UMapping.string.getProperty(element, "_createdBy") override val _updatedBy: Option[String] = UMapping.string.optional.getProperty(element, "_updatedBy") @@ -60,7 +60,7 @@ object ActionContext extends HasModel { } override def addEntity(a: ActionContext, entity: Entity): EEntity = new ActionContext with Entity { - override def _id: String = entity._id + override def _id: EntityId = entity._id override def _label: String = entity._label override def _createdBy: String = entity._createdBy override def _updatedBy: Option[String] = entity._updatedBy diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchema.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchemaDefinition.scala similarity index 100% rename from cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchema.scala rename to cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchemaDefinition.scala diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala index a94fbff6b7..e376e97cab 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala @@ -3,7 +3,7 @@ package org.thp.thehive.connector.cortex.models import java.util.Date import org.thp.scalligraph.models.Entity -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} import org.thp.thehive.models.{Observable, RichObservable} import play.api.libs.json.{Format, JsObject, Json} @@ -36,7 +36,7 @@ case class RichJob( job: Job with Entity, observables: Seq[(RichObservable, JsObject)] ) { - def _id: String = job._id + def _id: EntityId = job._id def _createdBy: String = job._createdBy def _updatedBy: Option[String] = job._updatedBy def _createdAt: Date = job._createdAt diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala index 7f35beb34b..69cf1d92fc 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala @@ -4,10 +4,10 @@ import java.util.Date import javax.inject.Inject import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.InternalError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.Entity import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{EntityIdOrName, InternalError} import org.thp.thehive.connector.cortex.models._ import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputTask @@ -41,8 +41,8 @@ class ActionOperationSrv @Inject() ( * @param authContext auth for access check * @return */ - def execute(entity: Entity, operation: ActionOperation, relatedCase: Option[Case with Entity], relatedTask: Option[Task with Entity])( - implicit graph: Graph, + def execute(entity: Entity, operation: ActionOperation, relatedCase: Option[Case with Entity], relatedTask: Option[Task with Entity])(implicit + graph: Graph, authContext: AuthContext ): Try[ActionOperationStatus] = { @@ -72,7 +72,7 @@ class ActionOperationSrv @Inject() ( case AddCustomFields(name, _, value) => for { c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddCustomFields without case")))(Success(_)) - _ <- caseSrv.setOrCreateCustomField(c, name, Some(value), None) + _ <- caseSrv.setOrCreateCustomField(c, EntityIdOrName(name), Some(value), None) } yield updateOperation(operation) case CloseTask() => @@ -96,7 +96,7 @@ class ActionOperationSrv @Inject() ( case AddArtifactToCase(_, dataType, dataMessage) => for { c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddArtifactToCase without case")))(Success(_)) - obsType <- observableTypeSrv.getOrFail(dataType) + obsType <- observableTypeSrv.getOrFail(EntityIdOrName(dataType)) richObservable <- observableSrv.create( Observable(Some(dataMessage), 2, ioc = false, sighted = false), obsType, @@ -110,8 +110,8 @@ class ActionOperationSrv @Inject() ( case AssignCase(owner) => for { c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AssignCase without case")))(Success(_)) - u <- userSrv.get(owner).getOrFail("User") - _ <- Try(caseSrv.startTraversal.getByIds(c._id).unassign()) + u <- userSrv.get(EntityIdOrName(owner)).getOrFail("User") + _ <- Try(caseSrv.startTraversal.getEntity(c).unassign()) _ <- caseSrv.assign(c, u) } yield updateOperation(operation) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala index f5f29ca018..1ad5e48d03 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala @@ -8,12 +8,12 @@ import javax.inject.Inject import org.apache.tinkerpop.gremlin.structure.{Element, Graph} import org.thp.cortex.client.CortexClient import org.thp.cortex.dto.v0.{InputAction => CortexAction, OutputJob => CortexJob} -import org.thp.scalligraph.NotFoundError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} +import org.thp.scalligraph.{EntityId, NotFoundError} import org.thp.thehive.connector.cortex.controllers.v0.Conversion._ import org.thp.thehive.connector.cortex.models._ import org.thp.thehive.connector.cortex.services.ActionOps._ @@ -137,7 +137,7 @@ class ActionSrv @Inject() ( * @param authContext context for db queries * @return */ - def finished(actionId: String, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Action with Entity] = + def finished(actionId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Action with Entity] = db.tryTransaction { implicit graph => getByIds(actionId).richAction.getOrFail("Action").flatMap { action => val operations: Seq[ActionOperationStatus] = cortexJob @@ -189,20 +189,20 @@ class ActionSrv @Inject() ( * @param graph db graph * @return */ - def relatedCase(id: String)(implicit graph: Graph): Option[Case with Entity] = + def relatedCase(id: EntityId)(implicit graph: Graph): Option[Case with Entity] = for { richAction <- startTraversal.getByIds(id).richAction.getOrFail("Action").toOption relatedCase <- entityHelper.parentCase(richAction.context) } yield relatedCase - def relatedTask(id: String)(implicit graph: Graph): Option[Task with Entity] = + def relatedTask(id: EntityId)(implicit graph: Graph): Option[Task with Entity] = for { richAction <- startTraversal.getByIds(id).richAction.getOrFail("Action").toOption relatedTask <- entityHelper.parentTask(richAction.context) } yield relatedTask // TODO to be tested - def listForEntity(id: String)(implicit graph: Graph): Seq[RichAction] = startTraversal.forEntity(id).richAction.toSeq + def listForEntity(id: EntityId)(implicit graph: Graph): Seq[RichAction] = startTraversal.forEntity(id).richAction.toSeq } object ActionOps { @@ -225,7 +225,7 @@ object ActionOps { RichAction(action, context) } - def forEntity(entityId: String): Traversal.V[Action] = + def forEntity(entityId: EntityId): Traversal.V[Action] = traversal.filter(_.out[ActionContext].hasId(entityId)) def context: Traversal[Product with Entity, Element, Converter[Product with Entity, Element]] = traversal.out[ActionContext].entity diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerTemplateSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerTemplateSrv.scala index f78b33c797..0467ddd4c2 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerTemplateSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerTemplateSrv.scala @@ -5,13 +5,13 @@ import java.util.zip.{ZipEntry, ZipFile} import com.google.inject.name.Named import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.CreateError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{CreateError, EntityIdOrName, EntityName} import org.thp.thehive.connector.cortex.controllers.v0.Conversion._ import org.thp.thehive.connector.cortex.models.AnalyzerTemplate import org.thp.thehive.connector.cortex.services.AnalyzerTemplateOps._ @@ -24,15 +24,14 @@ import scala.io.Source import scala.util.{Failure, Try} @Singleton -class AnalyzerTemplateSrv @Inject() ( - implicit @Named("with-thehive-cortex-schema") db: Database, +class AnalyzerTemplateSrv @Inject() (implicit + @Named("with-thehive-cortex-schema") db: Database, auditSrv: CortexAuditSrv, organisationSrv: OrganisationSrv ) extends VertexSrv[AnalyzerTemplate] { - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[AnalyzerTemplate] = - if (db.isValidId(idOrName)) getByIds(idOrName) - else startTraversal.getByAnalyzerId(idOrName) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[AnalyzerTemplate] = + startTraversal.getByAnalyzerId(name) def readZipEntry(file: ZipFile, entry: ZipEntry): Try[String] = Try { @@ -84,7 +83,7 @@ class AnalyzerTemplateSrv @Inject() ( .toSeq .groupBy(_.getName.takeWhile(c => c != '/' && c != '.')) .flatMap { - case (name, entries) if entries.lengthCompare(1) == 0 => List(name -> entries.head) + case (name, entries) if entries.lengthCompare(1) == 0 => List(name -> entries.head) case (name, entries) => entries.filterNot(_.getName.endsWith("short.html")).headOption.map(name -> _) } .foldLeft(Map.empty[String, Try[AnalyzerTemplate with Entity]]) { @@ -94,7 +93,7 @@ class AnalyzerTemplateSrv @Inject() ( .flatMap { content => db.tryTransaction { implicit graph => (for { - updated <- get(analyzerId).update(_.content, content).getOrFail("AnalyzerTemplate") + updated <- get(EntityName(analyzerId)).update(_.content, content).getOrFail("AnalyzerTemplate") _ <- auditSrv.analyzerTemplate.update(updated, Json.obj("content" -> content)) } yield updated).recoverWith { case _ => @@ -112,9 +111,8 @@ class AnalyzerTemplateSrv @Inject() ( object AnalyzerTemplateOps { implicit class AnalyzerTemplateOpsDefs(traversal: Traversal.V[AnalyzerTemplate]) { - def get(idOrAnalyzerId: String)(implicit db: Database): Traversal.V[AnalyzerTemplate] = - if (db.isValidId(idOrAnalyzerId)) traversal.getByIds(idOrAnalyzerId) - else getByAnalyzerId(idOrAnalyzerId) + def get(idOrAnalyzerId: EntityIdOrName): Traversal.V[AnalyzerTemplate] = + idOrAnalyzerId.fold(traversal.getByIds(_), getByAnalyzerId) /** * Looks for a template that has the workerId supplied @@ -122,6 +120,6 @@ object AnalyzerTemplateOps { * @param workerId the id to look for * @return */ - def getByAnalyzerId(workerId: String): Traversal.V[AnalyzerTemplate] = traversal.has("workerId", workerId).v[AnalyzerTemplate] + def getByAnalyzerId(workerId: String): Traversal.V[AnalyzerTemplate] = traversal.has(_.workerId, workerId) } } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/CortexActor.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/CortexActor.scala index adfb16a6b3..fef75d6f44 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/CortexActor.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/CortexActor.scala @@ -7,6 +7,7 @@ import akka.pattern.pipe import javax.inject.Inject import org.thp.client.ApplicationError import org.thp.cortex.dto.v0.{JobStatus, JobType, OutputJob => CortexJob} +import org.thp.scalligraph.EntityId import org.thp.scalligraph.auth.AuthContext import play.api.Logger @@ -15,9 +16,9 @@ import scala.concurrent.duration._ object CortexActor { final case class CheckJob( - jobId: Option[String], + jobId: Option[EntityId], cortexJobId: String, - actionId: Option[String], + actionId: Option[EntityId], cortexId: String, authContext: AuthContext ) @@ -45,9 +46,8 @@ class CortexActor @Inject() (connector: Connector, jobSrv: JobSrv, actionSrv: Ac case cj @ CheckJob(jobId, cortexJobId, actionId, cortexId, _) => logger.info(s"CortexActor received job or action (${jobId.getOrElse(actionId.get)}, $cortexJobId, $cortexId) to check, added to $checkedJobs") - if (!timers.isTimerActive(CheckJobsKey)) { + if (!timers.isTimerActive(CheckJobsKey)) timers.startSingleTimer(CheckJobsKey, FirstCheckJobs, 500.millis) - } context.become(receive(cj :: checkedJobs, failuresCount)) case CheckJobs if checkedJobs.isEmpty => diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/EntityHelper.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/EntityHelper.scala index e0133deca7..9e767333f8 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/EntityHelper.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/EntityHelper.scala @@ -2,10 +2,10 @@ package org.thp.thehive.connector.cortex.services import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.BadRequestError import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.models.Entity import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{BadRequestError, EntityIdOrName} import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseOps._ @@ -38,13 +38,16 @@ class EntityHelper @Inject() ( * @param authContext auth for permission check * @return */ - def get(objectType: String, objectId: String, permission: Permission)(implicit graph: Graph, authContext: AuthContext): Try[Product with Entity] = + def get(objectType: String, objectId: EntityIdOrName, permission: Permission)(implicit + graph: Graph, + authContext: AuthContext + ): Try[Product with Entity] = objectType match { - case "Task" => taskSrv.getByIds(objectId).can(permission).getOrFail("Task") + case "Task" => taskSrv.get(objectId).can(permission).getOrFail("Task") case "Case" => caseSrv.get(objectId).can(permission).getOrFail("Case") - case "Observable" => observableSrv.getByIds(objectId).can(permission).getOrFail("Observable") - case "Log" => logSrv.getByIds(objectId).can(permission).getOrFail("Log") - case "Alert" => alertSrv.getByIds(objectId).can(permission).getOrFail("Alert") + case "Observable" => observableSrv.get(objectId).can(permission).getOrFail("Observable") + case "Log" => logSrv.get(objectId).can(permission).getOrFail("Log") + case "Alert" => alertSrv.get(objectId).can(permission).getOrFail("Alert") case _ => Failure(BadRequestError(s"objectType $objectType is not recognised")) } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala index 9ca07d3afa..85845bedcc 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala @@ -14,13 +14,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.cortex.client.CortexClient import org.thp.cortex.dto.v0.{InputArtifact, OutputArtifact, Attachment => CortexAttachment, OutputJob => CortexJob} -import org.thp.scalligraph.NotFoundError import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} +import org.thp.scalligraph.{EntityId, EntityIdOrName, NotFoundError} import org.thp.thehive.connector.cortex.controllers.v0.Conversion._ import org.thp.thehive.connector.cortex.models._ import org.thp.thehive.connector.cortex.services.Conversion._ @@ -30,6 +30,7 @@ import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.{AttachmentSrv, ObservableSrv, ObservableTypeSrv, ReportTagSrv} import play.api.libs.json.Json @@ -65,24 +66,27 @@ class JobSrv @Inject() ( * @param authContext auth context instance * @return */ - def submit(cortexId: String, workerId: String, observable: RichObservable, `case`: Case with Entity)( - implicit authContext: AuthContext + def submit(cortexId: String, workerId: String, observable: RichObservable, `case`: Case with Entity)(implicit + authContext: AuthContext ): Future[RichJob] = for { - cortexClient <- serviceHelper - .availableCortexClients(connector.clients, authContext.organisation) - .find(_.name == cortexId) - .fold[Future[CortexClient]](Future.failed(NotFoundError(s"Cortex $cortexId not found")))(Future.successful) - analyzer <- cortexClient.getAnalyzer(workerId).recoverWith { case _ => cortexClient.getAnalyzerByName(workerId) } // if get analyzer using cortex2 API fails, try using legacy API + cortexClient <- + serviceHelper + .availableCortexClients(connector.clients, authContext.organisation) + .find(_.name == cortexId) + .fold[Future[CortexClient]](Future.failed(NotFoundError(s"Cortex $cortexId not found")))(Future.successful) + analyzer <- cortexClient.getAnalyzer(workerId).recoverWith { + case _ => cortexClient.getAnalyzerByName(workerId) + } // if get analyzer using cortex2 API fails, try using legacy API cortexArtifact <- (observable.attachment, observable.data) match { case (None, Some(data)) => Future.successful( - InputArtifact(observable.tlp, `case`.pap, observable.`type`.name, `case`._id, Some(data.data), None) + InputArtifact(observable.tlp, `case`.pap, observable.`type`.name, `case`.number.toString, Some(data.data), None) ) case (Some(a), None) => val attachment = CortexAttachment(a.name, a.size, a.contentType, attachmentSrv.source(a)) Future.successful( - InputArtifact(observable.tlp, `case`.pap, observable.`type`.name, `case`._id, None, Some(attachment)) + InputArtifact(observable.tlp, `case`.pap, observable.`type`.name, `case`.number.toString, None, Some(attachment)) ) case _ => Future.failed(new Exception(s"Invalid Observable data for ${observable.observable._id}")) } @@ -135,14 +139,15 @@ class JobSrv @Inject() ( * @param authContext the auth context for db queries * @return the updated job */ - def finished(cortexId: String, jobId: String, cortexJob: CortexJob)( - implicit authContext: AuthContext + def finished(cortexId: String, jobId: EntityId, cortexJob: CortexJob)(implicit + authContext: AuthContext ): Future[Job with Entity] = for { - cortexClient <- serviceHelper - .availableCortexClients(connector.clients, authContext.organisation) - .find(_.name == cortexId) - .fold[Future[CortexClient]](Future.failed(NotFoundError(s"Cortex $cortexId not found")))(Future.successful) + cortexClient <- + serviceHelper + .availableCortexClients(connector.clients, authContext.organisation) + .find(_.name == cortexId) + .fold[Future[CortexClient]](Future.failed(NotFoundError(s"Cortex $cortexId not found")))(Future.successful) job <- Future.fromTry(updateJobStatus(jobId, cortexJob)) _ <- importCortexArtifacts(job, cortexJob, cortexClient) _ <- Future.fromTry(importAnalyzerTags(job, cortexJob)) @@ -156,7 +161,7 @@ class JobSrv @Inject() ( * @param authContext the authentication context * @return the updated job */ - private def updateJobStatus(jobId: String, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Job with Entity] = + private def updateJobStatus(jobId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Job with Entity] = db.tryTransaction { implicit graph => getOrFail(jobId).flatMap { job => val report = cortexJob.report.flatMap(r => r.full orElse r.errorMessage.map(m => Json.obj("errorMessage" -> m))) @@ -192,8 +197,8 @@ class JobSrv @Inject() ( * @param authContext the authentication context * @return */ - private def importCortexArtifacts(job: Job with Entity, cortexJob: CortexJob, cortexClient: CortexClient)( - implicit authContext: AuthContext + private def importCortexArtifacts(job: Job with Entity, cortexJob: CortexJob, cortexClient: CortexClient)(implicit + authContext: AuthContext ): Future[Done] = { val artifacts = cortexJob .report @@ -201,7 +206,7 @@ class JobSrv @Inject() ( .flatMap(_.artifacts) Future .traverse(artifacts) { artifact => - db.tryTransaction(graph => observableTypeSrv.getOrFail(artifact.dataType)(graph)) match { + db.tryTransaction(graph => observableTypeSrv.getOrFail(EntityIdOrName(artifact.dataType))(graph)) match { case Success(attachmentType) if attachmentType.isAttachment => importCortexAttachment(job, artifact, attachmentType, cortexClient) case Success(dataType) => Future @@ -239,8 +244,8 @@ class JobSrv @Inject() ( artifact: OutputArtifact, attachmentType: ObservableType with Entity, cortexClient: CortexClient - )( - implicit authContext: AuthContext + )(implicit + authContext: AuthContext ): Future[Attachment with Entity] = artifact .attachment @@ -277,12 +282,7 @@ object JobOps { * @return */ def visible(implicit authContext: AuthContext): Traversal.V[Job] = - traversal.filter( - _.in[ObservableJob] - .in[ShareObservable] - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.observable.organisations.visible) /** * Checks if a job is accessible if the user and @@ -294,13 +294,7 @@ object JobOps { */ def can(permission: Permission)(implicit authContext: AuthContext): Traversal.V[Job] = if (authContext.permissions.contains(permission)) - traversal.filter( - _.in[ObservableJob] - .in[ShareObservable] - .filter(_.out[ShareProfile].has("permissions", permission)) - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.observable.can(permission)) else traversal.limit(0) def observable: Traversal.V[Observable] = traversal.in[ObservableJob].v[Observable] @@ -313,7 +307,7 @@ object JobOps { */ def reportObservables: Traversal.V[Observable] = traversal.out[ReportObservable].v[Observable] - def richJob(implicit authContext: AuthContext) = { + def richJob(implicit authContext: AuthContext): Traversal[RichJob, JMap[String, Any], Converter[RichJob, JMap[String, Any]]] = { val thisJob = StepLabel.v[Job] traversal .as(thisJob) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala index f4f6c1d133..e350a2c8f6 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala @@ -3,6 +3,7 @@ package org.thp.thehive.connector.cortex.services import com.google.inject.name.Named import javax.inject.{Inject, Singleton} import org.thp.cortex.dto.v0.OutputWorker +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.Database import org.thp.thehive.controllers.v0.Conversion.toObjectType @@ -36,22 +37,23 @@ class ResponderSrv @Inject() ( */ def getRespondersByType( entityType: String, - entityId: String + entityId: EntityIdOrName )(implicit authContext: AuthContext): Future[Map[OutputWorker, Seq[String]]] = for { entity <- Future.fromTry(db.roTransaction(implicit graph => entityHelper.get(toObjectType(entityType), entityId, Permissions.manageAction))) (_, tlp, pap) <- Future.fromTry(db.roTransaction(implicit graph => entityHelper.entityInfo(entity))) - responders <- Future - .traverse(serviceHelper.availableCortexClients(connector.clients, authContext.organisation))(client => - client - .getRespondersByType(entityType) - .transform { - case Success(analyzers) => Success(analyzers.map(_ -> client.name)) - case Failure(error) => - logger.error(s"List Cortex analyzers fails on ${client.name}", error) - Success(Nil) - } - ) + responders <- + Future + .traverse(serviceHelper.availableCortexClients(connector.clients, authContext.organisation))(client => + client + .getRespondersByType(entityType) + .transform { + case Success(analyzers) => Success(analyzers.map(_ -> client.name)) + case Failure(error) => + logger.error(s"List Cortex analyzers fails on ${client.name}", error) + Success(Nil) + } + ) } yield serviceHelper.flattenList(responders).filter { case (w, _) => w.maxTlp >= tlp && w.maxPap >= pap } /** diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ServiceHelper.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ServiceHelper.scala index 15b77720ce..0ce824fc26 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ServiceHelper.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ServiceHelper.scala @@ -5,10 +5,12 @@ import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.cortex.client.CortexClient import org.thp.cortex.dto.v0.OutputWorker +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.models.Organisation +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services._ import play.api.Logger @@ -26,23 +28,22 @@ class ServiceHelper @Inject() ( * @param organisationName the concerned organisation to get available clients for * @return */ - def availableCortexClients(clients: Seq[CortexClient], organisationName: String): Iterable[CortexClient] = db.roTransaction { implicit graph => - val l = clients - .filter(c => - organisationFilter( - organisationSrv.startTraversal, - c.includedTheHiveOrganisations, - c.excludedTheHiveOrganisations - ).toList - .exists(_.name == organisationName) - ) + def availableCortexClients(clients: Seq[CortexClient], organisationName: EntityIdOrName): Iterable[CortexClient] = + db.roTransaction { implicit graph => + val l = clients + .filter(c => + organisationFilter( + organisationSrv.startTraversal, + c.includedTheHiveOrganisations, + c.excludedTheHiveOrganisations + ).get(organisationName).exists + ) - if (l.isEmpty) { - logger.warn(s"No CortexClient found for organisation $organisationName in list ${clients.map(_.name)}") - } + if (l.isEmpty) + logger.warn(s"No CortexClient found for organisation $organisationName in list ${clients.map(_.name)}") - l - } + l + } /** * Returns the filtered organisations according to the supplied lists (mainly conf based) @@ -59,9 +60,9 @@ class ServiceHelper @Inject() ( ): Traversal.V[Organisation] = { val includedOrgs = if (includedTheHiveOrganisations.contains("*") || includedTheHiveOrganisations.isEmpty) organisationSteps - else organisationSteps.has("name", P.within(includedTheHiveOrganisations)) + else organisationSteps.has(_.name, P.within(includedTheHiveOrganisations: _*)) if (excludedTheHiveOrganisations.isEmpty) includedOrgs - else includedOrgs.has("name", P.without(excludedTheHiveOrganisations)) + else includedOrgs.has(_.name, P.without(excludedTheHiveOrganisations: _*)) } /** @@ -75,7 +76,7 @@ class ServiceHelper @Inject() ( // f: ((OutputWorker, Seq[String])) => Boolean ): Map[OutputWorker, Seq[String]] = l // Iterable[Seq[(worker, cortexId)]] - .flatten // Seq[(worker, cortexId)] + .flatten // Seq[(worker, cortexId)] .groupBy(_._1.name) // Map[workerName, Seq[(worker, cortexId)]] .values // Seq[Seq[(worker, cortexId)]] .map(a => a.head._1 -> a.map(_._2).toSeq) // Map[worker, Seq[CortexId] ] diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrlTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrlTest.scala index c62a6153b5..15ffbdbb95 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrlTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrlTest.scala @@ -29,7 +29,7 @@ class JobCtrlTest extends PlaySpecification with TestAppBuilder { "job controller" should { "get a job" in testApp { app => val observable = app[Database].roTransaction { implicit graph => - app[ObservableSrv].startTraversal.has("message", "Some weird domain").getOrFail("Observable").get + app[ObservableSrv].startTraversal.has(_.message, "Some weird domain").getOrFail("Observable").get } val requestSearch = FakeRequest("POST", s"/api/connector/cortex/job/_search?range=0-200&sort=-startDate") diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala index c6aba2221d..e0c2aabcb9 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala @@ -4,10 +4,10 @@ import java.util.Date import org.thp.cortex.client.{CortexClient, TestCortexClientProvider} import org.thp.cortex.dto.v0.OutputJob -import org.thp.scalligraph.AppBuilder import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv, Schema} import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{AppBuilder, EntityName} import org.thp.thehive.connector.cortex.models.{Job, JobStatus, TheHiveCortexSchemaProvider} import org.thp.thehive.connector.cortex.services.JobOps._ import org.thp.thehive.models.Permissions @@ -86,10 +86,10 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder { "submit a job" in testApp { app => val x = for { observable <- app[Database].roTransaction { implicit graph => - app[ObservableSrv].startTraversal.has("message", "Some weird domain").richObservable.getOrFail("Observable") + app[ObservableSrv].startTraversal.has(_.message, "Some weird domain").richObservable.getOrFail("Observable") } case0 <- app[Database].roTransaction { implicit graph => - app[CaseSrv].getOrFail("#1") + app[CaseSrv].getOrFail(EntityName("1")) } } yield await(app[JobSrv].submit("test", "anaTest1", observable, case0)) diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ServiceHelperTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ServiceHelperTest.scala index 1891d571de..99051a0cb8 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ServiceHelperTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ServiceHelperTest.scala @@ -1,9 +1,9 @@ package org.thp.thehive.connector.cortex.services import org.thp.cortex.client.{CortexClient, TestCortexClientProvider} -import org.thp.scalligraph.AppBuilder import org.thp.scalligraph.models.{Database, Schema} import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{AppBuilder, EntityName} import org.thp.thehive.connector.cortex.models.TheHiveCortexSchemaProvider import org.thp.thehive.models.Organisation import org.thp.thehive.services._ @@ -51,7 +51,7 @@ class ServiceHelperTest extends PlaySpecification with TestAppBuilder { "return the correct filtered CortexClient list" in testApp { app => val client = app[CortexClient] - val r = app[ServiceHelper].availableCortexClients(Seq(client), Organisation.administration.name) + val r = app[ServiceHelper].availableCortexClients(Seq(client), EntityName(Organisation.administration.name)) r must contain(client) } diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Audit.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Audit.scala index c8391d8242..79889dd505 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Audit.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Audit.scala @@ -10,14 +10,15 @@ case class OutputEntity(_type: String, _id: String, _createdAt: Date, _createdBy object OutputEntity { implicit val format: OFormat[OutputEntity] = Json.format[OutputEntity] - def apply(e: Entity): OutputEntity = OutputEntity( - e._label, - e._id, - e._createdAt, - e._createdBy, - e._updatedAt, - e._updatedBy - ) + def apply(e: Entity): OutputEntity = + OutputEntity( + e._label, + e._id.toString, + e._createdAt, + e._createdBy, + e._updatedAt, + e._updatedBy + ) } case class OutputAudit( diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Alert.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Alert.scala index 37fb1ac05c..994b315bcd 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Alert.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Alert.scala @@ -46,7 +46,7 @@ case class OutputAlert( pap: Int, read: Boolean, follow: Boolean, - customFields: Set[OutputCustomFieldValue] = Set.empty, + customFields: Seq[OutputCustomFieldValue] = Seq.empty, caseTemplate: Option[String] = None, observableCount: Long, caseId: Option[String], @@ -75,7 +75,7 @@ object OutputAlert { pap <- (json \ "pap").validate[Int] read <- (json \ "read").validate[Boolean] follow <- (json \ "follow").validate[Boolean] - customFields <- (json \ "customFields").validate[Set[OutputCustomFieldValue]] + customFields <- (json \ "customFields").validate[Seq[OutputCustomFieldValue]] caseTemplate <- (json \ "caseTemplate").validateOpt[String] observableCount <- (json \ "observableCount").validate[Long] caseId <- (json \ "caseId").validateOpt[String] diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Audit.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Audit.scala index 1a32e9c06b..318d92ea17 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Audit.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Audit.scala @@ -10,14 +10,15 @@ case class OutputEntity(_type: String, _id: String, _createdAt: Date, _createdBy object OutputEntity { implicit val format: OFormat[OutputEntity] = Json.format[OutputEntity] - def apply(e: Entity): OutputEntity = OutputEntity( - e._label, - e._id, - e._createdAt, - e._createdBy, - e._updatedAt, - e._updatedBy - ) + def apply(e: Entity): OutputEntity = + OutputEntity( + e._label, + e._id.toString, + e._createdAt, + e._createdBy, + e._updatedAt, + e._updatedBy + ) } case class OutputAudit( diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala index f4a75c1c04..464a15864e 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala @@ -48,7 +48,7 @@ case class OutputCase( impactStatus: Option[String] = None, resolutionStatus: Option[String] = None, assignee: Option[String], - customFields: Set[OutputCustomFieldValue] = Set.empty, + customFields: Seq[OutputCustomFieldValue] = Seq.empty, extraData: JsObject ) @@ -77,7 +77,7 @@ object OutputCase { impactStatus <- (json \ "impactStatus").validateOpt[String] resolutionStatus <- (json \ "resolutionStatus").validateOpt[String] assignee <- (json \ "assignee").validateOpt[String] - customFields <- (json \ "customFields").validate[Set[OutputCustomFieldValue]] + customFields <- (json \ "customFields").validate[Seq[OutputCustomFieldValue]] extraData <- (json \ "extraData").validate[JsObject] } yield OutputCase( _id, diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala index c43c0496ac..06d942a849 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CaseTemplate.scala @@ -42,7 +42,7 @@ case class OutputCaseTemplate( tlp: Option[Int], pap: Option[Int], summary: Option[String], - customFields: Set[OutputCustomFieldValue] = Set.empty, + customFields: Seq[OutputCustomFieldValue] = Seq.empty, tasks: Seq[OutputTask] ) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index 7be924aa0d..095142e98d 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -71,7 +71,7 @@ object InputCustomFieldValue { } } -case class OutputCustomFieldValue(name: String, description: String, `type`: String, value: JsValue, order: Int) +case class OutputCustomFieldValue(_id: String, name: String, description: String, `type`: String, value: JsValue, order: Int) object OutputCustomFieldValue { implicit val format: OFormat[OutputCustomFieldValue] = Json.format[OutputCustomFieldValue] diff --git a/migration/src/main/scala/org/thp/thehive/migration/IdMapping.scala b/migration/src/main/scala/org/thp/thehive/migration/IdMapping.scala index 310af63848..dcadd3928c 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/IdMapping.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/IdMapping.scala @@ -1,3 +1,5 @@ package org.thp.thehive.migration -case class IdMapping(inputId: String, outputId: String) +import org.thp.scalligraph.EntityId + +case class IdMapping(inputId: String, outputId: EntityId) diff --git a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala index 47f8f173e3..aa12e5990f 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala @@ -3,7 +3,7 @@ package org.thp.thehive.migration import akka.NotUsed import akka.stream.Materializer import akka.stream.scaladsl.{Sink, Source} -import org.thp.scalligraph.{NotFoundError, RichOptionTry} +import org.thp.scalligraph.{EntityId, NotFoundError, RichOptionTry} import org.thp.thehive.migration.dto.{InputAlert, InputAudit, InputCase, InputCaseTemplate} import play.api.Logger @@ -124,14 +124,14 @@ trait MigrationOps { implicit class IdMappingOpsDefs(idMappings: Seq[IdMapping]) { - def fromInput(id: String): Try[String] = + def fromInput(id: String): Try[EntityId] = idMappings .find(_.inputId == id) - .fold[Try[String]](Failure(NotFoundError(s"Id $id not found")))(m => Success(m.outputId)) + .fold[Try[EntityId]](Failure(NotFoundError(s"Id $id not found")))(m => Success(m.outputId)) } - def migrate[A](name: String, source: Source[Try[A], NotUsed], create: A => Try[IdMapping], exists: A => Boolean = (_: A) => true)( - implicit mat: Materializer + def migrate[A](name: String, source: Source[Try[A], NotUsed], create: A => Try[IdMapping], exists: A => Boolean = (_: A) => true)(implicit + mat: Materializer ): Future[Seq[IdMapping]] = source .mapConcat { @@ -149,7 +149,7 @@ trait MigrationOps { name: String, parentIds: Seq[IdMapping], source: Source[Try[(String, A)], NotUsed], - create: (String, A) => Try[IdMapping] + create: (EntityId, A) => Try[IdMapping] )(implicit mat: Materializer): Future[Seq[IdMapping]] = source .mapConcat { @@ -168,8 +168,8 @@ trait MigrationOps { } .runWith(Sink.seq) - def migrateAudit(ids: Seq[IdMapping], source: Source[Try[(String, InputAudit)], NotUsed], create: (String, InputAudit) => Try[Unit])( - implicit ec: ExecutionContext, + def migrateAudit(ids: Seq[IdMapping], source: Source[Try[(String, InputAudit)], NotUsed], create: (EntityId, InputAudit) => Try[Unit])(implicit + ec: ExecutionContext, mat: Materializer ): Future[Unit] = source @@ -195,15 +195,16 @@ trait MigrationOps { inputCaseTemplate: InputCaseTemplate )(implicit ec: ExecutionContext, mat: Materializer): Future[Unit] = migrationStats("CaseTemplate")(output.createCaseTemplate(inputCaseTemplate)).fold( - _ => Future.successful(()), { + _ => Future.successful(()), + { case caseTemplateId @ IdMapping(inputCaseTemplateId, _) => migrateWithParent("CaseTemplate/Task", Seq(caseTemplateId), input.listCaseTemplateTask(inputCaseTemplateId), output.createCaseTemplateTask) .map(_ => ()) } ) - def migrateWholeCaseTemplates(input: Input, output: Output, filter: Filter)( - implicit ec: ExecutionContext, + def migrateWholeCaseTemplates(input: Input, output: Output, filter: Filter)(implicit + ec: ExecutionContext, mat: Materializer ): Future[Unit] = input @@ -225,7 +226,8 @@ trait MigrationOps { inputCase: InputCase )(implicit ec: ExecutionContext, mat: Materializer): Future[Option[IdMapping]] = migrationStats("Case")(output.createCase(inputCase)).fold[Future[Option[IdMapping]]]( - _ => Future.successful(None), { + _ => Future.successful(None), + { case caseId @ IdMapping(inputCaseId, _) => for { caseTaskIds <- migrateWithParent("Case/Task", Seq(caseId), input.listCaseTasks(inputCaseId), output.createCaseTask) @@ -269,7 +271,8 @@ trait MigrationOps { inputAlert: InputAlert )(implicit ec: ExecutionContext, mat: Materializer): Future[Unit] = migrationStats("Alert")(output.createAlert(inputAlert)).fold( - _ => Future.successful(()), { + _ => Future.successful(()), + { case alertId @ IdMapping(inputAlertId, _) => for { alertObservableIds <- migrateWithParent( @@ -296,8 +299,8 @@ trait MigrationOps { // .runWith(Sink.ignore) // .map(_ => ()) - def migrate(input: Input, output: Output, filter: Filter)( - implicit ec: ExecutionContext, + def migrate(input: Input, output: Output, filter: Filter)(implicit + ec: ExecutionContext, mat: Materializer ): Future[Unit] = { val pendingAlertCase: mutable.Map[String, mutable.Buffer[InputAlert]] = mutable.HashMap.empty[String, mutable.Buffer[InputAlert]] @@ -350,7 +353,7 @@ trait MigrationOps { .fold( _ => Future.successful(caseIds), caseId => - migrateAWholeAlert(input, output, filter)(alert.updateCaseId(caseId)) + migrateAWholeAlert(input, output, filter)(alert.updateCaseId(caseId.map(_.toString))) .map(_ => caseIds) ) } @@ -361,7 +364,9 @@ trait MigrationOps { if (caseId.isEmpty) logger.warn(s"Case ID $caseId not found. Link with alert is ignored") - alerts.foldLeft(f1)((f2, alert) => f2.flatMap(_ => migrateAWholeAlert(input, output, filter)(alert.updateCaseId(caseId)))) + alerts.foldLeft(f1)((f2, alert) => + f2.flatMap(_ => migrateAWholeAlert(input, output, filter)(alert.updateCaseId(caseId.map(_.toString)))) + ) } } } diff --git a/migration/src/main/scala/org/thp/thehive/migration/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/Output.scala index ba37f46b85..cd72e8399c 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Output.scala @@ -1,5 +1,6 @@ package org.thp.thehive.migration +import org.thp.scalligraph.EntityId import org.thp.thehive.migration.dto._ import scala.util.Try @@ -23,17 +24,17 @@ trait Output { def createResolutionStatus(inputResolutionStatus: InputResolutionStatus): Try[IdMapping] def caseTemplateExists(inputCaseTemplate: InputCaseTemplate): Boolean def createCaseTemplate(inputCaseTemplate: InputCaseTemplate): Try[IdMapping] - def createCaseTemplateTask(caseTemplateId: String, inputTask: InputTask): Try[IdMapping] + def createCaseTemplateTask(caseTemplateId: EntityId, inputTask: InputTask): Try[IdMapping] def caseExists(inputCase: InputCase): Boolean def createCase(inputCase: InputCase): Try[IdMapping] - def createCaseObservable(caseId: String, inputObservable: InputObservable): Try[IdMapping] - def createJob(observableId: String, inputJob: InputJob): Try[IdMapping] - def createJobObservable(jobId: String, inputObservable: InputObservable): Try[IdMapping] - def createCaseTask(caseId: String, inputTask: InputTask): Try[IdMapping] - def createCaseTaskLog(taskId: String, inputLog: InputLog): Try[IdMapping] + def createCaseObservable(caseId: EntityId, inputObservable: InputObservable): Try[IdMapping] + def createJob(observableId: EntityId, inputJob: InputJob): Try[IdMapping] + def createJobObservable(jobId: EntityId, inputObservable: InputObservable): Try[IdMapping] + def createCaseTask(caseId: EntityId, inputTask: InputTask): Try[IdMapping] + def createCaseTaskLog(taskId: EntityId, inputLog: InputLog): Try[IdMapping] def alertExists(inputAlert: InputAlert): Boolean def createAlert(inputAlert: InputAlert): Try[IdMapping] - def createAlertObservable(alertId: String, inputObservable: InputObservable): Try[IdMapping] - def createAction(objectId: String, inputAction: InputAction): Try[IdMapping] - def createAudit(contextId: String, inputAudit: InputAudit): Try[Unit] + def createAlertObservable(alertId: EntityId, inputObservable: InputObservable): Try[IdMapping] + def createAction(objectId: EntityId, inputAction: InputAction): Try[IdMapping] + def createAudit(contextId: EntityId, inputAudit: InputAudit): Try[Unit] } diff --git a/migration/src/main/scala/org/thp/thehive/migration/dto/InputAudit.scala b/migration/src/main/scala/org/thp/thehive/migration/dto/InputAudit.scala index 993e4be343..c5581282f2 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/dto/InputAudit.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/dto/InputAudit.scala @@ -1,7 +1,8 @@ package org.thp.thehive.migration.dto +import org.thp.scalligraph.EntityId import org.thp.thehive.models.Audit case class InputAudit(metaData: MetaData, audit: Audit) { - def updateObjectId(objectId: Option[String]): InputAudit = copy(audit = audit.copy(objectId = objectId)) + def updateObjectId(objectId: Option[EntityId]): InputAudit = copy(audit = audit.copy(objectId = objectId.map(_.value))) } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index b7a942a6d5..880f31d881 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -96,14 +96,15 @@ trait Conversion { sighted <- (json \ "sighted").validate[Boolean] dataType <- (json \ "dataType").validate[String] tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty) - dataOrAttachment <- (json \ "data") - .validate[String] - .map(Left.apply) - .orElse( - (json \ "attachment") - .validate[Attachment] - .map(a => Right(InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id)))) - ) + dataOrAttachment <- + (json \ "data") + .validate[String] + .map(Left.apply) + .orElse( + (json \ "attachment") + .validate[Attachment] + .map(a => Right(InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id)))) + ) } yield InputObservable( metaData, Observable(message, tlp, ioc, sighted), @@ -151,9 +152,10 @@ trait Conversion { message <- (json \ "message").validate[String] date <- (json \ "startDate").validate[Date] deleted = (json \ "status").asOpt[String].contains("Deleted") - attachment = (json \ "attachment") - .asOpt[Attachment] - .map(a => InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id))) + attachment = + (json \ "attachment") + .asOpt[Attachment] + .map(a => InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id))) } yield InputLog(metaData, Log(message, date, deleted), attachment.toSeq) } @@ -207,31 +209,33 @@ trait Conversion { ) } - def alertObservableReads(metaData: MetaData): Reads[InputObservable] = Reads[InputObservable] { json => - for { - dataType <- (json \ "dataType").validate[String] - message <- (json \ "message").validateOpt[String] - tlp <- (json \ "tlp").validateOpt[Int] - tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty) - ioc <- (json \ "ioc").validateOpt[Boolean] - dataOrAttachment <- (json \ "data") - .validate[String] - .map(Left.apply) - .orElse( - (json \ "attachment") - .validate[Attachment] - .map(a => Right(InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id)))) - ) - } yield InputObservable( - metaData, - Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false), - Nil, - dataType, - tags, - dataOrAttachment - ) + def alertObservableReads(metaData: MetaData): Reads[InputObservable] = + Reads[InputObservable] { json => + for { + dataType <- (json \ "dataType").validate[String] + message <- (json \ "message").validateOpt[String] + tlp <- (json \ "tlp").validateOpt[Int] + tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty) + ioc <- (json \ "ioc").validateOpt[Boolean] + dataOrAttachment <- + (json \ "data") + .validate[String] + .map(Left.apply) + .orElse( + (json \ "attachment") + .validate[Attachment] + .map(a => Right(InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id)))) + ) + } yield InputObservable( + metaData, + Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false), + Nil, + dataType, + tags, + dataOrAttachment + ) - } + } def normaliseLogin(login: String): String = { def validSegment(value: String) = { @@ -257,17 +261,19 @@ trait Conversion { locked = status == "Locked" password <- (json \ "password").validateOpt[String] role <- (json \ "roles").validateOpt[Seq[String]].map(_.getOrElse(Nil)) - organisationProfiles = if (role.contains("admin")) - Map(Organisation.administration.name -> Profile.admin.name, mainOrganisation -> Profile.orgAdmin.name) - else if (role.contains("write")) Map(mainOrganisation -> Profile.analyst.name) - else if (role.contains("read")) Map(mainOrganisation -> Profile.readonly.name) - else Map(mainOrganisation -> Profile.readonly.name) - avatar = (json \ "avatar") - .asOpt[String] - .map { base64 => - val data = Base64.getDecoder.decode(base64) - InputAttachment(s"$login.avatar", data.size.toLong, "image/png", Nil, Source.single(ByteString(data))) - } + organisationProfiles = + if (role.contains("admin")) + Map(Organisation.administration.name -> Profile.admin.name, mainOrganisation -> Profile.orgAdmin.name) + else if (role.contains("write")) Map(mainOrganisation -> Profile.analyst.name) + else if (role.contains("read")) Map(mainOrganisation -> Profile.readonly.name) + else Map(mainOrganisation -> Profile.readonly.name) + avatar = + (json \ "avatar") + .asOpt[String] + .map { base64 => + val data = Base64.getDecoder.decode(base64) + InputAttachment(s"$login.avatar", data.size.toLong, "image/png", Nil, Source.single(ByteString(data))) + } } yield InputUser(metaData, User(normaliseLogin(login), name, apikey, locked, password, None), organisationProfiles, avatar) } @@ -362,35 +368,36 @@ trait Conversion { ) } - def caseTemplateTaskReads(metaData: MetaData): Reads[InputTask] = Reads[InputTask] { json => - for { - title <- (json \ "title").validate[String] - group <- (json \ "group").validateOpt[String] - description <- (json \ "description").validateOpt[String] - status <- (json \ "status").validateOpt[TaskStatus.Value] - flag <- (json \ "flag").validateOpt[Boolean] - startDate <- (json \ "startDate").validateOpt[Date] - endDate <- (json \ "endDate").validateOpt[Date] - order <- (json \ "order").validateOpt[Int] - dueDate <- (json \ "dueDate").validateOpt[Date] - owner <- (json \ "owner").validateOpt[String] - } yield InputTask( - metaData, - Task( - title, - group.getOrElse("default"), - description, - status.getOrElse(TaskStatus.Waiting), - flag.getOrElse(false), - startDate, - endDate, - order.getOrElse(1), - dueDate - ), - owner.map(normaliseLogin), - Seq(mainOrganisation) - ) - } + def caseTemplateTaskReads(metaData: MetaData): Reads[InputTask] = + Reads[InputTask] { json => + for { + title <- (json \ "title").validate[String] + group <- (json \ "group").validateOpt[String] + description <- (json \ "description").validateOpt[String] + status <- (json \ "status").validateOpt[TaskStatus.Value] + flag <- (json \ "flag").validateOpt[Boolean] + startDate <- (json \ "startDate").validateOpt[Date] + endDate <- (json \ "endDate").validateOpt[Date] + order <- (json \ "order").validateOpt[Int] + dueDate <- (json \ "dueDate").validateOpt[Date] + owner <- (json \ "owner").validateOpt[String] + } yield InputTask( + metaData, + Task( + title, + group.getOrElse("default"), + description, + status.getOrElse(TaskStatus.Waiting), + flag.getOrElse(false), + startDate, + endDate, + order.getOrElse(1), + dueDate + ), + owner.map(normaliseLogin), + Seq(mainOrganisation) + ) + } lazy val jobReads: Reads[InputJob] = Reads[InputJob] { json => for { @@ -423,30 +430,31 @@ trait Conversion { ) } - def jobObservableReads(metaData: MetaData): Reads[InputObservable] = Reads[InputObservable] { json => - for { - message <- (json \ "message").validateOpt[String] orElse (json \ "attributes" \ "message").validateOpt[String] - tlp <- (json \ "tlp").validate[Int] orElse (json \ "attributes" \ "tlp").validate[Int] orElse JsSuccess(2) - ioc <- (json \ "ioc").validate[Boolean] orElse (json \ "attributes" \ "ioc").validate[Boolean] orElse JsSuccess(false) - sighted <- (json \ "sighted").validate[Boolean] orElse (json \ "attributes" \ "sighted").validate[Boolean] orElse JsSuccess(false) - dataType <- (json \ "dataType").validate[String] orElse (json \ "type").validate[String] orElse (json \ "attributes").validate[String] - tags <- (json \ "tags").validate[Set[String]] orElse (json \ "attributes" \ "tags").validate[Set[String]] orElse JsSuccess(Set.empty[String]) - dataOrAttachment <- ((json \ "data").validate[String] orElse (json \ "value").validate[String]) - .map(Left.apply) - .orElse( - (json \ "attachment") - .validate[Attachment] - .map(a => Right(InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id)))) - ) - } yield InputObservable( - metaData, - Observable(message, tlp, ioc, sighted), - Seq(mainOrganisation), - dataType, - tags, - dataOrAttachment - ) - } + def jobObservableReads(metaData: MetaData): Reads[InputObservable] = + Reads[InputObservable] { json => + for { + message <- (json \ "message").validateOpt[String] orElse (json \ "attributes" \ "message").validateOpt[String] + tlp <- (json \ "tlp").validate[Int] orElse (json \ "attributes" \ "tlp").validate[Int] orElse JsSuccess(2) + ioc <- (json \ "ioc").validate[Boolean] orElse (json \ "attributes" \ "ioc").validate[Boolean] orElse JsSuccess(false) + sighted <- (json \ "sighted").validate[Boolean] orElse (json \ "attributes" \ "sighted").validate[Boolean] orElse JsSuccess(false) + dataType <- (json \ "dataType").validate[String] orElse (json \ "type").validate[String] orElse (json \ "attributes").validate[String] + tags <- (json \ "tags").validate[Set[String]] orElse (json \ "attributes" \ "tags").validate[Set[String]] orElse JsSuccess(Set.empty[String]) + dataOrAttachment <- ((json \ "data").validate[String] orElse (json \ "value").validate[String]) + .map(Left.apply) + .orElse( + (json \ "attachment") + .validate[Attachment] + .map(a => Right(InputAttachment(a.name, a.size, a.contentType, a.hashes.map(_.toString), readAttachment(a.id)))) + ) + } yield InputObservable( + metaData, + Observable(message, tlp, ioc, sighted), + Seq(mainOrganisation), + dataType, + tags, + dataOrAttachment + ) + } implicit val actionReads: Reads[(String, InputAction)] = Reads[(String, InputAction)] { json => for { diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala index 336fa585b6..d3de9d4ee2 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala @@ -92,11 +92,11 @@ class DBFind(pageSize: Int, keepAlive: FiniteDuration, db: DBConfiguration, impl logger.debug( s"search in ${searchRequest.indexesTypes.indexes.mkString(",")} / ${searchRequest.indexesTypes.types.mkString(",")} ${db.client.show(searchRequest)}" ) - val (src, total) = if (limit > 2 * pageSize) { - searchWithScroll(searchRequest, offset, limit) - } else { - searchWithoutScroll(searchRequest, offset, limit) - } + val (src, total) = + if (limit > 2 * pageSize) + searchWithScroll(searchRequest, offset, limit) + else + searchWithoutScroll(searchRequest, offset, limit) (src.map(DBUtils.hit2json), total) } @@ -114,13 +114,12 @@ class DBFind(pageSize: Int, keepAlive: FiniteDuration, db: DBConfiguration, impl db.execute(searchRequest) .recoverWith { case t: InternalError => Future.failed(t) - case t => Future.failed(SearchError("Invalid search query")) + case _ => Future.failed(SearchError("Invalid search query")) } } } -class SearchWithScroll(db: DBConfiguration, SearchRequest: SearchRequest, keepAliveStr: String, offset: Int, max: Int)( - implicit +class SearchWithScroll(db: DBConfiguration, SearchRequest: SearchRequest, keepAliveStr: String, offset: Int, max: Int)(implicit ec: ExecutionContext ) extends GraphStage[SourceShape[SearchHit]] { @@ -130,83 +129,82 @@ class SearchWithScroll(db: DBConfiguration, SearchRequest: SearchRequest, keepAl val firstResults: Future[SearchResponse] = db.execute(SearchRequest.scroll(keepAliveStr)) val totalHits: Future[Long] = firstResults.map(_.totalHits) - override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { - var processed: Long = 0 - var skip: Long = offset.toLong - val queue: mutable.Queue[SearchHit] = mutable.Queue.empty - var scrollId: Future[String] = firstResults.map(_.scrollId.get) - var firstResultProcessed = false - - setHandler( - out, - new OutHandler { - - def pushNextHit(): Unit = { - push(out, queue.dequeue()) - processed += 1 - if (processed >= max) { - completeStage() + override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = + new GraphStageLogic(shape) { + var processed: Long = 0 + var skip: Long = offset.toLong + val queue: mutable.Queue[SearchHit] = mutable.Queue.empty + var scrollId: Future[String] = firstResults.map(_.scrollId.get) + var firstResultProcessed = false + + setHandler( + out, + new OutHandler { + + def pushNextHit(): Unit = { + push(out, queue.dequeue()) + processed += 1 + if (processed >= max) + completeStage() } - } - val firstCallback: AsyncCallback[Try[SearchResponse]] = getAsyncCallback[Try[SearchResponse]] { - case Success(searchResponse) if skip > 0 => - if (searchResponse.hits.size <= skip) - skip -= searchResponse.hits.size - else { - queue ++= searchResponse.hits.hits.drop(skip.toInt) - skip = 0 - } - firstResultProcessed = true - onPull() - case Success(searchResponse) => - queue ++= searchResponse.hits.hits - firstResultProcessed = true - onPull() - case Failure(error) => - logger.warn("Search error", error) - failStage(error) - } + val firstCallback: AsyncCallback[Try[SearchResponse]] = getAsyncCallback[Try[SearchResponse]] { + case Success(searchResponse) if skip > 0 => + if (searchResponse.hits.size <= skip) + skip -= searchResponse.hits.size + else { + queue ++= searchResponse.hits.hits.drop(skip.toInt) + skip = 0 + } + firstResultProcessed = true + onPull() + case Success(searchResponse) => + queue ++= searchResponse.hits.hits + firstResultProcessed = true + onPull() + case Failure(error) => + logger.warn("Search error", error) + failStage(error) + } - override def onPull(): Unit = - if (firstResultProcessed) { - if (processed >= max) completeStage() - - if (queue.isEmpty) { - val callback = getAsyncCallback[Try[SearchResponse]] { - case Success(searchResponse) if searchResponse.isTimedOut => - logger.warn("Search timeout") - failStage(SearchError("Request terminated early or timed out")) - case Success(searchResponse) if searchResponse.isEmpty => - completeStage() - case Success(searchResponse) if skip > 0 => - if (searchResponse.hits.size <= skip) { - skip -= searchResponse.hits.size - onPull() - } else { - queue ++= searchResponse.hits.hits.drop(skip.toInt) - skip = 0 + override def onPull(): Unit = + if (firstResultProcessed) { + if (processed >= max) completeStage() + + if (queue.isEmpty) { + val callback = getAsyncCallback[Try[SearchResponse]] { + case Success(searchResponse) if searchResponse.isTimedOut => + logger.warn("Search timeout") + failStage(SearchError("Request terminated early or timed out")) + case Success(searchResponse) if searchResponse.isEmpty => + completeStage() + case Success(searchResponse) if skip > 0 => + if (searchResponse.hits.size <= skip) { + skip -= searchResponse.hits.size + onPull() + } else { + queue ++= searchResponse.hits.hits.drop(skip.toInt) + skip = 0 + pushNextHit() + } + case Success(searchResponse) => + queue ++= searchResponse.hits.hits pushNextHit() - } - case Success(searchResponse) => - queue ++= searchResponse.hits.hits - pushNextHit() - case Failure(error) => - logger.warn("Search error", error) - failStage(SearchError("Request terminated early or timed out")) - } - val futureSearchResponse = scrollId.flatMap(s => db.execute(searchScroll(s).keepAlive(keepAliveStr))) - scrollId = futureSearchResponse.map(_.scrollId.get) - futureSearchResponse.onComplete(callback.invoke) - } else { - pushNextHit() - } - } else firstResults.onComplete(firstCallback.invoke) - } - ) - override def postStop(): Unit = - scrollId.foreach { s => - db.execute(clearScroll(s)) - } - } + case Failure(error) => + logger.warn("Search error", error) + failStage(SearchError("Request terminated early or timed out")) + } + val futureSearchResponse = scrollId.flatMap(s => db.execute(searchScroll(s).keepAlive(keepAliveStr))) + scrollId = futureSearchResponse.map(_.scrollId.get) + futureSearchResponse.onComplete(callback.invoke) + } else + pushNextHit() + } else firstResults.onComplete(firstCallback.invoke) + } + ) + override def postStop(): Unit = + scrollId.foreach { s => + db.execute(clearScroll(s)) + } + } } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index bb203d570b..d320cbc0d8 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -144,7 +144,7 @@ class Output @Inject() ( db.roTransaction { implicit graph => Traversal .V() - .has( + .unsafeHas( "_label", P.within( Seq( @@ -208,17 +208,19 @@ class Output @Inject() ( caseTemplates = caseTemplatesBuilder.result() caseNumbers = caseNumbersBuilder.result() alerts = alertsBuilder.result() - if (profiles.nonEmpty || - organisations.nonEmpty || - users.nonEmpty || - impactStatuses.nonEmpty || - resolutionStatuses.nonEmpty || - observableTypes.nonEmpty || - customFields.nonEmpty || - caseTemplates.nonEmpty || - caseNumbers.nonEmpty || - alerts.nonEmpty) - logger.info(s"""Already migrated: + if ( + profiles.nonEmpty || + organisations.nonEmpty || + users.nonEmpty || + impactStatuses.nonEmpty || + resolutionStatuses.nonEmpty || + observableTypes.nonEmpty || + customFields.nonEmpty || + caseTemplates.nonEmpty || + caseNumbers.nonEmpty || + alerts.nonEmpty + ) + logger.info(s"""Already migrated: | ${profiles.size} profiles\n | ${organisations.size} organisations\n | ${users.size} users\n @@ -273,8 +275,8 @@ class Output @Inject() ( def getAuthContext(userId: String): AuthContext = if (userId.startsWith("init@")) LocalUserSrv.getSystemAuthContext - else if (userId.contains('@')) AuthContextImpl(userId, userId, "admin", "mig-request", Permissions.all) - else AuthContextImpl(s"$userId@$defaultUserDomain", s"$userId@$defaultUserDomain", "admin", "mig-request", Permissions.all) + else if (userId.contains('@')) AuthContextImpl(userId, userId, EntityName("admin"), "mig-request", Permissions.all) + else AuthContextImpl(s"$userId@$defaultUserDomain", s"$userId@$defaultUserDomain", EntityName("admin"), "mig-request", Permissions.all) def authTransaction[A](userId: String)(body: Graph => AuthContext => Try[A]): Try[A] = db.tryTransaction { implicit graph => @@ -476,7 +478,7 @@ class Output @Inject() ( } yield IdMapping(inputCaseTemplate.metaData.id, richCaseTemplate._id) } - override def createCaseTemplateTask(caseTemplateId: String, inputTask: InputTask): Try[IdMapping] = + override def createCaseTemplateTask(caseTemplateId: EntityId, inputTask: InputTask): Try[IdMapping] = authTransaction(inputTask.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create task ${inputTask.task.title} in case template $caseTemplateId") for { @@ -490,7 +492,7 @@ class Output @Inject() ( override def caseExists(inputCase: InputCase): Boolean = caseNumbers.contains(inputCase.`case`.number) - private def getCase(caseId: String)(implicit graph: Graph): Try[Case with Entity] = caseSrv.getByIds(caseId).getOrFail("Case") + private def getCase(caseId: EntityId)(implicit graph: Graph): Try[Case with Entity] = caseSrv.getByIds(caseId).getOrFail("Case") override def createCase(inputCase: InputCase): Try[IdMapping] = authTransaction(inputCase.metaData.createdBy) { implicit graph => implicit authContext => @@ -559,7 +561,7 @@ class Output @Inject() ( } } - override def createCaseTask(caseId: String, inputTask: InputTask): Try[IdMapping] = + override def createCaseTask(caseId: EntityId, inputTask: InputTask): Try[IdMapping] = authTransaction(inputTask.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create task ${inputTask.task.title} in case $caseId") val owner = inputTask.owner.flatMap(getUser(_).toOption) @@ -573,7 +575,7 @@ class Output @Inject() ( } yield IdMapping(inputTask.metaData.id, richTask._id) } - def createCaseTaskLog(taskId: String, inputLog: InputLog): Try[IdMapping] = + def createCaseTaskLog(taskId: EntityId, inputLog: InputLog): Try[IdMapping] = authTransaction(inputLog.metaData.createdBy) { implicit graph => implicit authContext => for { task <- taskSrv.getOrFail(taskId) @@ -588,7 +590,7 @@ class Output @Inject() ( } yield IdMapping(inputLog.metaData.id, log._id) } - override def createCaseObservable(caseId: String, inputObservable: InputObservable): Try[IdMapping] = + override def createCaseObservable(caseId: EntityId, inputObservable: InputObservable): Try[IdMapping] = authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in case $caseId") for { @@ -615,7 +617,7 @@ class Output @Inject() ( } yield IdMapping(inputObservable.metaData.id, richObservable._id) } - override def createJob(observableId: String, inputJob: InputJob): Try[IdMapping] = + override def createJob(observableId: EntityId, inputJob: InputJob): Try[IdMapping] = authTransaction(inputJob.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create job ${inputJob.job.cortexId}:${inputJob.job.workerName}:${inputJob.job.cortexJobId}") for { @@ -625,7 +627,7 @@ class Output @Inject() ( } yield IdMapping(inputJob.metaData.id, job._id) } - override def createJobObservable(jobId: String, inputObservable: InputObservable): Try[IdMapping] = + override def createJobObservable(jobId: EntityId, inputObservable: InputObservable): Try[IdMapping] = authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in job $jobId") for { @@ -678,11 +680,11 @@ class Output @Inject() ( _ <- tags.toTry(t => alertSrv.alertTagSrv.create(AlertTag(), alert, t)) _ <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, name, value) } _ = updateMetaData(alert, inputAlert.metaData) - _ = inputAlert.caseId.flatMap(getCase(_).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert, _)) + _ = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert, _)) } yield IdMapping(inputAlert.metaData.id, alert._id) } - override def createAlertObservable(alertId: String, inputObservable: InputObservable): Try[IdMapping] = + override def createAlertObservable(alertId: EntityId, inputObservable: InputObservable): Try[IdMapping] = authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in alert $alertId") for { @@ -708,7 +710,7 @@ class Output @Inject() ( } yield IdMapping(inputObservable.metaData.id, richObservable._id) } - private def getEntity(entityType: String, entityId: String)(implicit graph: Graph): Try[Product with Entity] = + private def getEntity(entityType: String, entityId: EntityId)(implicit graph: Graph): Try[Product with Entity] = entityType match { case "Task" => taskSrv.getOrFail(entityId) case "Case" => getCase(entityId) @@ -719,7 +721,7 @@ class Output @Inject() ( case _ => Failure(BadRequestError(s"objectType $entityType is not recognised")) } - override def createAction(objectId: String, inputAction: InputAction): Try[IdMapping] = + override def createAction(objectId: EntityId, inputAction: InputAction): Try[IdMapping] = authTransaction(inputAction.metaData.createdBy) { implicit graph => implicit authContext => logger.debug( s"Create action ${inputAction.action.cortexId}:${inputAction.action.workerName}:${inputAction.action.cortexJobId} for ${inputAction.objectType} $objectId" @@ -731,14 +733,14 @@ class Output @Inject() ( } yield IdMapping(inputAction.metaData.id, action._id) } - override def createAudit(contextId: String, inputAudit: InputAudit): Try[Unit] = + override def createAudit(contextId: EntityId, inputAudit: InputAudit): Try[Unit] = authTransaction(inputAudit.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create audit ${inputAudit.audit.action} on ${inputAudit.audit.objectType} ${inputAudit.audit.objectId}") for { obj <- (for { t <- inputAudit.audit.objectType i <- inputAudit.audit.objectId - } yield getEntity(t, i)).flip + } yield getEntity(t, new EntityId(i))).flip ctxType = obj.map(_._label).map { case "Alert" => "Alert" case "Log" | "Task" | "Observable" | "Case" | "Job" => "Case" diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/controllers/v0/MispCtrl.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/controllers/v0/MispCtrl.scala index 125cd68ef3..a9b06ff214 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/controllers/v0/MispCtrl.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/controllers/v0/MispCtrl.scala @@ -3,6 +3,7 @@ package org.thp.thehive.connector.misp.controllers.v0 import akka.actor.ActorRef import com.google.inject.name.Named import javax.inject.{Inject, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.Entrypoint import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ @@ -40,7 +41,7 @@ class MispCtrl @Inject() ( for { c <- Future.fromTry(db.roTransaction { implicit graph => caseSrv - .get(caseIdOrNumber) + .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageShare) .getOrFail("Case") }) @@ -53,7 +54,7 @@ class MispCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => alertSrv .startTraversal - .has("type", "misp") + .filterByType("misp") .visible .toIterator .toTry(alertSrv.remove(_)) diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/AttributeConverter.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/AttributeConverter.scala index d887d8521b..e0363f9ff5 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/AttributeConverter.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/AttributeConverter.scala @@ -1,8 +1,9 @@ package org.thp.thehive.connector.misp.services +import org.thp.scalligraph.EntityName import play.api.libs.json.{Format, Json} -case class AttributeConverter(mispCategory: String, mispType: String, `type`: String, tags: Seq[String]) +case class AttributeConverter(mispCategory: String, mispType: String, `type`: EntityName, tags: Seq[String]) object AttributeConverter { implicit val format: Format[AttributeConverter] = Json.format[AttributeConverter] diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/Connector.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/Connector.scala index c94e84b16a..db70d09654 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/Connector.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/Connector.scala @@ -28,7 +28,7 @@ class Connector @Inject() (appConfig: ApplicationConfig, system: ActorSystem, ma attributeConvertersConfig.get.reverseIterator.find(a => a.mispCategory == attributeCategory && a.mispType == attributeType) def attributeConverter(`type`: ObservableType): Option[(String, String)] = - attributeConvertersConfig.get.reverseIterator.find(_.`type` == `type`.name).map(a => a.mispCategory -> a.mispType) + attributeConvertersConfig.get.reverseIterator.find(_.`type`.value == `type`.name).map(a => a.mispCategory -> a.mispType) val syncIntervalConfig: ConfigItem[FiniteDuration, FiniteDuration] = appConfig.item[FiniteDuration]("misp.syncInterval", "") def syncInterval: FiniteDuration = syncIntervalConfig.get @@ -47,9 +47,10 @@ class Connector @Inject() (appConfig: ApplicationConfig, system: ActorSystem, ma .traverse(clients)(client => client.getStatus) .foreach { statusDetails => val distinctStatus = statusDetails.map(s => (s \ "status").as[String]).toSet - val healthStatus = if (distinctStatus.contains("OK")) { - if (distinctStatus.size > 1) "WARNING" else "OK" - } else "ERROR" + val healthStatus = + if (distinctStatus.contains("OK")) + if (distinctStatus.size > 1) "WARNING" else "OK" + else "ERROR" cachedStatus = Json.obj("enabled" -> true, "servers" -> statusDetails, "status" -> healthStatus) system.scheduler.scheduleOnce(statusCheckInterval)(updateStatus()) } @@ -62,10 +63,11 @@ class Connector @Inject() (appConfig: ApplicationConfig, system: ActorSystem, ma .traverse(clients)(_.getHealth) .foreach { healthStatus => val distinctStatus = healthStatus.toSet - cachedHealth = if (distinctStatus.contains(HealthStatus.Ok)) { - if (distinctStatus.size > 1) HealthStatus.Warning else HealthStatus.Ok - } else if (distinctStatus.contains(HealthStatus.Error)) HealthStatus.Error - else HealthStatus.Warning + cachedHealth = + if (distinctStatus.contains(HealthStatus.Ok)) + if (distinctStatus.size > 1) HealthStatus.Warning else HealthStatus.Ok + else if (distinctStatus.contains(HealthStatus.Error)) HealthStatus.Error + else HealthStatus.Warning system.scheduler.scheduleOnce(statusCheckInterval)(updateHealth()) } diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala index e91746d231..fdcfabbb56 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala @@ -10,6 +10,7 @@ import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{AuthorizationError, BadRequestError, NotFoundError} import org.thp.thehive.models._ +import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.{AlertSrv, AttachmentSrv, CaseSrv, OrganisationSrv} @@ -72,12 +73,12 @@ class MispExportSrv @Inject() ( caseSrv .get(`case`) .alert - .has("type", "misp") - .has("source", orgName) + .filterBySource(orgName) + .filterByType("misp") .headOption def getAttributes(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Iterator[Attribute] = - caseSrv.get(`case`).observables.has("ioc", true).richObservable.toIterator.flatMap(observableToAttribute) + caseSrv.get(`case`).observables.isIoc.richObservable.toIterator.flatMap(observableToAttribute) def removeDuplicateAttributes(attributes: Iterator[Attribute]): Seq[Attribute] = { var attrSet = Set.empty[(String, String, String)] @@ -92,8 +93,8 @@ class MispExportSrv @Inject() ( builder.result() } - def createEvent(client: TheHiveMispClient, `case`: Case, attributes: Seq[Attribute], extendsEvent: Option[String])( - implicit ec: ExecutionContext + def createEvent(client: TheHiveMispClient, `case`: Case, attributes: Seq[Attribute], extendsEvent: Option[String])(implicit + ec: ExecutionContext ): Future[String] = client.createEvent( info = `case`.title, @@ -106,8 +107,8 @@ class MispExportSrv @Inject() ( extendsEvent = extendsEvent ) - def createAlert(client: TheHiveMispClient, `case`: Case with Entity, eventId: String)( - implicit graph: Graph, + def createAlert(client: TheHiveMispClient, `case`: Case with Entity, eventId: String)(implicit + graph: Graph, authContext: AuthContext ): Try[RichAlert] = for { diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala index 216272bfb9..1f57a57a3d 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala @@ -9,11 +9,11 @@ import akka.util.ByteString import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.misp.dto.{Attribute, Event, Tag => MispTag} -import org.thp.scalligraph.RichSeq import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.FFile -import org.thp.scalligraph.models.{Database, Entity} +import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{EntityName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.ObservableOps._ @@ -82,8 +82,8 @@ class MispImportSrv @Inject() ( } db.roTransaction { implicit graph => obsTypeFromConfig - .orElse(observableTypeSrv.get(attributeType).headOption.map(_ -> Nil)) - .fold(observableTypeSrv.getOrFail("other").map(_ -> Seq.empty[String]))(Success(_)) + .orElse(observableTypeSrv.get(EntityName(attributeType)).headOption.map(_ -> Nil)) + .fold(observableTypeSrv.getOrFail(EntityName("other")).map(_ -> Seq.empty[String]))(Success(_)) } } @@ -159,11 +159,10 @@ class MispImportSrv @Inject() ( .group( _.by, _.by( - _.in[AlertOrganisation] - .v[Alert] - .has("source", mispOrganisation) - .has("type", "misp") - .value(_.lastSyncDate) + _.alerts + .filterBySource(mispOrganisation) + .filterByType("misp") + .value(a => a.lastSyncDate) .max ) ) @@ -199,13 +198,14 @@ class MispImportSrv @Inject() ( case Some(richObservable) => logger.debug(s"Observable ${observableType.name}:$data exists, update it") for { - updatedObservable <- Some(observableSrv.get(richObservable.observable)) - .map(t => if (richObservable.message != observable.message) t.update(_.message, observable.message) else t) - .map(t => if (richObservable.tlp != observable.tlp) t.update(_.tlp, observable.tlp) else t) - .map(t => if (richObservable.ioc != observable.ioc) t.update(_.ioc, observable.ioc) else t) - .map(t => if (richObservable.sighted != observable.sighted) t.update(_.sighted, observable.sighted) else t) - .get - .getOrFail("Observable") + updatedObservable <- + Some(observableSrv.get(richObservable.observable)) + .map(t => if (richObservable.message != observable.message) t.update(_.message, observable.message) else t) + .map(t => if (richObservable.tlp != observable.tlp) t.update(_.tlp, observable.tlp) else t) + .map(t => if (richObservable.ioc != observable.ioc) t.update(_.ioc, observable.ioc) else t) + .map(t => if (richObservable.sighted != observable.sighted) t.update(_.sighted, observable.sighted) else t) + .get + .getOrFail("Observable") _ <- observableSrv.updateTagNames(updatedObservable, tags) } yield updatedObservable } @@ -252,21 +252,22 @@ class MispImportSrv @Inject() ( Future.fromTry { db.tryTransaction { implicit graph => for { - updatedObservable <- Some(observableSrv.get(richObservable.observable)) - .map(t => if (richObservable.message != observable.message) t.update(_.message, observable.message) else t) - .map(t => if (richObservable.tlp != observable.tlp) t.update(_.tlp, observable.tlp) else t) - .map(t => if (richObservable.ioc != observable.ioc) t.update(_.ioc, observable.ioc) else t) - .map(t => if (richObservable.sighted != observable.sighted) t.update(_.sighted, observable.sighted) else t) - .get - .getOrFail("Observable") + updatedObservable <- + Some(observableSrv.get(richObservable.observable)) + .map(t => if (richObservable.message != observable.message) t.update(_.message, observable.message) else t) + .map(t => if (richObservable.tlp != observable.tlp) t.update(_.tlp, observable.tlp) else t) + .map(t => if (richObservable.ioc != observable.ioc) t.update(_.ioc, observable.ioc) else t) + .map(t => if (richObservable.sighted != observable.sighted) t.update(_.sighted, observable.sighted) else t) + .get + .getOrFail("Observable") _ <- observableSrv.updateTagNames(updatedObservable, tags) } yield updatedObservable } } } - def importAttibutes(client: TheHiveMispClient, event: Event, alert: Alert with Entity, lastSynchro: Option[Date])( - implicit authContext: AuthContext + def importAttibutes(client: TheHiveMispClient, event: Event, alert: Alert with Entity, lastSynchro: Option[Date])(implicit + authContext: AuthContext ): Future[Unit] = { logger.debug(s"importAttibutes ${client.name}#${event.id}") val startSyncDate = new Date @@ -288,10 +289,13 @@ class MispImportSrv @Inject() ( .runWith(Sink.foreachAsync(1) { case (observable, observableType, tags, Left(data)) => updateOrCreateObservable(alert, observable, observableType, data, tags) - .fold(error => { - logger.error(s"Unable to create observable $observable ${observableType.name}:$data", error) - Future.failed(error) - }, _ => Future.successful(())) + .fold( + error => { + logger.error(s"Unable to create observable $observable ${observableType.name}:$data", error) + Future.failed(error) + }, + _ => Future.successful(()) + ) case (observable, observableType, tags, Right((filename, contentType, src))) => updateOrCreateObservable(alert, observable, observableType, filename, contentType, src, tags) .transform { @@ -335,8 +339,8 @@ class MispImportSrv @Inject() ( mispOrganisation: String, event: Event, caseTemplate: Option[CaseTemplate with Entity] - )( - implicit authContext: AuthContext + )(implicit + authContext: AuthContext ): Try[Alert with Entity] = { logger.debug(s"updateOrCreateAlert ${client.name}#${event.id} for organisation ${organisation.name}") eventToAlert(client, event).flatMap { alert => @@ -352,20 +356,21 @@ class MispImportSrv @Inject() ( alertSrv .create(alert, organisation, event.tags.map(_.name).toSet, Map.empty[String, Option[Any]], caseTemplate) .map(_.alert) - case someAlert @ Some(richAlert) => + case Some(richAlert) => logger.debug(s"Event ${client.name}#${event.id} have already been imported for organisation ${organisation.name}, updating the alert") for { - updatedAlert <- Some(alertSrv.get(richAlert.alert)) - .map(t => if (richAlert.title != alert.title) t.update(_.title, alert.title) else t) - .map(t => if (richAlert.lastSyncDate != alert.lastSyncDate) t.update(_.lastSyncDate, alert.lastSyncDate) else t) - .map(t => if (richAlert.description != alert.description) t.update(_.description, alert.description) else t) - .map(t => if (richAlert.severity != alert.severity) t.update(_.severity, alert.severity) else t) - .map(t => if (richAlert.date != alert.date) t.update(_.date, alert.date) else t) - .map(t => if (richAlert.tlp != alert.tlp) t.update(_.tlp, alert.tlp) else t) - .map(t => if (richAlert.pap != alert.pap) t.update(_.pap, alert.pap) else t) - .map(t => if (richAlert.externalLink != alert.externalLink) t.update(_.externalLink, alert.externalLink) else t) - .get - .getOrFail("Alert") + updatedAlert <- + Some(alertSrv.get(richAlert.alert)) + .map(t => if (richAlert.title != alert.title) t.update(_.title, alert.title) else t) + .map(t => if (richAlert.lastSyncDate != alert.lastSyncDate) t.update(_.lastSyncDate, alert.lastSyncDate) else t) + .map(t => if (richAlert.description != alert.description) t.update(_.description, alert.description) else t) + .map(t => if (richAlert.severity != alert.severity) t.update(_.severity, alert.severity) else t) + .map(t => if (richAlert.date != alert.date) t.update(_.date, alert.date) else t) + .map(t => if (richAlert.tlp != alert.tlp) t.update(_.tlp, alert.tlp) else t) + .map(t => if (richAlert.pap != alert.pap) t.update(_.pap, alert.pap) else t) + .map(t => if (richAlert.externalLink != alert.externalLink) t.update(_.externalLink, alert.externalLink) else t) + .get + .getOrFail("Alert") _ <- alertSrv.updateTagNames(updatedAlert, event.tags.map(_.name).toSet) } yield updatedAlert } @@ -377,7 +382,7 @@ class MispImportSrv @Inject() ( Future.fromTry(client.currentOrganisationName).flatMap { mispOrganisation => lazy val caseTemplate = client.caseTemplate.flatMap { caseTemplateName => db.roTransaction { implicit graph => - caseTemplateSrv.get(caseTemplateName).headOption + caseTemplateSrv.get(EntityName(caseTemplateName)).headOption } } logger.debug(s"Get eligible organisations") diff --git a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala index d97fb7ac64..915ef429c0 100644 --- a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala +++ b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala @@ -5,10 +5,10 @@ import java.util.{Date, UUID} import akka.stream.Materializer import akka.stream.scaladsl.Sink import org.thp.misp.dto.{Event, Organisation, Tag, User} -import org.thp.scalligraph.AppBuilder import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{AppBuilder, EntityName} import org.thp.thehive.TestAppBuilder import org.thp.thehive.models.{Alert, Permissions} import org.thp.thehive.services.AlertOps._ @@ -98,7 +98,7 @@ class MispImportSrvTest(implicit ec: ExecutionContext) extends PlaySpecification val observables = app[Database] .roTransaction { implicit graph => app[OrganisationSrv] - .get("admin") + .get(EntityName("admin")) .alerts .getBySourceId("misp", "ORGNAME", "1") .observables diff --git a/thehive/app/org/thp/thehive/controllers/dav/Router.scala b/thehive/app/org/thp/thehive/controllers/dav/Router.scala index 56427ce74f..dbe2c6930b 100644 --- a/thehive/app/org/thp/thehive/controllers/dav/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/dav/Router.scala @@ -3,6 +3,7 @@ package org.thp.thehive.controllers.dav import akka.stream.scaladsl.StreamConverters import akka.util.ByteString import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.thehive.services.AttachmentSrv @@ -37,14 +38,15 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s case _ => debug() } - def debug(): Action[AnyContent] = entrypoint("DAV options") { request => - logger.debug(s"request ${request.method} ${request.path}") - request.headers.headers.foreach { - case (k, v) => logger.debug(s"$k: $v") + def debug(): Action[AnyContent] = + entrypoint("DAV options") { request => + logger.debug(s"request ${request.method} ${request.path}") + request.headers.headers.foreach { + case (k, v) => logger.debug(s"$k: $v") + } + logger.debug(request.body.toString) + Success(Results.Ok("")) } - logger.debug(request.body.toString) - Success(Results.Ok("")) - } def options(): Action[AnyContent] = entrypoint("DAV options") @@ -101,7 +103,7 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s def downloadFile(id: String): Action[AnyContent] = entrypoint("download attachment") .authRoTransaction(db) { request => implicit graph => - attachmentSrv.getOrFail(id).map { attachment => + attachmentSrv.getOrFail(EntityIdOrName(id)).map { attachment => val range = request.headers.get("Range") range match { case Some(rangeExtract(from, maybeTo)) => diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 5912a61e08..8677037f5a 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -7,11 +7,11 @@ import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ -import org.thp.scalligraph.models.{Database, IdMapping, UMapping} +import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} -import org.thp.scalligraph.{AuthorizationError, BadRequestError, InvalidFormatAttributeError, RichSeq} +import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityId, EntityIdOrName, EntityName, InvalidFormatAttributeError, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase} import org.thp.thehive.models._ @@ -53,7 +53,7 @@ class AlertCtrl @Inject() ( val inputAlert: InputAlert = request.body("alert") val observables: Seq[InputObservable] = request.body("observables") val customFields = inputAlert.customFields.map(c => c.name -> c.value).toMap - val caseTemplate = caseTemplateName.flatMap(caseTemplateSrv.get(_).visible.headOption) + val caseTemplate = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption) for { organisation <- userSrv @@ -83,7 +83,8 @@ class AlertCtrl @Inject() ( .withFieldConst(_.iocCount, similarStats.ioc._2) .withFieldConst(_.similarArtifactCount, similarStats.observable._1) .withFieldConst(_.similarIocCount, similarStats.ioc._1) - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.id, _._id.toString) .withFieldRenamed(_.number, _.caseId) .withFieldComputed(_.status, _.status.toString) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) @@ -100,7 +101,7 @@ class AlertCtrl @Inject() ( val similarity: Option[Boolean] = request.body("similarity") val alert = alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .visible if (similarity.contains(true)) alert @@ -125,13 +126,13 @@ class AlertCtrl @Inject() ( } - def update(alertId: String): Action[AnyContent] = + def update(alertIdOrName: String): Action[AnyContent] = entrypoint("update alert") .extract("alert", FieldsParser.update("alert", publicData.publicProperties)) .authTransaction(db) { implicit request => implicit graph => val propertyUpdaters: Seq[PropertyUpdater] = request.body("alert") alertSrv - .update(_.get(alertId).can(Permissions.manageAlert), propertyUpdaters) + .update(_.get(EntityIdOrName(alertIdOrName)).can(Permissions.manageAlert), propertyUpdaters) .flatMap { case (alertSteps, _) => alertSteps.richAlert.getOrFail("Alert") } .map { richAlert => val alertWithObservables: (RichAlert, Seq[RichObservable]) = richAlert -> alertSrv.get(richAlert.alert).observables.richObservable.toSeq @@ -139,13 +140,13 @@ class AlertCtrl @Inject() ( } } - def delete(alertId: String): Action[AnyContent] = + def delete(alertIdOrName: String): Action[AnyContent] = entrypoint("delete alert") .authTransaction(db) { implicit request => implicit graph => for { alert <- alertSrv - .get(alertId) + .get(EntityIdOrName(alertIdOrName)) .can(Permissions.manageAlert) .getOrFail("Alert") _ <- alertSrv.remove(alert) @@ -162,7 +163,7 @@ class AlertCtrl @Inject() ( for { alert <- alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .can(Permissions.manageAlert) .getOrFail("Alert") _ <- alertSrv.remove(alert) @@ -171,14 +172,14 @@ class AlertCtrl @Inject() ( .map(_ => Results.NoContent) } - def mergeWithCase(alertId: String, caseId: String): Action[AnyContent] = + def mergeWithCase(alertIdOrName: String, caseIdOrName: String): Action[AnyContent] = entrypoint("merge alert with case") .authTransaction(db) { implicit request => implicit graph => for { - alert <- alertSrv.get(alertId).can(Permissions.manageAlert).getOrFail("Alert") - case0 <- caseSrv.get(caseId).can(Permissions.manageCase).getOrFail("Case") + alert <- alertSrv.get(EntityIdOrName(alertIdOrName)).can(Permissions.manageAlert).getOrFail("Alert") + case0 <- caseSrv.get(EntityIdOrName(caseIdOrName)).can(Permissions.manageCase).getOrFail("Case") _ <- alertSrv.mergeInCase(alert, case0) - richCase <- caseSrv.get(caseId).richCase.getOrFail("Case") + richCase <- caseSrv.get(EntityIdOrName(caseIdOrName)).richCase.getOrFail("Case") } yield Results.Ok(richCase.toJson) } @@ -190,15 +191,15 @@ class AlertCtrl @Inject() ( val alertIds: Seq[String] = request.body("alertIds") val caseId: String = request.body("caseId") for { - case0 <- caseSrv.get(caseId).can(Permissions.manageCase).getOrFail("Case") + case0 <- caseSrv.get(EntityIdOrName(caseId)).can(Permissions.manageCase).getOrFail("Case") _ <- alertIds.toTry { alertId => alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .can(Permissions.manageAlert) .getOrFail("Alert") .flatMap(alertSrv.mergeInCase(_, case0)) } - richCase <- caseSrv.get(caseId).richCase.getOrFail("Case") + richCase <- caseSrv.get(EntityIdOrName(caseId)).richCase.getOrFail("Case") } yield Results.Ok(richCase.toJson) } @@ -206,11 +207,11 @@ class AlertCtrl @Inject() ( entrypoint("mark alert as read") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .can(Permissions.manageAlert) .existsOrFail .map { _ => - alertSrv.markAsRead(alertId) + alertSrv.markAsRead(EntityIdOrName(alertId)) Results.NoContent } } @@ -219,11 +220,11 @@ class AlertCtrl @Inject() ( entrypoint("mark alert as unread") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .can(Permissions.manageAlert) .existsOrFail .map { _ => - alertSrv.markAsUnread(alertId) + alertSrv.markAsUnread(EntityIdOrName(alertId)) Results.NoContent } } @@ -236,7 +237,7 @@ class AlertCtrl @Inject() ( for { (alert, organisation) <- alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .can(Permissions.manageAlert) .alertUserOrganisation(Permissions.manageCase) .getOrFail("Alert") @@ -249,11 +250,11 @@ class AlertCtrl @Inject() ( entrypoint("follow alert") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .can(Permissions.manageAlert) .existsOrFail .map { _ => - alertSrv.followAlert(alertId) + alertSrv.followAlert(EntityIdOrName(alertId)) Results.NoContent } } @@ -262,11 +263,11 @@ class AlertCtrl @Inject() ( entrypoint("unfollow alert") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertId)) .can(Permissions.manageAlert) .existsOrFail .map { _ => - alertSrv.unfollowAlert(alertId) + alertSrv.unfollowAlert(EntityIdOrName(alertId)) Results.NoContent } } @@ -276,7 +277,7 @@ class AlertCtrl @Inject() ( authContext: AuthContext ): Try[Seq[RichObservable]] = observableTypeSrv - .getOrFail(observable.dataType) + .getOrFail(EntityName(observable.dataType)) .flatMap { case attachmentType if attachmentType.isAttachment => observable.data.map(_.split(';')).toTry { @@ -296,16 +297,17 @@ class AlertCtrl @Inject() ( class PublicAlert @Inject() ( alertSrv: AlertSrv, organisationSrv: OrganisationSrv, - customFieldSrv: CustomFieldSrv + customFieldSrv: CustomFieldSrv, + @Named("with-thehive-schema") db: Database ) extends PublicData { override val entityName: String = "alert" override val initialQuery: Query = Query .init[Traversal.V[Alert]]("listAlert", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).alerts) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Alert]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Alert]]( "getAlert", - FieldsParser[IdOrName], - (param, graph, authContext) => alertSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => alertSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Alert], IteratorOutput]( @@ -382,18 +384,18 @@ class PublicAlert @Inject() ( }.custom { case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) => for { - c <- alertSrv.getByIds(vertex.id.toString)(graph).getOrFail("Alert") + c <- alertSrv.getByIds(EntityId(vertex.id))(graph).getOrFail("Alert") _ <- alertSrv.setOrCreateCustomField(c, name, Some(value))(graph, authContext) } yield Json.obj(s"customField.$name" -> value) case (FPathElem(_, FPathEmpty), values: JsObject, vertex, _, graph, authContext) => for { c <- alertSrv.get(vertex)(graph).getOrFail("Alert") - cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(n)(graph).map(_ -> v) } + cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(EntityIdOrName(n))(graph).map(_ -> v) } _ <- alertSrv.updateCustomField(c, cfv)(graph, authContext) } yield Json.obj("customFields" -> values) case _ => Failure(BadRequestError("Invalid custom fields format")) }) - .property("case", IdMapping)(_.select(_.`case`._id).readonly) + .property("case", db.idMapping)(_.select(_.`case`._id).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/AttachmentCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AttachmentCtrl.scala index f7aee5c2dc..7ca3615bd2 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AttachmentCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AttachmentCtrl.scala @@ -7,11 +7,11 @@ import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.ZipParameters import net.lingala.zip4j.model.enums.{CompressionLevel, EncryptionMethod} -import org.thp.scalligraph.NotFoundError import org.thp.scalligraph.controllers.Entrypoint import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{EntityIdOrName, NotFoundError} import org.thp.thehive.controllers.HttpHeaderParameterEncoding import org.thp.thehive.services.AttachmentOps._ import org.thp.thehive.services.AttachmentSrv @@ -38,7 +38,7 @@ class AttachmentCtrl @Inject() ( Success(Results.BadRequest("File name is invalid")) else attachmentSrv - .get(id) + .get(EntityIdOrName(id)) .visible .getOrFail("Attachment") .filter(attachmentSrv.exists) @@ -66,7 +66,7 @@ class AttachmentCtrl @Inject() ( Success(Results.BadRequest("File name is invalid")) else attachmentSrv - .get(id) + .get(EntityIdOrName(id)) .visible .getOrFail("Attachment") .filter(attachmentSrv.exists) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AuditCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AuditCtrl.scala index 8fc026423c..1e14c0873f 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AuditCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AuditCtrl.scala @@ -4,8 +4,9 @@ import akka.actor.ActorRef import akka.pattern.ask import akka.util.Timeout import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} -import org.thp.scalligraph.models.{Database, IdMapping, UMapping} +import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} @@ -36,7 +37,7 @@ class AuditCtrl @Inject() ( def flow(caseId: Option[String]): Action[AnyContent] = entrypoint("audit flow") .asyncAuth { implicit request => - (flowActor ? FlowId(request.organisation, caseId.filterNot(_ == "any"))).map { + (flowActor ? FlowId(request.organisation, caseId.filterNot(_ == "any").map(EntityIdOrName(_)))).map { case AuditIds(auditIds) if auditIds.isEmpty => Results.Ok(JsArray.empty) case AuditIds(auditIds) => val audits = db.roTransaction { implicit graph => @@ -64,11 +65,11 @@ class AuditCtrl @Inject() ( } @Singleton -class PublicAudit @Inject() (auditSrv: AuditSrv) extends PublicData { - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Audit]]( +class PublicAudit @Inject() (auditSrv: AuditSrv, @Named("with-thehive-schema") db: Database) extends PublicData { + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Audit]]( "getAudit", - FieldsParser[IdOrName], - (param, graph, authContext) => auditSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => auditSrv.get(idOrName)(graph).visible(authContext) ) override val entityName: String = "audit" @@ -93,6 +94,6 @@ class PublicAudit @Inject() (auditSrv: AuditSrv) extends PublicData { .property("base", UMapping.boolean)(_.rename("mainAction").readonly) .property("startDate", UMapping.date)(_.rename("_createdAt").readonly) .property("requestId", UMapping.string)(_.field.readonly) - .property("rootId", IdMapping)(_.select(_.context._id).readonly) + .property("rootId", db.idMapping)(_.select(_.context._id).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/AuthenticationCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AuthenticationCtrl.scala index 8455440b90..7c679074f2 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AuthenticationCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AuthenticationCtrl.scala @@ -1,11 +1,11 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.AuthorizationError import org.thp.scalligraph.auth.{AuthSrv, RequestOrganisation} import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{AuthorizationError, EntityIdOrName, EntityName} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.services.UserOps._ import org.thp.thehive.services.UserSrv @@ -23,9 +23,10 @@ class AuthenticationCtrl @Inject() ( implicit val ec: ExecutionContext ) { - def logout: Action[AnyContent] = entrypoint("logout") { _ => - Success(Results.Ok.withNewSession) - } + def logout: Action[AnyContent] = + entrypoint("logout") { _ => + Success(Results.Ok.withNewSession) + } def login: Action[AnyContent] = entrypoint("login") @@ -33,17 +34,14 @@ class AuthenticationCtrl @Inject() ( .extract("password", FieldsParser[String].on("password")) .extract("organisation", FieldsParser[String].optional.on("organisation")) .extract("code", FieldsParser[String].optional.on("code")) { implicit request => - val login: String = request.body("login") - val password: String = request.body("password") - val organisation: Option[String] = request.body("organisation") orElse requestOrganisation(request) - val code: Option[String] = request.body("code") - db.roTransaction { implicit graph => - for { - authContext <- authSrv.authenticate(login, password, organisation, code) - user <- db.roTransaction(userSrv.getOrFail(authContext.userId)(_)) - _ <- if (user.locked) Failure(AuthorizationError("Your account is locked")) else Success(()) - body = organisation.flatMap(userSrv.get(user).richUser(_).headOption).fold(user.toJson)(_.toJson) - } yield authSrv.setSessionUser(authContext)(Results.Ok(body)) - } + val login: String = request.body("login") + val password: String = request.body("password") + val organisation: Option[EntityIdOrName] = request.body("organisation").map(EntityIdOrName(_)) orElse requestOrganisation(request) + val code: Option[String] = request.body("code") + for { + authContext <- authSrv.authenticate(login, password, organisation, code) + user <- db.roTransaction(userSrv.get(EntityName(authContext.userId))(_).richUser(authContext).getOrFail("User")) + _ <- if (user.locked) Failure(AuthorizationError("Your account is locked")) else Success(()) + } yield authSrv.setSessionUser(authContext)(Results.Ok(user.toJson)) } } diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index e8f7d3cb4b..53bbd8b1c8 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -54,10 +54,10 @@ class CaseCtrl @Inject() ( .organisations(Permissions.manageCase) .get(request.organisation) .orFail(AuthorizationError("Operation not permitted")) - caseTemplate <- caseTemplateName.map(caseTemplateSrv.get(_).visible.richCaseTemplate.getOrFail("CaseTemplate")).flip - user <- inputCase.user.map(userSrv.get(_).visible.getOrFail("User")).flip + caseTemplate <- caseTemplateName.map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.richCaseTemplate.getOrFail("CaseTemplate")).flip + user <- inputCase.user.map(u => userSrv.get(EntityIdOrName(u)).visible.getOrFail("User")).flip tags <- inputCase.tags.toTry(tagSrv.getOrCreate) - tasks <- inputTasks.toTry(t => t.owner.map(userSrv.getOrFail).flip.map(owner => t.toTask -> owner)) + tasks <- inputTasks.toTry(t => t.owner.map(o => userSrv.getOrFail(EntityIdOrName(o))).flip.map(owner => t.toTask -> owner)) richCase <- caseSrv.create( caseTemplate.fold(inputCase)(inputCase.withCaseTemplate).toCase, user, @@ -75,7 +75,7 @@ class CaseCtrl @Inject() ( .extract("stats", FieldsParser.boolean.optional.on("nstats")) .authRoTransaction(db) { implicit request => implicit graph => val c = caseSrv - .get(caseIdOrNumber) + .get(EntityIdOrName(caseIdOrNumber)) .visible val stats: Option[Boolean] = request.body("stats") if (stats.contains(true)) @@ -97,7 +97,7 @@ class CaseCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("case") caseSrv .update( - _.get(caseIdOrNumber) + _.get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase), propertyUpdaters ) @@ -121,7 +121,7 @@ class CaseCtrl @Inject() ( .toTry { caseIdOrNumber => caseSrv .update( - _.get(caseIdOrNumber).can(Permissions.manageCase), + _.get(EntityIdOrName(caseIdOrNumber)).can(Permissions.manageCase), propertyUpdaters ) .flatMap { @@ -138,7 +138,7 @@ class CaseCtrl @Inject() ( entrypoint("delete case") .authTransaction(db) { implicit request => implicit graph => caseSrv - .get(caseIdOrNumber) + .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase) .update(_.status, CaseStatus.Deleted) .getOrFail("Case") @@ -151,7 +151,7 @@ class CaseCtrl @Inject() ( for { c <- caseSrv - .get(caseIdOrNumber) + .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase) .getOrFail("Case") _ <- caseSrv.remove(c) @@ -164,9 +164,9 @@ class CaseCtrl @Inject() ( caseIdsOrNumbers .split(',') .toSeq - .toTry( + .toTry(c => caseSrv - .get(_) + .get(EntityIdOrName(c)) .visible .getOrFail("Case") ) @@ -180,7 +180,7 @@ class CaseCtrl @Inject() ( entrypoint("case link") .authRoTransaction(db) { implicit request => implicit graph => val relatedCases = caseSrv - .get(caseIdOrNumber) + .get(EntityIdOrName(caseIdOrNumber)) .visible .linkedCases .map { @@ -200,16 +200,16 @@ class PublicCase @Inject() ( organisationSrv: OrganisationSrv, userSrv: UserSrv, customFieldSrv: CustomFieldSrv, - @Named("with-thehive-schema") db: Database + @Named("with-thehive-schema") implicit val db: Database ) extends PublicData with CaseRenderer { override val entityName: String = "case" override val initialQuery: Query = Query.init[Traversal.V[Case]]("listCase", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).cases) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Case]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Case]]( "getCase", - FieldsParser[IdOrName], - (param, graph, authContext) => caseSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => caseSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Case], IteratorOutput]( "page", @@ -256,7 +256,7 @@ class PublicCase @Inject() ( .property("owner", UMapping.string.optional)(_.select(_.user.value(_.login)).custom { (_, login, vertex, _, graph, authContext) => for { c <- caseSrv.get(vertex)(graph).getOrFail("Case") - user <- login.map(userSrv.get(_)(graph).getOrFail("User")).flip + user <- login.map(u => userSrv.get(EntityIdOrName(u))(graph).getOrFail("User")).flip _ <- user match { case Some(u) => caseSrv.assign(c, u)(graph, authContext) case None => caseSrv.unassign(c)(graph, authContext) @@ -276,7 +276,7 @@ class PublicCase @Inject() ( .property("impactStatus", UMapping.string.optional)(_.select(_.impactStatus.value(_.value)).custom { (_, impactStatus, vertex, _, graph, authContext) => for { - c <- caseSrv.getByIds(vertex.id.toString)(graph).getOrFail("Case") + c <- caseSrv.get(vertex)(graph).getOrFail("Case") _ <- impactStatus match { case Some(s) => caseSrv.setImpactStatus(c, s)(graph, authContext) case None => caseSrv.unsetImpactStatus(c)(graph, authContext) @@ -286,20 +286,20 @@ class PublicCase @Inject() ( .property("customFields", UMapping.jsonNative)(_.subSelect { case (FPathElem(_, FPathElem(name, _)), caseSteps) => caseSteps - .customFields(name) + .customFields(EntityIdOrName(name)) .jsonValue case (_, caseSteps) => caseSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) } .filter { case (FPathElem(_, FPathElem(name, _)), caseTraversal) => db - .roTransaction(implicit graph => customFieldSrv.get(name).value(_.`type`).getOrFail("CustomField")) + .roTransaction(implicit graph => customFieldSrv.get(EntityIdOrName(name)).value(_.`type`).getOrFail("CustomField")) .map { - case CustomFieldType.boolean => caseTraversal.customFields(name).value(_.booleanValue) - case CustomFieldType.date => caseTraversal.customFields(name).value(_.dateValue) - case CustomFieldType.float => caseTraversal.customFields(name).value(_.floatValue) - case CustomFieldType.integer => caseTraversal.customFields(name).value(_.integerValue) - case CustomFieldType.string => caseTraversal.customFields(name).value(_.stringValue) + case CustomFieldType.boolean => caseTraversal.customFields(EntityIdOrName(name)).value(_.booleanValue) + case CustomFieldType.date => caseTraversal.customFields(EntityIdOrName(name)).value(_.dateValue) + case CustomFieldType.float => caseTraversal.customFields(EntityIdOrName(name)).value(_.floatValue) + case CustomFieldType.integer => caseTraversal.customFields(EntityIdOrName(name)).value(_.integerValue) + case CustomFieldType.string => caseTraversal.customFields(EntityIdOrName(name)).value(_.stringValue) } .getOrElse(caseTraversal.constant2(null)) case (_, caseTraversal) => caseTraversal.constant2(null) @@ -308,7 +308,7 @@ class PublicCase @Inject() ( case FPathElem(_, FPathElem(name, _)) => db .roTransaction { implicit graph => - customFieldSrv.get(name).value(_.`type`).getOrFail("CustomField") + customFieldSrv.get(EntityIdOrName(name)).value(_.`type`).getOrFail("CustomField") } .map { case CustomFieldType.boolean => new Converter[Any, JsValue] { def apply(x: JsValue): Any = x.as[Boolean] } @@ -323,13 +323,13 @@ class PublicCase @Inject() ( .custom { case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) => for { - c <- caseSrv.getByIds(vertex.id.toString)(graph).getOrFail("Case") - _ <- caseSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) + c <- caseSrv.get(vertex)(graph).getOrFail("Case") + _ <- caseSrv.setOrCreateCustomField(c, EntityIdOrName(name), Some(value), None)(graph, authContext) } yield Json.obj(s"customField.$name" -> value) case (FPathElem(_, FPathEmpty), values: JsObject, vertex, _, graph, authContext) => for { c <- caseSrv.get(vertex)(graph).getOrFail("Case") - cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(n)(graph).map(cf => (cf, v, None)) } + cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(EntityIdOrName(n))(graph).map(cf => (cf, v, None)) } _ <- caseSrv.updateCustomField(c, cfv)(graph, authContext) } yield Json.obj("customFields" -> values) case _ => Failure(BadRequestError("Invalid custom fields format")) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala index 631b7ce686..adb4a1195f 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala @@ -3,12 +3,14 @@ package org.thp.thehive.controllers.v0 import java.lang.{Long => JLong} import java.util.{Collection => JCollection, List => JList, Map => JMap} +import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.traversal.Converter.CList import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IdentityConverter, Traversal} import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ import play.api.libs.json._ @@ -55,8 +57,12 @@ trait CaseRenderer { def shareCountStats: Traversal.V[Case] => Traversal[Long, JLong, Converter[Long, JLong]] = _.organisations.count - def isOwnerStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[Boolean, String, Converter[Boolean, String]] = - _.origin.value(_.name).domainMap(_ == authContext.organisation) + def isOwnerStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[Boolean, Boolean, Converter.Identity[Boolean]] = + _.origin + .current + .fold + .count + .choose(_.is(P.gt(0)), onTrue = true, onFalse = false) def caseStatsRenderer(implicit authContext: AuthContext diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala index c93bf9b70d..b8f36f5df3 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala @@ -7,7 +7,7 @@ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.scalligraph.{AttributeCheckingError, BadRequestError, RichSeq} +import org.thp.scalligraph.{AttributeCheckingError, BadRequestError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputCaseTemplate, InputTask} import org.thp.thehive.models.{CaseTemplate, Permissions, RichCaseTemplate} @@ -40,7 +40,7 @@ class CaseTemplateCtrl @Inject() ( val inputCaseTemplate: InputCaseTemplate = request.body("caseTemplate") val customFields = inputCaseTemplate.customFields.sortBy(_.order.getOrElse(0)).map(c => c.name -> c.value) for { - tasks <- inputCaseTemplate.tasks.toTry(t => t.owner.map(userSrv.getOrFail).flip.map(t.toTask -> _)) + tasks <- inputCaseTemplate.tasks.toTry(t => t.owner.map(o => userSrv.getOrFail(EntityIdOrName(o))).flip.map(t.toTask -> _)) organisation <- userSrv.current.organisations(Permissions.manageCaseTemplate).get(request.organisation).getOrFail("CaseTemplate") richCaseTemplate <- caseTemplateSrv.create(inputCaseTemplate.toCaseTemplate, organisation, inputCaseTemplate.tags, tasks, customFields) } yield Results.Created(richCaseTemplate.toJson) @@ -50,7 +50,7 @@ class CaseTemplateCtrl @Inject() ( entrypoint("get case template") .authRoTransaction(db) { implicit request => implicit graph => caseTemplateSrv - .get(caseTemplateNameOrId) + .get(EntityIdOrName(caseTemplateNameOrId)) .visible .richCaseTemplate .getOrFail("CaseTemplate") @@ -64,7 +64,7 @@ class CaseTemplateCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("caseTemplate") caseTemplateSrv .update( - _.get(caseTemplateNameOrId) + _.get(EntityIdOrName(caseTemplateNameOrId)) .can(Permissions.manageCaseTemplate), propertyUpdaters ) @@ -76,7 +76,7 @@ class CaseTemplateCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => for { organisation <- organisationSrv.getOrFail(request.organisation) - template <- caseTemplateSrv.get(caseTemplateNameOrId).can(Permissions.manageCaseTemplate).getOrFail("CaseTemplate") + template <- caseTemplateSrv.get(EntityIdOrName(caseTemplateNameOrId)).can(Permissions.manageCaseTemplate).getOrFail("CaseTemplate") _ = caseTemplateSrv.get(template).remove() _ <- auditSrv.caseTemplate.delete(template, organisation) } yield Results.Ok @@ -96,10 +96,10 @@ class PublicCaseTemplate @Inject() ( override val initialQuery: Query = Query .init[Traversal.V[CaseTemplate]]("listCaseTemplate", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).caseTemplates) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[CaseTemplate]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[CaseTemplate]]( "getCaseTemplate", - FieldsParser[IdOrName], - (param, graph, authContext) => caseTemplateSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => caseTemplateSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[CaseTemplate], IteratorOutput]( "page", @@ -134,13 +134,13 @@ class PublicCaseTemplate @Inject() ( }.custom { case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) => for { - c <- caseTemplateSrv.getByIds(vertex.id.toString)(graph).getOrFail("CaseTemplate") + c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") _ <- caseTemplateSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) } yield Json.obj(s"customFields.$name" -> value) case (FPathElem(_, FPathEmpty), values: JsObject, vertex, _, graph, authContext) => for { c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") - cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(n)(graph).map(_ -> v) } + cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(EntityIdOrName(n))(graph).map(_ -> v) } _ <- caseTemplateSrv.updateCustomField(c, cfv)(graph, authContext) } yield Json.obj("customFields" -> values) case _ => Failure(BadRequestError("Invalid custom fields format")) @@ -151,11 +151,16 @@ class PublicCaseTemplate @Inject() ( caseTemplateSrv.get(vertex)(graph).tasks.remove() for { - caseTemplate <- caseTemplateSrv.getByIds(vertex.id.toString)(graph).getOrFail("CaseTemplate") + caseTemplate <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") tasks <- value.validatedBy(t => fp(Field(t))).badMap(AttributeCheckingError(_)).toTry createdTasks <- tasks - .toTry(t => t.owner.map(userSrv.getOrFail(_)(graph)).flip.flatMap(owner => taskSrv.create(t.toTask, owner)(graph, authContext))) + .toTry(t => + t.owner + .map(o => userSrv.getOrFail(EntityIdOrName(o))(graph)) + .flip + .flatMap(owner => taskSrv.create(t.toTask, owner)(graph, authContext)) + ) _ <- createdTasks.toTry(t => caseTemplateSrv.addTask(caseTemplate, t.task)(graph, authContext)) } yield Json.obj("tasks" -> createdTasks.map(_.toJson)) }) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala index df2a2d3d74..8f63df3e47 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala @@ -1,12 +1,14 @@ package org.thp.thehive.controllers.v0 import com.typesafe.config.{Config, ConfigRenderOptions} -import javax.inject.{Inject, Singleton} +import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} +import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.{AuthorizationError, NotFoundError} import org.thp.thehive.models.Permissions -import org.thp.thehive.services.{OrganisationConfigContext, UserConfigContext} +import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.{OrganisationConfigContext, OrganisationSrv, UserConfigContext} import play.api.libs.json.{JsNull, JsValue, Json, Writes} import play.api.mvc.{Action, AnyContent, Results} import play.api.{ConfigLoader, Logger} @@ -18,7 +20,9 @@ class ConfigCtrl @Inject() ( appConfig: ApplicationConfig, userConfigContext: UserConfigContext, organisationConfigContext: OrganisationConfigContext, - entrypoint: Entrypoint + organisationSrv: OrganisationSrv, + entrypoint: Entrypoint, + @Named("with-thehive-schema") db: Database ) { lazy val logger: Logger = Logger(getClass) @@ -37,8 +41,8 @@ class ConfigCtrl @Inject() ( def list: Action[AnyContent] = entrypoint("list configuration items") - .authPermitted(Permissions.manageConfig) { request => - if (request.organisation != "admin") + .authPermittedTransaction(db, Permissions.manageConfig) { implicit request => implicit graph => + if (!organisationSrv.current.isAdmin) Failure(AuthorizationError("You must be in `admin` organisation to view global configuration")) else Success(Results.Ok(Json.toJson(appConfig.list))) @@ -47,8 +51,8 @@ class ConfigCtrl @Inject() ( def set(path: String): Action[AnyContent] = entrypoint("set configuration item") .extract("value", FieldsParser.json.on("value")) - .authPermitted(Permissions.manageConfig) { request => - if (request.organisation != "admin") + .authPermittedTransaction(db, Permissions.manageConfig) { implicit request => implicit graph => + if (!organisationSrv.current.isAdmin) Failure(AuthorizationError("You must be in `admin` organisation to change global configuration")) else { logger.info(s"app config value set: $path ${request.body("value")}") @@ -58,8 +62,8 @@ class ConfigCtrl @Inject() ( def get(path: String): Action[AnyContent] = entrypoint("get configuration item") - .authPermitted(Permissions.manageConfig) { request => - if (request.organisation != "admin") + .authPermittedTransaction(db, Permissions.manageConfig) { implicit request => implicit graph => + if (!organisationSrv.current.isAdmin) Failure(AuthorizationError("You must be in `admin` organisation to change global configuration")) else appConfig.get(path) match { diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index b260be6fb9..145981b6cc 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -18,21 +18,23 @@ object Conversion { val adminPermissions: Set[Permission] = Set(Permissions.manageUser, Permissions.manageOrganisation) - def actionToOperation(action: String): String = action match { - case "create" => "Creation" - case "update" => "Update" - case "delete" => "Delete" - case _ => "Unknown" - } + def actionToOperation(action: String): String = + action match { + case "create" => "Creation" + case "update" => "Update" + case "delete" => "Delete" + case _ => "Unknown" + } - def fromObjectType(objectType: String): String = objectType match { - // case "Case" =>"case" - case "Task" => "case_task" - case "Log" => "case_task_log" - case "Observable" => "case_artifact" - case "Job" => "case_artifact_job" - case other => other.toLowerCase() - } + def fromObjectType(objectType: String): String = + objectType match { + // case "Case" =>"case" + case "Task" => "case_task" + case "Log" => "case_task_log" + case "Observable" => "case_artifact" + case "Job" => "case_artifact_job" + case other => other.toLowerCase() + } implicit val alertOutput: Renderer.Aux[RichAlert, OutputAlert] = Renderer.toJson[RichAlert, OutputAlert](richAlert => richAlert @@ -40,10 +42,12 @@ object Conversion { .withFieldComputed(_.customFields, rc => JsObject(rc.customFields.map(cf => cf.name -> Json.obj(cf.typeName -> cf.toJson)))) .withFieldRenamed(_._createdAt, _.createdAt) .withFieldRenamed(_._createdBy, _.createdBy) - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.id, _._id.toString) + .withFieldComputed(_.id, _._id.toString) .withFieldConst(_._type, "alert") .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) - .withFieldComputed(_.`case`, _.caseId) + .withFieldComputed(_.`case`, _.caseId.map(_.toString)) .withFieldComputed( _.status, alert => @@ -64,12 +68,13 @@ object Conversion { ._1 .into[OutputAlert] .withFieldComputed(_.customFields, rc => JsObject(rc.customFields.map(cf => cf.name -> Json.obj(cf.typeName -> cf.toJson)))) - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.id, _._id.toString) .withFieldRenamed(_._createdAt, _.createdAt) .withFieldRenamed(_._createdBy, _.createdBy) .withFieldConst(_._type, "alert") .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) - .withFieldComputed(_.`case`, _.caseId) + .withFieldComputed(_.`case`, _.caseId.map(_.toString)) .withFieldComputed( _.status, alert => @@ -111,16 +116,17 @@ object Conversion { implicit val auditOutput: Renderer.Aux[RichAudit, OutputAudit] = Renderer.toJson[RichAudit, OutputAudit]( _.into[OutputAudit] .withFieldComputed(_.operation, a => actionToOperation(a.action)) - .withFieldComputed(_.id, _._id) + .withFieldComputed(_.id, _._id.toString) + .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.createdAt, _._createdAt) .withFieldComputed(_.createdBy, _._createdBy) .withFieldConst(_._type, "audit") .withFieldComputed(_.`object`, _.`object`.map(OutputEntity.apply)) //objectToJson)) .withFieldConst(_.base, true) .withFieldComputed(_.details, a => Json.parse(a.details.getOrElse("{}")).as[JsObject]) - .withFieldComputed(_.objectId, a => a.objectId.getOrElse(a.context._id)) + .withFieldComputed(_.objectId, a => a.objectId.getOrElse(a.context._id).toString) .withFieldComputed(_.objectType, a => fromObjectType(a.objectType.getOrElse(a.context._label))) - .withFieldComputed(_.rootId, _.context._id) + .withFieldComputed(_.rootId, _.context._id.toString) .withFieldComputed(_.startDate, _._createdAt) .withFieldComputed( _.summary, @@ -137,7 +143,8 @@ object Conversion { ) .withFieldComputed(_.status, _.status.toString) .withFieldConst(_._type, "case") - .withFieldComputed(_.id, _._id) + .withFieldComputed(_.id, _._id.toString) + .withFieldComputed(_._id, _._id.toString) .withFieldRenamed(_.number, _.caseId) .withFieldRenamed(_.assignee, _.owner) .withFieldRenamed(_._updatedAt, _.updatedAt) @@ -190,7 +197,8 @@ object Conversion { .withFieldComputed(_.customFields, rc => JsObject(rc.customFields.map(cf => cf.name -> Json.obj(cf.typeName -> cf.toJson)))) .withFieldComputed(_.status, _.status.toString) .withFieldConst(_._type, "case") - .withFieldComputed(_.id, _._id) + .withFieldComputed(_.id, _._id.toString) + .withFieldComputed(_._id, _._id.toString) .withFieldRenamed(_.number, _.caseId) .withFieldRenamed(_.assignee, _.owner) .withFieldRenamed(_._updatedAt, _.updatedAt) @@ -219,7 +227,8 @@ object Conversion { _.customFields, rc => JsObject(rc.customFields.map(cf => cf.name -> Json.obj(cf.typeName -> cf.toJson, "order" -> cf.order))) ) - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.id, _._id.toString) .withFieldRenamed(_._updatedAt, _.updatedAt) .withFieldRenamed(_._updatedBy, _.updatedBy) .withFieldRenamed(_._createdAt, _.createdAt) @@ -235,10 +244,13 @@ object Conversion { implicit val richCustomFieldOutput: Renderer.Aux[RichCustomField, OutputCustomFieldValue] = Renderer.toJson[RichCustomField, OutputCustomFieldValue]( _.into[OutputCustomFieldValue] - .withFieldComputed(_.value, _.value.map { - case d: Date => d.getTime.toString - case other => other.toString - }) + .withFieldComputed( + _.value, + _.value.map { + case d: Date => d.getTime.toString + case other => other.toString + } + ) .withFieldComputed(_.tpe, _.typeName) .transform ) @@ -263,15 +275,15 @@ object Conversion { .withFieldComputed(_.`type`, _.`type`.toString) .withFieldComputed(_.reference, _.name) .withFieldComputed(_.name, _.displayName) - .withFieldConst(_.id, customField._id) + .withFieldConst(_.id, customField._id.toString) .transform ) implicit val dashboardOutput: Renderer.Aux[RichDashboard, OutputDashboard] = Renderer.toJson[RichDashboard, OutputDashboard](dashboard => dashboard .into[OutputDashboard] - .withFieldConst(_.id, dashboard._id) - .withFieldConst(_._id, dashboard._id) + .withFieldConst(_.id, dashboard._id.toString) + .withFieldConst(_._id, dashboard._id.toString) .withFieldComputed(_.status, d => if (d.organisationShares.nonEmpty) "Shared" else "Private") .withFieldConst(_._type, "dashboard") .withFieldConst(_.updatedAt, dashboard._updatedAt) @@ -295,8 +307,8 @@ object Conversion { richLog .into[OutputLog] .withFieldConst(_._type, "case_task_log") - .withFieldComputed(_.id, _._id) - .withFieldComputed(_._id, _._id) + .withFieldComputed(_.id, _._id.toString) + .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.updatedAt, _._updatedAt) .withFieldComputed(_.updatedBy, _._updatedBy) .withFieldComputed(_.createdAt, _._createdAt) @@ -336,8 +348,8 @@ object Conversion { implicit val observableOutput: Renderer.Aux[RichObservable, OutputObservable] = Renderer.toJson[RichObservable, OutputObservable]( _.into[OutputObservable] .withFieldConst(_._type, "case_artifact") - .withFieldComputed(_.id, _.observable._id) - .withFieldComputed(_._id, _.observable._id) + .withFieldComputed(_.id, _.observable._id.toString) + .withFieldComputed(_._id, _.observable._id.toString) .withFieldComputed(_.updatedAt, _.observable._updatedAt) .withFieldComputed(_.updatedBy, _.observable._updatedBy) .withFieldComputed(_.createdAt, _.observable._createdAt) @@ -374,8 +386,8 @@ object Conversion { richObservable .into[OutputObservable] .withFieldConst(_._type, "case_artifact") - .withFieldComputed(_.id, _.observable._id) - .withFieldComputed(_._id, _.observable._id) + .withFieldComputed(_.id, _.observable._id.toString) + .withFieldComputed(_._id, _.observable._id.toString) .withFieldComputed(_.updatedAt, _.observable._updatedAt) .withFieldComputed(_.updatedBy, _.observable._updatedBy) .withFieldComputed(_.createdAt, _.observable._createdAt) @@ -386,7 +398,8 @@ object Conversion { .withFieldComputed(_.data, _.data.map(_.data)) .withFieldComputed(_.attachment, _.attachment.map(_.toValue)) .withFieldComputed( - _.reports, { a => + _.reports, + a => JsObject(a.reportTags.groupBy(_.origin).map { case (origin, tags) => origin -> Json.obj( @@ -394,7 +407,6 @@ object Conversion { .map(t => Json.obj("level" -> t.level.toString, "namespace" -> t.namespace, "predicate" -> t.predicate, "value" -> t.value)) ) }) - } ) .withFieldConst(_.stats, stats) .withFieldConst(_.`case`, richCase.map(_.toValue)) @@ -407,8 +419,8 @@ object Conversion { richObservable .into[OutputObservable] .withFieldConst(_._type, "case_artifact") - .withFieldComputed(_.id, _.observable._id) - .withFieldComputed(_._id, _.observable._id) + .withFieldComputed(_.id, _.observable._id.toString) + .withFieldComputed(_._id, _.observable._id.toString) .withFieldComputed(_.updatedAt, _.observable._updatedAt) .withFieldComputed(_.updatedBy, _.observable._updatedBy) .withFieldComputed(_.createdAt, _.observable._createdAt) @@ -419,7 +431,8 @@ object Conversion { .withFieldComputed(_.data, _.data.map(_.data)) .withFieldComputed(_.attachment, _.attachment.map(_.toValue)) .withFieldComputed( - _.reports, { a => + _.reports, + a => JsObject(a.reportTags.groupBy(_.origin).map { case (origin, tags) => origin -> Json.obj( @@ -427,7 +440,6 @@ object Conversion { .map(t => Json.obj("level" -> t.level.toString, "namespace" -> t.namespace, "predicate" -> t.predicate, "value" -> t.value)) ) }) - } ) .withFieldConst(_.stats, stats) .withFieldConst(_.`case`, None) @@ -447,8 +459,8 @@ object Conversion { OutputOrganisation( organisation.name, organisation.description, - organisation._id, - organisation._id, + organisation._id.toString, + organisation._id.toString, organisation._createdAt, organisation._createdBy, organisation._updatedAt, @@ -463,8 +475,8 @@ object Conversion { OutputOrganisation( organisation.name, organisation.description, - organisation._id, - organisation._id, + organisation._id.toString, + organisation._id.toString, organisation._createdAt, organisation._createdBy, organisation._updatedAt, @@ -478,8 +490,8 @@ object Conversion { profile .asInstanceOf[Profile] .into[OutputProfile] - .withFieldConst(_._id, profile._id) - .withFieldConst(_.id, profile._id) + .withFieldConst(_._id, profile._id.toString) + .withFieldConst(_.id, profile._id.toString) .withFieldConst(_.updatedAt, profile._updatedAt) .withFieldConst(_.updatedBy, profile._updatedBy) .withFieldConst(_.createdAt, profile._createdAt) @@ -502,7 +514,8 @@ object Conversion { implicit val shareOutput: Renderer.Aux[RichShare, OutputShare] = Renderer.toJson[RichShare, OutputShare]( _.into[OutputShare] - .withFieldComputed(_._id, _.share._id) + .withFieldComputed(_._id, _.share._id.toString) + .withFieldComputed(_.caseId, _.caseId.toString) .withFieldComputed(_.createdAt, _.share._createdAt) .withFieldComputed(_.createdBy, _.share._createdBy) .transform @@ -528,7 +541,8 @@ object Conversion { implicit val taskOutput: Renderer.Aux[RichTask, OutputTask] = Renderer.toJson[RichTask, OutputTask]( _.into[OutputTask] - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.id, _._id.toString) .withFieldComputed(_.status, _.status.toString) .withFieldConst(_._type, "case_task") .withFieldConst(_.`case`, None) @@ -545,7 +559,8 @@ object Conversion { case (richTask, richCase) => richTask .into[OutputTask] - .withFieldRenamed(_._id, _.id) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.id, _._id.toString) .withFieldComputed(_.status, _.status.toString) .withFieldConst(_._type, "case_task") .withFieldConst(_.`case`, richCase.map(_.toValue)) @@ -575,6 +590,7 @@ object Conversion { _.into[OutputUser] .withFieldComputed(_.roles, u => permissions2Roles(u.permissions)) .withFieldRenamed(_.login, _.id) + .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.hasKey, _.apikey.isDefined) .withFieldComputed(_.status, u => if (u.locked) "Locked" else "Ok") .withFieldRenamed(_._createdBy, _.createdBy) @@ -588,8 +604,8 @@ object Conversion { implicit val simpleUserOutput: Renderer.Aux[User with Entity, OutputUser] = Renderer.toJson[User with Entity, OutputUser](u => u.asInstanceOf[User] .into[OutputUser] - .withFieldConst(_._id, u._id) - .withFieldConst(_.id, u._id) + .withFieldConst(_._id, u._id.toString) + .withFieldConst(_.id, u._id.toString) .withFieldConst(_.organisation, "") .withFieldConst(_.roles, Set[String]()) .withFieldRenamed(_.login, _.id) @@ -603,15 +619,16 @@ object Conversion { .transform ) - implicit val pageOutput: Renderer.Aux[Page with Entity, OutputPage] = Renderer.toJson[Page with Entity, OutputPage](p => - p.asInstanceOf[Page] + implicit val pageOutput: Renderer.Aux[Page with Entity, OutputPage] = Renderer.toJson[Page with Entity, OutputPage](page => + page + .asInstanceOf[Page] .into[OutputPage] - .withFieldConst(_._id, p._id) - .withFieldConst(_.id, p._id) - .withFieldConst(_.createdBy, p._createdBy) - .withFieldConst(_.createdAt, p._createdAt) - .withFieldConst(_.updatedBy, p._updatedBy) - .withFieldConst(_.updatedAt, p._updatedAt) + .withFieldConst(_._id, page._id.toString) + .withFieldConst(_.id, page._id.toString) + .withFieldConst(_.createdBy, page._createdBy) + .withFieldConst(_.createdAt, page._createdAt) + .withFieldConst(_.updatedBy, page._updatedBy) + .withFieldConst(_.updatedAt, page._updatedAt) .withFieldConst(_._type, "page") .withFieldComputed(_.content, _.content) .withFieldComputed(_.title, _.title) @@ -622,15 +639,16 @@ object Conversion { Renderer.toJson[PermissionDesc, OutputPermission](_.into[OutputPermission].transform) implicit val observableTypeOutput: Renderer.Aux[ObservableType with Entity, OutputObservableType] = - Renderer.toJson[ObservableType with Entity, OutputObservableType](ot => - ot.asInstanceOf[ObservableType] + Renderer.toJson[ObservableType with Entity, OutputObservableType](outputObservableType => + outputObservableType + .asInstanceOf[ObservableType] .into[OutputObservableType] - .withFieldConst(_._id, ot._id) - .withFieldConst(_.id, ot._id) - .withFieldConst(_.createdBy, ot._createdBy) - .withFieldConst(_.createdAt, ot._createdAt) - .withFieldConst(_.updatedBy, ot._updatedBy) - .withFieldConst(_.updatedAt, ot._updatedAt) + .withFieldConst(_._id, outputObservableType._id.toString) + .withFieldConst(_.id, outputObservableType._id.toString) + .withFieldConst(_.createdBy, outputObservableType._createdBy) + .withFieldConst(_.createdAt, outputObservableType._createdAt) + .withFieldConst(_.updatedBy, outputObservableType._updatedBy) + .withFieldConst(_.updatedAt, outputObservableType._updatedAt) .withFieldConst(_._type, "observableType") .transform ) @@ -654,15 +672,16 @@ object Conversion { .transform } - def toObjectType(t: String): String = t match { - case "case" => "Case" - case "case_artifact" => "Observable" - case "case_task" => "Task" - case "case_task_log" => "Log" - case "alert" => "Alert" - case "case_artifact_job" => "Job" - case "action" => "Action" - } + def toObjectType(t: String): String = + t match { + case "case" => "Case" + case "case_artifact" => "Observable" + case "case_task" => "Task" + case "case_task_log" => "Log" + case "alert" => "Alert" + case "case_artifact_job" => "Job" + case "action" => "Action" + } def permissions2Roles(permissions: Set[Permission]): Set[String] = { val roles = diff --git a/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala index 76133ef5dc..fd9ae73206 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ @@ -46,7 +47,7 @@ class CustomFieldCtrl @Inject() ( def get(id: String): Action[AnyContent] = entrypoint("get custom field") .authRoTransaction(db) { _ => implicit graph => - customFieldSrv.get(id).getOrFail("CustomField").map(cf => Results.Ok(cf.toJson)) + customFieldSrv.get(EntityIdOrName(id)).getOrFail("CustomField").map(cf => Results.Ok(cf.toJson)) } def delete(id: String): Action[AnyContent] = @@ -55,7 +56,7 @@ class CustomFieldCtrl @Inject() ( .authPermittedTransaction(db, Permissions.manageCustomField) { implicit request => implicit graph => val force = request.body("force").getOrElse(false) for { - cf <- customFieldSrv.getOrFail(id) + cf <- customFieldSrv.getOrFail(EntityIdOrName(id)) _ <- customFieldSrv.delete(cf, force) } yield Results.NoContent } @@ -67,7 +68,7 @@ class CustomFieldCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("customField") for { - updated <- customFieldSrv.update(customFieldSrv.get(id), propertyUpdaters) + updated <- customFieldSrv.update(customFieldSrv.get(EntityIdOrName(id)), propertyUpdaters) cf <- updated._1.getOrFail("CustomField") } yield Results.Ok(cf.toJson) } @@ -75,7 +76,7 @@ class CustomFieldCtrl @Inject() ( def useCount(id: String): Action[AnyContent] = entrypoint("get use count of custom field") .authPermittedTransaction(db, Permissions.manageCustomField) { _ => implicit graph => - customFieldSrv.getOrFail(id).map(customFieldSrv.useCount).map { countMap => + customFieldSrv.getOrFail(EntityIdOrName(id)).map(customFieldSrv.useCount).map { countMap => val total = countMap.valuesIterator.sum val countStats = JsObject(countMap.map { case (k, v) => fromObjectType(k) -> JsNumber(v) @@ -98,10 +99,10 @@ class PublicCustomField @Inject() (customFieldSrv: CustomFieldSrv) extends Publi } ) override val outputQuery: Query = Query.output[CustomField with Entity] - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[CustomField]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[CustomField]]( "getCustomField", - FieldsParser[IdOrName], - (param, graph, _) => customFieldSrv.get(param.idOrName)(graph) + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => customFieldSrv.get(idOrName)(graph) ) override val publicProperties: PublicProperties = PublicPropertyListBuilder[CustomField] diff --git a/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala index a5d0bf4315..3402cd5798 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala @@ -1,12 +1,12 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.InvalidFormatAttributeError import org.thp.scalligraph.controllers.{Entrypoint, FString, FieldsParser} import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.{EntityIdOrName, InvalidFormatAttributeError} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputDashboard import org.thp.thehive.models.{Dashboard, RichDashboard} @@ -40,7 +40,7 @@ class DashboardCtrl @Inject() ( entrypoint("get dashboard") .authRoTransaction(db) { implicit request => implicit graph => dashboardSrv - .getByIds(dashboardId) + .get(EntityIdOrName(dashboardId)) .visible .richDashboard .getOrFail("Dashboard") @@ -53,7 +53,7 @@ class DashboardCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val propertyUpdaters: Seq[PropertyUpdater] = request.body("dashboard") dashboardSrv - .update(_.getByIds(dashboardId).canUpdate, propertyUpdaters) // TODO check permission + .update(_.get(EntityIdOrName(dashboardId)).canUpdate, propertyUpdaters) // TODO check permission .flatMap { case (dashboardSteps, _) => dashboardSteps.richDashboard.getOrFail("Dashboard") } .map(dashboard => Results.Ok(dashboard.toJson)) } @@ -64,7 +64,7 @@ class DashboardCtrl @Inject() ( userSrv .current .dashboards - .getByIds(dashboardId) + .get(EntityIdOrName(dashboardId)) .getOrFail("Dashboard") .map { dashboard => dashboardSrv.remove(dashboard) @@ -77,8 +77,7 @@ class DashboardCtrl @Inject() ( class PublicDashboard @Inject() ( dashboardSrv: DashboardSrv, organisationSrv: OrganisationSrv, - userSrv: UserSrv, - @Named("with-thehive-schema") db: Database + userSrv: UserSrv ) extends PublicData { val entityName: String = "dashboard" @@ -88,16 +87,16 @@ class PublicDashboard @Inject() ( (graph, authContext) => Traversal .union( - organisationSrv.filterTraversal(_).get(authContext.organisation)(db).dashboards, - userSrv.filterTraversal(_).get(authContext.userId)(db).dashboards + organisationSrv.filterTraversal(_).get(authContext.organisation).dashboards, + userSrv.filterTraversal(_).getByName(authContext.userId).dashboards )(graph) .dedup ) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Dashboard]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Dashboard]]( "getDashboard", - FieldsParser[IdOrName], - (param, graph, authContext) => dashboardSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => dashboardSrv.get(idOrName)(graph).visible(authContext) ) val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Dashboard], IteratorOutput]( diff --git a/thehive/app/org/thp/thehive/controllers/v0/ListCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ListCtrl.scala index 90e6946cec..2e0e96da88 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ListCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ListCtrl.scala @@ -46,7 +46,7 @@ class ListCtrl @Inject() ( .roTransaction { implicit grap => customFieldSrv.startTraversal.toSeq } - .map(cf => cf._id -> cf.toJson) + .map(cf => cf._id.toString -> cf.toJson) JsObject(cf) case _ => JsObject.empty } @@ -61,24 +61,26 @@ class ListCtrl @Inject() ( val value: JsObject = request.body("value") listName match { case "custom_fields" => { - for { - inputCustomField <- value.validate[InputCustomField] - } yield inputCustomField - } fold ( - errors => Failure(new Exception(errors.mkString)), - _ => Success(Results.Ok) - ) + for { + inputCustomField <- value.validate[InputCustomField] + } yield inputCustomField + } fold ( + errors => Failure(new Exception(errors.mkString)), + _ => Success(Results.Ok) + ) case _ => Success(Results.Locked("")) } } - def deleteItem(itemId: String): Action[AnyContent] = entrypoint("delete list item") { _ => - Success(Results.Locked("")) - } + def deleteItem(itemId: String): Action[AnyContent] = + entrypoint("delete list item") { _ => + Success(Results.Locked("")) + } - def updateItem(itemId: String): Action[AnyContent] = entrypoint("update list item") { _ => - Success(Results.Locked("")) - } + def updateItem(itemId: String): Action[AnyContent] = + entrypoint("update list item") { _ => + Success(Results.Locked("")) + } def itemExists(listName: String): Action[AnyContent] = entrypoint("check if item exist in list") diff --git a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala index a4b6c3ac35..583a43105f 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala @@ -1,8 +1,9 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} -import org.thp.scalligraph.models.{Database, IdMapping, UMapping} +import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} @@ -34,7 +35,7 @@ class LogCtrl @Inject() ( for { task <- taskSrv - .getByIds(taskId) + .get(EntityIdOrName(taskId)) .can(Permissions.manageTask) .getOrFail("Task") createdLog <- logSrv.create(inputLog.toLog, task) @@ -50,7 +51,7 @@ class LogCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("log") logSrv .update( - _.getByIds(logId) + _.get(EntityIdOrName(logId)) .can(Permissions.manageTask), propertyUpdaters ) @@ -61,7 +62,7 @@ class LogCtrl @Inject() ( entrypoint("delete log") .authTransaction(db) { implicit req => implicit graph => for { - log <- logSrv.get(logId).can(Permissions.manageTask).getOrFail("Log") + log <- logSrv.get(EntityIdOrName(logId)).can(Permissions.manageTask).getOrFail("Log") _ <- logSrv.cascadeRemove(log) } yield Results.NoContent } @@ -72,10 +73,10 @@ class PublicLog @Inject() (logSrv: LogSrv, organisationSrv: OrganisationSrv) ext override val entityName: String = "log" override val initialQuery: Query = Query.init[Traversal.V[Log]]("listLog", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).shares.tasks.logs) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Log]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Log]]( "getLog", - FieldsParser[IdOrName], - (param, graph, authContext) => logSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => logSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Log], IteratorOutput]( "page", @@ -89,6 +90,6 @@ class PublicLog @Inject() (logSrv: LogSrv, organisationSrv: OrganisationSrv) ext .property("deleted", UMapping.boolean)(_.field.updatable) .property("startDate", UMapping.date)(_.rename("date").readonly) .property("status", UMapping.string)(_.select(_.constant("Ok")).readonly) - .property("attachment", IdMapping)(_.select(_.attachments._id).readonly) + .property("attachment", UMapping.string)(_.select(_.attachments.value(_.attachmentId)).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index eef29c9051..329706882a 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -40,10 +40,10 @@ class ObservableCtrl @Inject() ( for { case0 <- caseSrv - .get(caseId) + .get(EntityIdOrName(caseId)) .can(Permissions.manageObservable) .orFail(AuthorizationError("Operation not permitted")) - observableType <- observableTypeSrv.getOrFail(inputObservable.dataType) + observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) observablesWithData <- inputObservable .data @@ -65,7 +65,7 @@ class ObservableCtrl @Inject() ( entrypoint("get observable") .authRoTransaction(db) { implicit request => implicit graph => observableSrv - .getByIds(observableId) + .get(EntityIdOrName(observableId)) .visible .richObservable .getOrFail("Observable") @@ -81,17 +81,17 @@ class ObservableCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("observable") observableSrv .update( - _.getByIds(observableId).can(Permissions.manageObservable), + _.get(EntityIdOrName(observableId)).can(Permissions.manageObservable), propertyUpdaters ) .map(_ => Results.NoContent) } - def findSimilar(obsId: String): Action[AnyContent] = + def findSimilar(observableId: String): Action[AnyContent] = entrypoint("find similar") .authRoTransaction(db) { implicit request => implicit graph => val observables = observableSrv - .getByIds(obsId) + .get(EntityIdOrName(observableId)) .visible .similar .visible @@ -111,18 +111,18 @@ class ObservableCtrl @Inject() ( ids .toTry { id => observableSrv - .update(_.getByIds(id).can(Permissions.manageObservable), properties) + .update(_.get(EntityIdOrName(id)).can(Permissions.manageObservable), properties) } .map(_ => Results.NoContent) } - def delete(obsId: String): Action[AnyContent] = + def delete(observableId: String): Action[AnyContent] = entrypoint("delete") .authTransaction(db) { implicit request => implicit graph => for { observable <- observableSrv - .getByIds(obsId) + .get(EntityIdOrName(observableId)) .can(Permissions.manageObservable) .getOrFail("Observable") _ <- observableSrv.remove(observable) @@ -142,10 +142,10 @@ class PublicObservable @Inject() ( "listObservable", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).shares.observables ) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Observable]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Observable]]( "getObservable", - FieldsParser[IdOrName], - (param, graph, authContext) => observableSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => observableSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Observable], IteratorOutput]( @@ -182,7 +182,7 @@ class PublicObservable @Inject() ( _.select(_.tags.displayName) .custom { (_, value, vertex, _, graph, authContext) => observableSrv - .getByIds(vertex.id.toString)(graph) + .get(vertex)(graph) .getOrFail("Observable") .flatMap(observable => observableSrv.updateTagNames(observable, value)(graph, authContext)) .map(_ => Json.obj("tags" -> value)) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableTypeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableTypeCtrl.scala index 1de267b2b6..ce4e22dab1 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableTypeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableTypeCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ @@ -23,7 +24,7 @@ class ObservableTypeCtrl @Inject() ( def get(idOrName: String): Action[AnyContent] = entrypoint("get observable type").authRoTransaction(db) { _ => implicit graph => observableTypeSrv - .get(idOrName) + .get(EntityIdOrName(idOrName)) .getOrFail("Observable") .map(ot => Results.Ok(ot.toJson)) } @@ -41,7 +42,7 @@ class ObservableTypeCtrl @Inject() ( def delete(idOrName: String): Action[AnyContent] = entrypoint("delete observable type") .authPermittedTransaction(db, Permissions.manageObservableTemplate) { _ => implicit graph => - observableTypeSrv.remove(idOrName).map(_ => Results.NoContent) + observableTypeSrv.remove(EntityIdOrName(idOrName)).map(_ => Results.NoContent) } } @@ -57,10 +58,10 @@ class PublicObservableType @Inject() (observableTypeSrv: ObservableTypeSrv) exte (range, observableTypeSteps, _) => observableTypeSteps.richPage(range.from, range.to, withTotal = true)(identity) ) override val outputQuery: Query = Query.output[ObservableType with Entity] - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[ObservableType]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[ObservableType]]( "getObservableType", - FieldsParser[IdOrName], - (param, graph, _) => observableTypeSrv.get(param.idOrName)(graph) + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => observableTypeSrv.get(idOrName)(graph) ) override val publicProperties: PublicProperties = PublicPropertyListBuilder[ObservableType] .property("name", UMapping.string)(_.field.readonly) diff --git a/thehive/app/org/thp/thehive/controllers/v0/OrganisationCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/OrganisationCtrl.scala index 5ed92e9564..fdc5c9445c 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/OrganisationCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/OrganisationCtrl.scala @@ -1,12 +1,12 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.NotFoundError import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.{EntityIdOrName, EntityName, NotFoundError} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputOrganisation import org.thp.thehive.models.{CaseTemplate, Organisation, Permissions, User} @@ -32,7 +32,7 @@ class OrganisationCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val inputOrganisation: InputOrganisation = request.body("organisation") for { - _ <- userSrv.current.organisations(Permissions.manageOrganisation).get(Organisation.administration.name).existsOrFail + _ <- userSrv.current.organisations(Permissions.manageOrganisation).get(EntityName(Organisation.administration.name)).existsOrFail org <- organisationSrv.create(inputOrganisation.toOrganisation) } yield Results.Created(org.toJson) @@ -42,7 +42,7 @@ class OrganisationCtrl @Inject() ( entrypoint("get an organisation") .authRoTransaction(db) { implicit request => implicit graph => organisationSrv - .get(organisationId) + .get(EntityIdOrName(organisationId)) .visible .richOrganisation .getOrFail("Organisation") @@ -68,7 +68,7 @@ class OrganisationCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("organisation") for { - organisation <- organisationSrv.getOrFail(organisationId) + organisation <- organisationSrv.getOrFail(EntityIdOrName(organisationId)) _ <- organisationSrv.update(organisationSrv.get(organisation), propertyUpdaters) } yield Results.NoContent } @@ -77,8 +77,8 @@ class OrganisationCtrl @Inject() ( entrypoint("link organisations") .authPermittedTransaction(db, Permissions.manageOrganisation) { implicit request => implicit graph => for { - fromOrg <- organisationSrv.getOrFail(fromOrganisationId) - toOrg <- organisationSrv.getOrFail(toOrganisationId) + fromOrg <- organisationSrv.getOrFail(EntityIdOrName(fromOrganisationId)) + toOrg <- organisationSrv.getOrFail(EntityIdOrName(toOrganisationId)) _ <- organisationSrv.doubleLink(fromOrg, toOrg) } yield Results.Created } @@ -90,8 +90,8 @@ class OrganisationCtrl @Inject() ( val organisations: Seq[String] = request.body("organisations") for { - fromOrg <- organisationSrv.getOrFail(fromOrganisationId) - _ <- organisationSrv.updateLink(fromOrg, organisations) + fromOrg <- organisationSrv.getOrFail(EntityIdOrName(fromOrganisationId)) + _ <- organisationSrv.updateLink(fromOrg, organisations.map(EntityIdOrName(_))) } yield Results.Created } @@ -99,8 +99,8 @@ class OrganisationCtrl @Inject() ( entrypoint("unlink organisations") .authPermittedTransaction(db, Permissions.manageOrganisation) { _ => implicit graph => for { - fromOrg <- organisationSrv.getOrFail(fromOrganisationId) - toOrg <- organisationSrv.getOrFail(toOrganisationId) + fromOrg <- organisationSrv.getOrFail(EntityIdOrName(fromOrganisationId)) + toOrg <- organisationSrv.getOrFail(EntityIdOrName(toOrganisationId)) _ <- if (organisationSrv.linkExists(fromOrg, toOrg)) Success(organisationSrv.doubleUnlink(fromOrg, toOrg)) else Failure(NotFoundError(s"Organisation $fromOrganisationId is not linked to $toOrganisationId")) @@ -110,15 +110,15 @@ class OrganisationCtrl @Inject() ( def listLinks(organisationId: String): Action[AnyContent] = entrypoint("list organisation links") .authRoTransaction(db) { implicit request => implicit graph => - val isInDefaultOrganisation = userSrv.current.organisations.get(Organisation.administration.name).exists + val isInDefaultOrganisation = userSrv.current.organisations.get(EntityName(Organisation.administration.name)).exists val organisation = if (isInDefaultOrganisation) - organisationSrv.get(organisationId) + organisationSrv.get(EntityIdOrName(organisationId)) else userSrv .current .organisations - .get(organisationId) + .get(EntityIdOrName(organisationId)) val organisations = organisation.links.toSeq Success(Results.Ok(organisations.toJson)) @@ -137,10 +137,10 @@ class PublicOrganisation @Inject() (organisationSrv: OrganisationSrv) extends Pu (range, organisationSteps, _) => organisationSteps.page(range.from, range.to, withTotal = true) ) override val outputQuery: Query = Query.output[Organisation with Entity] - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Organisation]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Organisation]]( "getOrganisation", - FieldsParser[IdOrName], - (param, graph, authContext) => organisationSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => organisationSrv.get(idOrName)(graph).visible(authContext) ) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Organisation], Traversal.V[Organisation]]("visible", (organisationSteps, _) => organisationSteps.visibleOrganisationsFrom), diff --git a/thehive/app/org/thp/thehive/controllers/v0/PageCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/PageCtrl.scala index 766475ddc8..db97600f86 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/PageCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/PageCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ @@ -26,7 +27,7 @@ class PageCtrl @Inject() ( entrypoint("get a page") .authRoTransaction(db) { implicit request => implicit graph => pageSrv - .get(idOrTitle) + .get(EntityIdOrName(idOrTitle)) .visible .getOrFail("Page") .map(p => Results.Ok(p.toJson)) @@ -50,7 +51,7 @@ class PageCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("page") for { - page <- pageSrv.get(idOrTitle).visible.getOrFail("Page") + page <- pageSrv.get(EntityIdOrName(idOrTitle)).visible.getOrFail("Page") updated <- pageSrv.update(page, propertyUpdaters) } yield Results.Ok(updated.toJson) } @@ -59,7 +60,7 @@ class PageCtrl @Inject() ( entrypoint("delete a page") .authPermittedTransaction(db, Permissions.managePage) { implicit request => implicit graph => for { - page <- pageSrv.get(idOrTitle).visible.getOrFail("Page") + page <- pageSrv.get(EntityIdOrName(idOrTitle)).visible.getOrFail("Page") _ <- pageSrv.delete(page) } yield Results.NoContent } @@ -70,10 +71,10 @@ class PublicPage @Inject() (pageSrv: PageSrv, organisationSrv: OrganisationSrv) override val entityName: String = "page" override val initialQuery: Query = Query.init[Traversal.V[Page]]("listPage", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).pages) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Page]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Page]]( "getPage", - FieldsParser[IdOrName], - (param, graph, authContext) => pageSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => pageSrv.get(idOrName)(graph).visible(authContext) ) val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Page], IteratorOutput]( "page", diff --git a/thehive/app/org/thp/thehive/controllers/v0/ProfileCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ProfileCtrl.scala index 3df629ab6a..da4a28d934 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ProfileCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ProfileCtrl.scala @@ -1,12 +1,12 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.AuthorizationError import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.{AuthorizationError, EntityIdOrName} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputProfile import org.thp.thehive.models.{Permissions, Profile} @@ -39,7 +39,7 @@ class ProfileCtrl @Inject() ( entrypoint("get profile") .authRoTransaction(db) { _ => implicit graph => profileSrv - .getOrFail(profileId) + .getOrFail(EntityIdOrName(profileId)) .map { profile => Results.Ok(profile.toJson) } @@ -52,7 +52,7 @@ class ProfileCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("profile") if (request.isPermitted(Permissions.manageProfile)) profileSrv - .update(_.get(profileId), propertyUpdaters) + .update(_.get(EntityIdOrName(profileId)), propertyUpdaters) .flatMap { case (profileSteps, _) => profileSteps.getOrFail("Profile") } .map(profile => Results.Ok(profile.toJson)) else @@ -63,7 +63,7 @@ class ProfileCtrl @Inject() ( entrypoint("delete profile") .authPermittedTransaction(db, Permissions.manageProfile) { implicit request => implicit graph => profileSrv - .getOrFail(profileId) + .getOrFail(EntityIdOrName(profileId)) .flatMap(profileSrv.remove) .map(_ => Results.NoContent) } @@ -73,10 +73,10 @@ class ProfileCtrl @Inject() ( class PublicProfile @Inject() (profileSrv: ProfileSrv) extends PublicData { val entityName: String = "profile" - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Profile]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Profile]]( "getProfile", - FieldsParser[IdOrName], - (param, graph, _) => profileSrv.get(param.idOrName)(graph) + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => profileSrv.get(idOrName)(graph) ) val initialQuery: Query = Query.init[Traversal.V[Profile]]("listProfile", (graph, _) => profileSrv.startTraversal(graph)) diff --git a/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala index 4467579d45..14e6237461 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala @@ -4,6 +4,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order import org.apache.tinkerpop.gremlin.structure.Graph import org.scalactic.Accumulation._ import org.scalactic.Good +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ @@ -16,15 +17,13 @@ import play.api.mvc.{Action, AnyContent, Results} import scala.reflect.runtime.{universe => ru} import scala.util.Try -case class IdOrName(idOrName: String) - trait PublicData { val entityName: String val publicProperties: PublicProperties val initialQuery: Query val pageQuery: ParamQuery[OutputParam] val outputQuery: Query - val getQuery: ParamQuery[IdOrName] + val getQuery: ParamQuery[EntityIdOrName] val extraQueries: Seq[ParamQuery[_]] = Nil } trait QueryCtrl { diff --git a/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala index a3d8f68c98..4d3c6f890c 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ShareCtrl.scala @@ -6,7 +6,7 @@ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.{AuthorizationError, BadRequestError, RichSeq} +import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputShare, ObservablesFilter, TasksFilter} import org.thp.thehive.models.Permissions @@ -38,18 +38,19 @@ class ShareCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val inputShares: Seq[InputShare] = request.body("shares") caseSrv - .get(caseId) + .get(EntityIdOrName(caseId)) .can(Permissions.manageShare) .getOrFail("Case") .flatMap { `case` => inputShares.toTry { inputShare => for { - organisation <- organisationSrv - .get(request.organisation) - .visibleOrganisationsFrom - .get(inputShare.organisationName) - .getOrFail("Organisation") - profile <- profileSrv.getOrFail(inputShare.profile) + organisation <- + organisationSrv + .get(request.organisation) + .visibleOrganisationsFrom + .get(EntityIdOrName(inputShare.organisationName)) + .getOrFail("Organisation") + profile <- profileSrv.getOrFail(EntityIdOrName(inputShare.profile)) share <- shareSrv.shareCase(owner = false, `case`, organisation, profile) richShare <- shareSrv.get(share).richShare.getOrFail("Share") _ <- if (inputShare.tasks == TasksFilter.all) shareSrv.shareCaseTasks(share) else Success(Nil) @@ -63,7 +64,7 @@ class ShareCtrl @Inject() ( def removeShare(shareId: String): Action[AnyContent] = entrypoint("remove share") .authTransaction(db) { implicit request => implicit graph => - doRemoveShare(shareId).map(_ => Results.NoContent) + doRemoveShare(EntityIdOrName(shareId)).map(_ => Results.NoContent) } def removeShares(): Action[AnyContent] = @@ -71,7 +72,7 @@ class ShareCtrl @Inject() ( .extract("shares", FieldsParser[String].sequence.on("ids")) .authTransaction(db) { implicit request => implicit graph => val shareIds: Seq[String] = request.body("shares") - shareIds.toTry(doRemoveShare(_)).map(_ => Results.NoContent) + shareIds.map(EntityIdOrName.apply).toTry(doRemoveShare(_)).map(_ => Results.NoContent) } def removeShares(caseId: String): Action[AnyContent] = @@ -80,17 +81,22 @@ class ShareCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val organisations: Seq[String] = request.body("organisations") organisations + .map(EntityIdOrName(_)) .toTry { organisationId => for { organisation <- organisationSrv.get(organisationId).getOrFail("Organisation") - _ <- if (organisation.name == request.organisation) Failure(BadRequestError("You cannot remove your own share")) else Success(()) - shareId <- caseSrv - .get(caseId) - .can(Permissions.manageShare) - .share(organisationId) - .has("owner", false) - ._id - .orFail(AuthorizationError("Operation not permitted")) + _ <- + if (request.organisation.fold(_ == organisation._id, _ == organisation.name)) + Failure(BadRequestError("You cannot remove your own share")) + else Success(()) + shareId <- + caseSrv + .get(EntityIdOrName(caseId)) + .can(Permissions.manageShare) + .share(organisationId) + .has(_.owner, false) + ._id + .orFail(AuthorizationError("Operation not permitted")) _ <- shareSrv.remove(shareId) } yield () } @@ -104,11 +110,11 @@ class ShareCtrl @Inject() ( val organisations: Seq[String] = request.body("organisations") taskSrv - .getOrFail(taskId) + .getOrFail(EntityIdOrName(taskId)) .flatMap { task => organisations.toTry { organisationName => organisationSrv - .getOrFail(organisationName) + .getOrFail(EntityIdOrName(organisationName)) .flatMap(shareSrv.removeShareTasks(task, _)) } } @@ -122,23 +128,23 @@ class ShareCtrl @Inject() ( val organisations: Seq[String] = request.body("organisations") observableSrv - .getOrFail(observableId) + .getOrFail(EntityIdOrName(observableId)) .flatMap { observable => organisations.toTry { organisationName => organisationSrv - .getOrFail(organisationName) + .getOrFail(EntityIdOrName(organisationName)) .flatMap(shareSrv.removeShareObservable(observable, _)) } } .map(_ => Results.NoContent) } - private def doRemoveShare(shareId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + private def doRemoveShare(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = if (!shareSrv.get(shareId).`case`.can(Permissions.manageShare).exists) Failure(AuthorizationError("You are not authorized to remove share")) - else if (shareSrv.get(shareId).byOrganisationName(authContext.organisation).exists) + else if (shareSrv.get(shareId).byOrganisation(authContext.organisation).exists) Failure(AuthorizationError("You can't remove your share")) - else if (shareSrv.get(shareId).has("owner", true).exists) + else if (shareSrv.get(shareId).has(_.owner, true).exists) Failure(AuthorizationError("You can't remove initial shares")) else shareSrv.remove(shareId) @@ -148,16 +154,16 @@ class ShareCtrl @Inject() ( .extract("profile", FieldsParser.string.on("profile")) .authTransaction(db) { implicit request => implicit graph => val profile: String = request.body("profile") - if (!shareSrv.get(shareId).`case`.can(Permissions.manageShare).exists) + if (!shareSrv.get(EntityIdOrName(shareId)).`case`.can(Permissions.manageShare).exists) Failure(AuthorizationError("You are not authorized to remove share")) for { - richShare <- shareSrv.get(shareId).richShare.getOrFail("Share") - _ <- organisationSrv - .get(request.organisation) - .visibleOrganisationsFrom - .get(richShare.organisationName) - .getOrFail("Share") - profile <- profileSrv.getOrFail(profile) + richShare <- + shareSrv + .get(EntityIdOrName(shareId)) + .filter(_.organisation.visibleOrganisationsTo.visible) + .richShare + .getOrFail("Share") + profile <- profileSrv.getOrFail(EntityIdOrName(profile)) _ <- shareSrv.update(richShare.share, profile) } yield Results.Ok } @@ -166,9 +172,9 @@ class ShareCtrl @Inject() ( entrypoint("list case shares") .authRoTransaction(db) { implicit request => implicit graph => val shares = caseSrv - .get(caseId) + .get(EntityIdOrName(caseId)) .shares - .filter(_.organisation.hasNot("name", request.organisation).visible) + .filter(_.organisation.filterNot(_.get(request.organisation)).visible) .richShare .toSeq @@ -179,11 +185,11 @@ class ShareCtrl @Inject() ( entrypoint("list task shares") .authRoTransaction(db) { implicit request => implicit graph => val shares = caseSrv - .get(caseId) + .get(EntityIdOrName(caseId)) .can(Permissions.manageShare) .shares - .filter(_.organisation.hasNot("name", request.organisation).visible) - .byTask(taskId) + .filter(_.organisation.filterNot(_.get(request.organisation)).visible) + .byTask(EntityIdOrName(taskId)) .richShare .toSeq @@ -194,11 +200,11 @@ class ShareCtrl @Inject() ( entrypoint("list observable shares") .authRoTransaction(db) { implicit request => implicit graph => val shares = caseSrv - .get(caseId) + .get(EntityIdOrName(caseId)) .can(Permissions.manageShare) .shares - .filter(_.organisation.hasNot("name", request.organisation).visible) - .byObservable(observableId) + .filter(_.organisation.filterNot(_.get(request.organisation)).visible) + .byObservable(EntityIdOrName(observableId)) .richShare .toSeq @@ -212,9 +218,9 @@ class ShareCtrl @Inject() ( val organisationIds: Seq[String] = request.body("organisations") for { - task <- taskSrv.getOrFail(taskId) + task <- taskSrv.getOrFail(EntityIdOrName(taskId)) _ <- taskSrv.get(task).`case`.can(Permissions.manageShare).existsOrFail - organisations <- organisationIds.toTry(organisationSrv.get(_).visible.getOrFail("Organisation")) + organisations <- organisationIds.map(EntityIdOrName(_)).toTry(organisationSrv.get(_).visible.getOrFail("Organisation")) _ <- shareSrv.addTaskShares(task, organisations) } yield Results.NoContent } @@ -225,9 +231,9 @@ class ShareCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val organisationIds: Seq[String] = request.body("organisations") for { - observable <- observableSrv.getOrFail(observableId) + observable <- observableSrv.getOrFail(EntityIdOrName(observableId)) _ <- observableSrv.get(observable).`case`.can(Permissions.manageShare).existsOrFail - organisations <- organisationIds.toTry(organisationSrv.get(_).visible.getOrFail("Organisation")) + organisations <- organisationIds.map(EntityIdOrName(_)).toTry(organisationSrv.get(_).visible.getOrFail("Organisation")) _ <- shareSrv.addObservableShares(observable, organisations) } yield Results.NoContent } diff --git a/thehive/app/org/thp/thehive/controllers/v0/StatusCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/StatusCtrl.scala index c15664976e..944368cc08 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/StatusCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/StatusCtrl.scala @@ -1,11 +1,11 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.ScalligraphApplicationLoader import org.thp.scalligraph.auth.{AuthCapability, AuthSrv, MultiAuthSrv} import org.thp.scalligraph.controllers.Entrypoint import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} +import org.thp.scalligraph.{EntityName, ScalligraphApplicationLoader} import org.thp.thehive.TheHiveModule import org.thp.thehive.models.{HealthStatus, User} import org.thp.thehive.services.{Connector, UserSrv} @@ -59,14 +59,15 @@ class StatusCtrl @Inject() ( def health: Action[AnyContent] = entrypoint("health") { _ => val dbStatus = db - .roTransaction(graph => userSrv.getOrFail(User.system.login)(graph)) + .roTransaction(graph => userSrv.getOrFail(EntityName(User.system.login))(graph)) .fold(_ => HealthStatus.Error, _ => HealthStatus.Ok) val connectorStatus = connectors.map(c => c.health) val distinctStatus = connectorStatus + dbStatus - val globalStatus = if (distinctStatus.contains(HealthStatus.Ok)) { - if (distinctStatus.size > 1) HealthStatus.Warning else HealthStatus.Ok - } else if (distinctStatus.contains(HealthStatus.Error)) HealthStatus.Error - else HealthStatus.Warning + val globalStatus = + if (distinctStatus.contains(HealthStatus.Ok)) + if (distinctStatus.size > 1) HealthStatus.Warning else HealthStatus.Ok + else if (distinctStatus.contains(HealthStatus.Error)) HealthStatus.Error + else HealthStatus.Warning Success(Results.Ok(globalStatus.toString)) } diff --git a/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala index fb2edf9b29..dd5f388911 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala @@ -4,12 +4,12 @@ import java.nio.file.Files import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Vertex -import org.thp.scalligraph.RichSeq import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser, Renderer} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} +import org.thp.scalligraph.{EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.models.{Permissions, Tag} import org.thp.thehive.services.TagOps._ @@ -98,7 +98,7 @@ class TagCtrl @Inject() ( entrypoint("get tag") .authRoTransaction(db) { _ => implicit graph => tagSrv - .getOrFail(tagId) + .getOrFail(EntityIdOrName(tagId)) .map { tag => Results.Ok(tag.toJson) } @@ -115,10 +115,10 @@ class PublicTag @Inject() (tagSrv: TagSrv) extends PublicData { (range, tagSteps, _) => tagSteps.page(range.from, range.to, withTotal = true) ) override val outputQuery: Query = Query.output[Tag with Entity] - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Tag]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Tag]]( "getTag", - FieldsParser[IdOrName], - (param, graph, _) => tagSrv.get(param.idOrName)(graph) + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => tagSrv.get(idOrName)(graph) ) implicit val stringRenderer: Renderer[String] = Renderer.toJson[String, String](identity) override val extraQueries: Seq[ParamQuery[_]] = Seq( diff --git a/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala index bdc3ab160c..bbc3924cc8 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala @@ -1,12 +1,12 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.RichOptionTry import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.{EntityIdOrName, RichOptionTry} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputTask import org.thp.thehive.models._ @@ -37,8 +37,8 @@ class TaskCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val inputTask: InputTask = request.body("task") for { - case0 <- caseSrv.get(caseId).can(Permissions.manageTask).getOrFail("Case") - owner <- inputTask.owner.map(userSrv.getOrFail).flip + case0 <- caseSrv.get(EntityIdOrName(caseId)).can(Permissions.manageTask).getOrFail("Case") + owner <- inputTask.owner.map(o => userSrv.getOrFail(EntityIdOrName(o))).flip createdTask <- taskSrv.create(inputTask.toTask, owner) organisation <- organisationSrv.getOrFail(request.organisation) _ <- shareSrv.shareTask(createdTask, case0, organisation) @@ -49,7 +49,7 @@ class TaskCtrl @Inject() ( entrypoint("get task") .authRoTransaction(db) { implicit request => implicit graph => taskSrv - .getByIds(taskId) + .get(EntityIdOrName(taskId)) .visible .richTask .getOrFail("Task") @@ -65,7 +65,7 @@ class TaskCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("task") taskSrv .update( - _.getByIds(taskId) + _.get(EntityIdOrName(taskId)) .can(Permissions.manageTask), propertyUpdaters ) @@ -83,7 +83,10 @@ class TaskCtrl @Inject() ( .extract( "query", searchParser( - Query.init[Traversal.V[Task]]("tasksInCase", (graph, authContext) => caseSrv.get(caseId)(graph).visible(authContext).tasks(authContext)) + Query.init[Traversal.V[Task]]( + "tasksInCase", + (graph, authContext) => caseSrv.get(EntityIdOrName(caseId))(graph).visible(authContext).tasks(authContext) + ) ) ) .auth { implicit request => @@ -109,10 +112,10 @@ class PublicTask @Inject() (taskSrv: TaskSrv, organisationSrv: OrganisationSrv, ) } ) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Task]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Task]]( "getTask", - FieldsParser[IdOrName], - (param, graph, authContext) => taskSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => taskSrv.get(idOrName)(graph).visible(authContext) ) override val outputQuery: Query = Query.output[RichTask, Traversal.V[Task]](_.richTask) override val extraQueries: Seq[ParamQuery[_]] = Seq( @@ -143,7 +146,7 @@ class PublicTask @Inject() (taskSrv: TaskSrv, organisationSrv: OrganisationSrv, .custom { (_, login: Option[String], vertex, _, graph, authContext) => for { task <- taskSrv.get(vertex)(graph).getOrFail("Task") - user <- login.map(userSrv.getOrFail(_)(graph)).flip + user <- login.map(l => userSrv.getOrFail(EntityIdOrName(l))(graph)).flip _ <- user match { case Some(u) => taskSrv.assign(task, u)(graph, authContext) case None => taskSrv.unassign(task)(graph, authContext) diff --git a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala index fad8c6790e..3db5418fe7 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala @@ -2,7 +2,6 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Provider, Singleton} import org.scalactic.Good -import org.thp.scalligraph.{BadRequestError, GlobalQueryExecutor} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{FObject, Field, FieldsParser} import org.thp.scalligraph.models._ @@ -10,6 +9,7 @@ import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.utils.RichType +import org.thp.scalligraph.{BadRequestError, EntityIdOrName, GlobalQueryExecutor} import org.thp.thehive.models.{Case, Log, Observable, Task} import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.LogOps._ @@ -119,10 +119,12 @@ class ParentIdInputFilter(parentId: String) extends InputQuery[Traversal.Unk, Tr .getTypeArgs(traversalType, ru.typeOf[Traversal[_, _, _]]) .headOption .collect { - case t if t <:< ru.typeOf[Task] => traversal.asInstanceOf[Traversal.V[Task]].filter(_.`case`.getByIds(parentId)).asInstanceOf[Traversal.Unk] + case t if t <:< ru.typeOf[Task] => + traversal.asInstanceOf[Traversal.V[Task]].filter(_.`case`.get(EntityIdOrName(parentId))).asInstanceOf[Traversal.Unk] case t if t <:< ru.typeOf[Observable] => - traversal.asInstanceOf[Traversal.V[Observable]].filter(_.`case`.getByIds(parentId)).asInstanceOf[Traversal.Unk] - case t if t <:< ru.typeOf[Log] => traversal.asInstanceOf[Traversal.V[Log]].filter(_.task.getByIds(parentId)).asInstanceOf[Traversal.Unk] + traversal.asInstanceOf[Traversal.V[Observable]].filter(_.`case`.get(EntityIdOrName(parentId))).asInstanceOf[Traversal.Unk] + case t if t <:< ru.typeOf[Log] => + traversal.asInstanceOf[Traversal.V[Log]].filter(_.task.get(EntityIdOrName(parentId))).asInstanceOf[Traversal.Unk] } .getOrElse(throw BadRequestError(s"$traversalType hasn't parent")) } diff --git a/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala index c74d2a19ee..bf4393a499 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala @@ -8,7 +8,7 @@ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.scalligraph.{AuthorizationError, InvalidFormatAttributeError, RichOptionTry} +import org.thp.scalligraph.{AuthorizationError, EntityIdOrName, EntityName, InvalidFormatAttributeError, RichOptionTry} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputUser import org.thp.thehive.models._ @@ -36,16 +36,16 @@ class UserCtrl @Inject() ( entrypoint("current user") .authRoTransaction(db) { implicit request => implicit graph => userSrv - .get(request.userId) - .richUser(request.organisation) + .current + .richUser .getOrFail("User") .orElse( userSrv - .get(request.userId) - .richUser(Organisation.administration.name) + .current + .richUser(request, EntityName(Organisation.administration.name)) .getOrFail("User") ) - .map(user => Results.Ok(user.toJson).withHeaders("X-Organisation" -> request.organisation)) + .map(user => Results.Ok(user.toJson).withHeaders("X-Organisation" -> request.organisation.toString)) } def create: Action[AnyContent] = @@ -54,22 +54,22 @@ class UserCtrl @Inject() ( .auth { implicit request => val inputUser: InputUser = request.body("user") db.tryTransaction { implicit graph => - val organisationName = inputUser.organisation.getOrElse(request.organisation) + val organisationIdOrName = inputUser.organisation.map(EntityIdOrName(_)).getOrElse(request.organisation) for { - _ <- userSrv.current.organisations(Permissions.manageUser).get(organisationName).existsOrFail - organisation <- organisationSrv.getOrFail(organisationName) + _ <- userSrv.current.organisations(Permissions.manageUser).get(organisationIdOrName).existsOrFail + organisation <- organisationSrv.getOrFail(organisationIdOrName) profile <- - if (inputUser.roles.contains("admin")) profileSrv.getOrFail(Profile.admin.name) - else if (inputUser.roles.contains("write")) profileSrv.getOrFail(Profile.analyst.name) - else if (inputUser.roles.contains("read")) profileSrv.getOrFail(Profile.readonly.name) - else profileSrv.getOrFail(Profile.readonly.name) + if (inputUser.roles.contains("admin")) profileSrv.getOrFail(EntityName(Profile.admin.name)) + else if (inputUser.roles.contains("write")) profileSrv.getOrFail(EntityName(Profile.analyst.name)) + else if (inputUser.roles.contains("read")) profileSrv.getOrFail(EntityName(Profile.readonly.name)) + else profileSrv.getOrFail(EntityName(Profile.readonly.name)) user <- userSrv.addOrCreateUser(inputUser.toUser, inputUser.avatar, organisation, profile) } yield user -> userSrv.canSetPassword(user.user) }.flatMap { case (user, true) => inputUser .password - .map(password => authSrv.setPassword(user._id, password)) + .map(password => authSrv.setPassword(user.login, password)) .flip .map(_ => Results.Created(user.toJson)) case (user, _) => Success(Results.Created(user.toJson)) @@ -80,7 +80,7 @@ class UserCtrl @Inject() ( entrypoint("lock user") .authTransaction(db) { implicit request => implicit graph => for { - user <- userSrv.current.organisations(Permissions.manageUser).users.get(userId).getOrFail("User") + user <- userSrv.current.organisations(Permissions.manageUser).users.get(EntityIdOrName(userId)).getOrFail("User") _ <- userSrv.lock(user) } yield Results.NoContent } @@ -89,8 +89,8 @@ class UserCtrl @Inject() ( entrypoint("delete user") .authTransaction(db) { implicit request => implicit graph => for { - organisation <- userSrv.current.organisations(Permissions.manageUser).has("name", request.organisation).getOrFail("Organisation") - user <- organisationSrv.get(organisation).users.get(userId).getOrFail("User") + organisation <- userSrv.current.organisations(Permissions.manageUser).get(request.organisation).getOrFail("Organisation") + user <- organisationSrv.get(organisation).users.get(EntityIdOrName(userId)).getOrFail("User") _ <- userSrv.delete(user, organisation) } yield Results.NoContent } @@ -99,9 +99,9 @@ class UserCtrl @Inject() ( entrypoint("get user") .authRoTransaction(db) { implicit request => implicit graph => userSrv - .get(userId) + .get(EntityIdOrName(userId)) .visible - .richUser(request.organisation) + .richUser .getOrFail("User") .map(user => Results.Ok(user.toJson)) } @@ -114,8 +114,8 @@ class UserCtrl @Inject() ( for { user <- userSrv - .update(userSrv.get(userId), propertyUpdaters) // Authorisation is managed in public properties - .flatMap { case (user, _) => user.richUser(request.organisation).getOrFail("User") } + .update(userSrv.get(EntityIdOrName(userId)), propertyUpdaters) // Authorisation is managed in public properties + .flatMap { case (user, _) => user.richUser.getOrFail("User") } } yield Results.Ok(user.toJson) } @@ -127,14 +127,14 @@ class UserCtrl @Inject() ( for { user <- db.roTransaction { implicit graph => userSrv - .get(userId) + .get(EntityIdOrName(userId)) .getOrFail("User") .flatMap { u => userSrv .current .organisations(Permissions.manageUser) .users - .getByIds(u._id) + .getEntity(u) .getOrFail("User") } } @@ -150,7 +150,7 @@ class UserCtrl @Inject() ( .auth { implicit request => if (userId == request.userId) for { - user <- db.roTransaction(implicit graph => userSrv.get(userId).getOrFail("User")) + user <- db.roTransaction(implicit graph => userSrv.get(EntityIdOrName(userId)).getOrFail("User")) _ <- authSrv.changePassword(userId, request.body("currentPassword"), request.body("password")) _ <- db.tryTransaction(implicit graph => auditSrv.user.update(user, Json.obj("password" -> ""))) } yield Results.NoContent @@ -163,20 +163,20 @@ class UserCtrl @Inject() ( for { user <- db.roTransaction { implicit graph => userSrv - .get(userId) + .get(EntityIdOrName(userId)) .getOrFail("User") .flatMap { u => userSrv .current .organisations(Permissions.manageUser) .users - .getByIds(u._id) + .getEntity(u) .getOrFail("User") } } key <- authSrv - .getKey(user._id) + .getKey(user.login) } yield Results.Ok(key) } @@ -186,14 +186,14 @@ class UserCtrl @Inject() ( for { user <- db.roTransaction { implicit graph => userSrv - .get(userId) + .get(EntityIdOrName(userId)) .getOrFail("User") .flatMap { u => userSrv .current .organisations(Permissions.manageUser) .users - .getByIds(u._id) + .getEntity(u) .getOrFail("User") } } @@ -209,14 +209,14 @@ class UserCtrl @Inject() ( for { user <- db.roTransaction { implicit graph => userSrv - .get(userId) + .get(EntityIdOrName(userId)) .getOrFail("User") .flatMap { u => userSrv .current .organisations(Permissions.manageUser) .users - .getByIds(u._id) + .getEntity(u) .getOrFail("User") } } @@ -231,10 +231,10 @@ class PublicUser @Inject() (userSrv: UserSrv, organisationSrv: OrganisationSrv, override val entityName: String = "user" override val initialQuery: Query = Query.init[Traversal.V[User]]("listUser", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).users) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[User]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[User]]( "getUser", - FieldsParser[IdOrName], - (param, graph, authContext) => userSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => userSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[User], IteratorOutput]( "page", @@ -248,17 +248,14 @@ class PublicUser @Inject() (userSrv: UserSrv, organisationSrv: OrganisationSrv, .property("login", UMapping.string)(_.field.readonly) .property("name", UMapping.string)(_.field.custom { (_, value, vertex, db, graph, authContext) => def isCurrentUser: Try[Unit] = - userSrv - .current(graph, authContext) - .getByIds(vertex.id.toString) - .existsOrFail + userSrv.get(vertex)(graph).current(authContext).existsOrFail def isUserAdmin: Try[Unit] = userSrv .current(graph, authContext) .organisations(Permissions.manageUser)(db) .users - .getByIds(vertex.id.toString) + .getElement(vertex) .existsOrFail isCurrentUser @@ -269,13 +266,13 @@ class PublicUser @Inject() (userSrv: UserSrv, organisationSrv: OrganisationSrv, } }) .property("status", UMapping.string)( - _.select(_.choose(predicate = _.value(_.locked).is(P.eq(true)), onTrue = _.constant("Locked"), onFalse = _.constant("Ok"))) + _.select(_.choose(predicate = _.value(_.locked).is(P.eq(true)), onTrue = "Locked", onFalse = "Ok")) .custom { (_, value, vertex, _, graph, authContext) => userSrv .current(graph, authContext) .organisations(Permissions.manageUser)(db) .users - .getByIds(vertex.id.toString) + .getElement(vertex) .orFail(AuthorizationError("Operation not permitted")) .flatMap { case user if value == "Ok" => diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 9c1d1ef0cb..4ad779213c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -3,6 +3,7 @@ package org.thp.thehive.controllers.v1 import java.util.{Map => JMap} import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -35,10 +36,10 @@ class AlertCtrl @Inject() ( override val publicProperties: PublicProperties = properties.alert override val initialQuery: Query = Query.init[Traversal.V[Alert]]("listAlert", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).alerts) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Alert]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Alert]]( "getAlert", - FieldsParser[IdOrName], - (param, graph, authContext) => alertSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => alertSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Alert], IteratorOutput]( "page", @@ -66,7 +67,7 @@ class AlertCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val caseTemplateName: Option[String] = request.body("caseTemplate") val inputAlert: InputAlert = request.body("alert") - val caseTemplate = caseTemplateName.flatMap(ct => caseTemplateSrv.get(ct).visible.headOption) + val caseTemplate = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption) for { organisation <- userSrv.current.organisations(Permissions.manageAlert).getOrFail("Organisation") customFields = inputAlert.customFieldValue.map(cf => cf.name -> cf.value).toMap @@ -74,25 +75,25 @@ class AlertCtrl @Inject() ( } yield Results.Created(richAlert.toJson) } - def get(alertId: String): Action[AnyContent] = + def get(alertIdOrName: String): Action[AnyContent] = entrypoint("get alert") .authRoTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertIdOrName)) .visible .richAlert .getOrFail("Alert") .map(alert => Results.Ok(alert.toJson)) } - def update(alertId: String): Action[AnyContent] = + def update(alertIdOrName: String): Action[AnyContent] = entrypoint("update alert") .extract("alert", FieldsParser.update("alertUpdate", publicProperties)) .authTransaction(db) { implicit request => implicit graph => val propertyUpdaters: Seq[PropertyUpdater] = request.body("alert") alertSrv .update( - _.get(alertId) + _.get(EntityIdOrName(alertIdOrName)) .can(Permissions.manageAlert), propertyUpdaters ) @@ -101,11 +102,11 @@ class AlertCtrl @Inject() ( // def mergeWithCase(alertId: String, caseId: String) = ??? - def markAsRead(alertId: String): Action[AnyContent] = + def markAsRead(alertIdOrName: String): Action[AnyContent] = entrypoint("mark alert as read") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertIdOrName)) .can(Permissions.manageAlert) .getOrFail("Alert") .map { alert => @@ -114,11 +115,11 @@ class AlertCtrl @Inject() ( } } - def markAsUnread(alertId: String): Action[AnyContent] = + def markAsUnread(alertIdOrName: String): Action[AnyContent] = entrypoint("mark alert as unread") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertIdOrName)) .can(Permissions.manageAlert) .getOrFail("Alert") .map { alert => @@ -127,20 +128,20 @@ class AlertCtrl @Inject() ( } } - def createCase(alertId: String): Action[AnyContent] = + def createCase(alertIdOrName: String): Action[AnyContent] = entrypoint("create case from alert") .authTransaction(db) { implicit request => implicit graph => for { - (alert, organisation) <- alertSrv.get(alertId).alertUserOrganisation(Permissions.manageCase).getOrFail("Alert") + (alert, organisation) <- alertSrv.get(EntityIdOrName(alertIdOrName)).alertUserOrganisation(Permissions.manageCase).getOrFail("Alert") richCase <- alertSrv.createCase(alert, None, organisation) } yield Results.Created(richCase.toJson) } - def followAlert(alertId: String): Action[AnyContent] = + def followAlert(alertIdOrName: String): Action[AnyContent] = entrypoint("follow alert") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertIdOrName)) .can(Permissions.manageAlert) .getOrFail("Alert") .map { alert => @@ -149,11 +150,11 @@ class AlertCtrl @Inject() ( } } - def unfollowAlert(alertId: String): Action[AnyContent] = + def unfollowAlert(alertIdOrName: String): Action[AnyContent] = entrypoint("unfollow alert") .authTransaction(db) { implicit request => implicit graph => alertSrv - .get(alertId) + .get(EntityIdOrName(alertIdOrName)) .can(Permissions.manageAlert) .getOrFail("Alert") .map { alert => diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala index d08472cd0b..83a3563834 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala @@ -21,9 +21,9 @@ trait AlertRenderer { "iocCount" -> similarStats.ioc._2 ) } - def similarCasesStats( - implicit authContext: AuthContext - ): Traversal.V[Alert] => Traversal[JsArray, JList[JMap[String, Any]], Converter[JsArray, JList[JMap[String, Any]]]] = { + def similarCasesStats(implicit + authContext: AuthContext + ): Traversal.V[Alert] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = { implicit val similarCaseOrdering: Ordering[(RichCase, SimilarStats)] = (x: (RichCase, SimilarStats), y: (RichCase, SimilarStats)) => //negative if x < y if (x._1._createdAt after y._1._createdAt) -1 @@ -38,8 +38,8 @@ trait AlertRenderer { _.similarCases.fold.domainMap(sc => JsArray(sc.sorted.map(Json.toJson(_)))) } - def alertStatsRenderer[D, G, C <: Converter[D, G]](extraData: Set[String])( - implicit authContext: AuthContext + def alertStatsRenderer[D, G, C <: Converter[D, G]](extraData: Set[String])(implicit + authContext: AuthContext ): Traversal.V[Alert] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { traversal => def addData[T]( name: String diff --git a/thehive/app/org/thp/thehive/controllers/v1/AuditCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AuditCtrl.scala index f98a424ae5..e3c315eb91 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AuditCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AuditCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Schema} import org.thp.scalligraph.query.{ParamQuery, PublicProperties, Query} @@ -28,10 +29,10 @@ class AuditCtrl @Inject() ( val initialQuery: Query = Query.init[Traversal.V[Audit]]("listAudit", (graph, authContext) => auditSrv.startTraversal(graph).visible(authContext)) val publicProperties: PublicProperties = properties.audit - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Audit]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Audit]]( "getAudit", - FieldsParser[IdOrName], - (param, graph, authContext) => auditSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => auditSrv.get(idOrName)(graph).visible(authContext) ) val pageQuery: ParamQuery[OutputParam] = diff --git a/thehive/app/org/thp/thehive/controllers/v1/AuthenticationCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AuthenticationCtrl.scala index a0ebd6b193..8b114353f3 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AuthenticationCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AuthenticationCtrl.scala @@ -5,7 +5,7 @@ import org.thp.scalligraph.auth.{AuthSrv, RequestOrganisation} import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.{AuthenticationError, AuthorizationError, BadRequestError, MultiFactorCodeRequired} +import org.thp.scalligraph.{AuthenticationError, AuthorizationError, BadRequestError, EntityIdOrName, MultiFactorCodeRequired} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models.Permissions import org.thp.thehive.services.OrganisationOps._ @@ -31,25 +31,26 @@ class AuthenticationCtrl @Inject() ( .extract("password", FieldsParser[String].on("password")) .extract("organisation", FieldsParser[String].optional.on("organisation")) .extract("code", FieldsParser[String].optional.on("code")) { implicit request => - val login: String = request.body("login") - val password: String = request.body("password") - val organisation: Option[String] = request.body("organisation") orElse requestOrganisation(request) - val code: Option[String] = request.body("code") + val login: String = request.body("login") + val password: String = request.body("password") + val organisation: Option[EntityIdOrName] = request.body("organisation").map(EntityIdOrName(_)) orElse requestOrganisation(request) + val code: Option[String] = request.body("code") for { authContext <- authSrv.authenticate(login, password, organisation, code) user <- db.roTransaction { implicit graph => userSrv - .get(authContext.userId) - .richUserWithCustomRenderer(authContext.organisation, _.organisationWithRole) + .current(graph, authContext) + .richUserWithCustomRenderer(authContext.organisation, _.organisationWithRole)(authContext) .getOrFail("User") } _ <- if (user._1.locked) Failure(AuthorizationError("Your account is locked")) else Success(()) } yield authSrv.setSessionUser(authContext)(Results.Ok(user.toJson)) } - def logout: Action[AnyContent] = entrypoint("logout") { _ => - Success(Results.Ok.withNewSession) - } + def logout: Action[AnyContent] = + entrypoint("logout") { _ => + Success(Results.Ok.withNewSession) + } def withTotpAuthSrv[A](body: TOTPAuthSrv => Try[A]): Try[A] = authSrv match { @@ -90,9 +91,9 @@ class AuthenticationCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => withTotpAuthSrv { totpAuthSrv => userSrv - .getOrFail(userId.getOrElse(request.userId)) + .getOrFail(EntityIdOrName(userId.getOrElse(request.userId))) .flatMap { user => - if (request.userId == user.login || userSrv.current.organisations(Permissions.manageUser).users.get(user._id).exists) + if (request.userId == user.login || userSrv.current.organisations(Permissions.manageUser).users.getEntity(user).exists) totpAuthSrv.unsetSecret(user.login) else Failure(AuthorizationError("You cannot unset TOTP secret of this user")) } diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index 21ef75e23f..aeb3be90a7 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -6,7 +6,7 @@ import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.scalligraph.{RichOptionTry, RichSeq} +import org.thp.scalligraph.{EntityIdOrName, RichOptionTry, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.{InputCase, InputTask} import org.thp.thehive.models._ @@ -37,10 +37,10 @@ class CaseCtrl @Inject() ( override val publicProperties: PublicProperties = properties.`case` override val initialQuery: Query = Query.init[Traversal.V[Case]]("listCase", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).cases) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Case]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Case]]( "getCase", - FieldsParser[IdOrName], - (param, graph, authContext) => caseSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => caseSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Case], IteratorOutput]( "page", @@ -71,10 +71,10 @@ class CaseCtrl @Inject() ( val inputCase: InputCase = request.body("case") val inputTasks: Seq[InputTask] = request.body("tasks") for { - caseTemplate <- caseTemplateName.map(caseTemplateSrv.get(_).visible.richCaseTemplate.getOrFail("CaseTemplate")).flip + caseTemplate <- caseTemplateName.map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.richCaseTemplate.getOrFail("CaseTemplate")).flip customFields = inputCase.customFieldValue.map(cf => (cf.name, cf.value, cf.order)) organisation <- userSrv.current.organisations(Permissions.manageCase).get(request.organisation).getOrFail("Organisation") - user <- inputCase.user.fold[Try[Option[User with Entity]]](Success(None))(u => userSrv.getOrFail(u).map(Some.apply)) + user <- inputCase.user.fold[Try[Option[User with Entity]]](Success(None))(u => userSrv.getOrFail(EntityIdOrName(u)).map(Some.apply)) tags <- inputCase.tags.toTry(tagSrv.getOrCreate) richCase <- caseSrv.create( caseTemplate.fold(inputCase)(inputCase.withCaseTemplate).toCase, @@ -83,7 +83,7 @@ class CaseCtrl @Inject() ( tags.toSet, customFields, caseTemplate, - inputTasks.map(t => t.toTask -> t.assignee.flatMap(userSrv.get(_).headOption)) + inputTasks.map(t => t.toTask -> t.assignee.flatMap(u => userSrv.get(EntityIdOrName(u)).headOption)) ) } yield Results.Created(richCase.toJson) } @@ -92,7 +92,7 @@ class CaseCtrl @Inject() ( entrypoint("get case") .authRoTransaction(db) { implicit request => implicit graph => caseSrv - .get(caseIdOrNumber) + .get(EntityIdOrName(caseIdOrNumber)) .visible .richCase .getOrFail("Case") @@ -105,7 +105,7 @@ class CaseCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val propertyUpdaters: Seq[PropertyUpdater] = request.body("case") caseSrv - .update(_.get(caseIdOrNumber).can(Permissions.manageCase), propertyUpdaters) + .update(_.get(EntityIdOrName(caseIdOrNumber)).can(Permissions.manageCase), propertyUpdaters) .map(_ => Results.NoContent) } @@ -113,7 +113,7 @@ class CaseCtrl @Inject() ( entrypoint("delete case") .authTransaction(db) { implicit request => implicit graph => caseSrv - .get(caseIdOrNumber) + .get(EntityIdOrName(caseIdOrNumber)) .can(Permissions.manageCase) .update(_.status, CaseStatus.Deleted) .getOrFail("Case") @@ -126,9 +126,9 @@ class CaseCtrl @Inject() ( caseIdsOrNumbers .split(',') .toSeq - .toTry( + .toTry(c => caseSrv - .get(_) + .get(EntityIdOrName(c)) .visible .getOrFail("Case") ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala index 5b03ccf93d..87e052d53b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala @@ -9,13 +9,14 @@ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.models.{Alert, AlertCase, Case} import org.thp.thehive.services.CaseOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ import play.api.libs.json._ trait CaseRenderer { - def observableStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsObject, JLong, Converter[JsObject, JLong]] = + def observableStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = _.share .observables .count @@ -23,7 +24,7 @@ trait CaseRenderer { def taskStats(implicit authContext: AuthContext - ): Traversal.V[Case] => Traversal[JsObject, JMap[String, JLong], Converter[JsObject, JMap[String, JLong]]] = + ): Traversal.V[Case] => Traversal[JsValue, JMap[String, JLong], Converter[JsValue, JMap[String, JLong]]] = _.share .tasks .active @@ -35,7 +36,7 @@ trait CaseRenderer { result + ("total" -> JsNumber(total)) } - def alertStats: Traversal.V[Case] => Traversal[JsArray, JMap[String, JCollection[String]], Converter[JsArray, JMap[String, JCollection[String]]]] = + def alertStats: Traversal.V[Case] => Traversal[JsValue, JMap[String, JCollection[String]], Converter[JsValue, JMap[String, JCollection[String]]]] = _.in[AlertCase] .v[Alert] .group(_.byValue(_.`type`), _.byValue(_.source)) @@ -46,10 +47,10 @@ trait CaseRenderer { } yield Json.obj("type" -> tpe, "source" -> source)).toSeq) } - def isOwnerStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsBoolean, JList[Vertex], Converter[JsBoolean, JList[Vertex]]] = - _.origin.has("name", authContext.organisation).fold.domainMap(l => JsBoolean(l.nonEmpty)) + def isOwnerStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = + _.origin.get(authContext.organisation).fold.domainMap(l => JsBoolean(l.nonEmpty)) - def shareCountStats: Traversal.V[Case] => Traversal[JsNumber, JLong, Converter[JsNumber, JLong]] = + def shareCountStats: Traversal.V[Case] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = _.organisations.count.domainMap(c => JsNumber(c - 1)) def permissions(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsValue, Vertex, Converter[JsValue, Vertex]] = diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseTemplateCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseTemplateCtrl.scala index 233fda6c3d..34b55c6403 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseTemplateCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseTemplateCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -30,10 +31,10 @@ class CaseTemplateCtrl @Inject() ( override val initialQuery: Query = Query .init[Traversal.V[CaseTemplate]]("listCaseTemplate", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).caseTemplates) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[CaseTemplate]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[CaseTemplate]]( "getCaseTemplate", - FieldsParser[IdOrName], - (param, graph, authContext) => caseTemplateSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => caseTemplateSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[CaseTemplate], IteratorOutput]( "page", @@ -49,7 +50,7 @@ class CaseTemplateCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val inputCaseTemplate: InputCaseTemplate = request.body("caseTemplate") for { - organisation <- organisationSrv.getOrFail(request.organisation) + organisation <- organisationSrv.current.getOrFail("Organisation") tasks = inputCaseTemplate.tasks.map(_.toTask -> None) customFields = inputCaseTemplate.customFieldValue.map(cf => cf.name -> cf.value) richCaseTemplate <- caseTemplateSrv.create(inputCaseTemplate.toCaseTemplate, organisation, inputCaseTemplate.tags, tasks, customFields) @@ -60,7 +61,7 @@ class CaseTemplateCtrl @Inject() ( entrypoint("get case template") .authRoTransaction(db) { implicit request => implicit graph => caseTemplateSrv - .get(caseTemplateNameOrId) + .get(EntityIdOrName(caseTemplateNameOrId)) .visible .richCaseTemplate .getOrFail("CaseTemplate") @@ -85,7 +86,7 @@ class CaseTemplateCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("caseTemplate") caseTemplateSrv .update( - _.get(caseTemplateNameOrId) + _.get(EntityIdOrName(caseTemplateNameOrId)) .can(Permissions.manageCaseTemplate), propertyUpdaters ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 1b3c2afa81..f7f2db0dbc 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -18,7 +18,9 @@ object Conversion { implicit val alertOutput: Renderer.Aux[RichAlert, OutputAlert] = Renderer.toJson[RichAlert, OutputAlert]( _.into[OutputAlert] .withFieldConst(_._type, "Alert") - .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).toSet) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.caseId, _.caseId.map(_.toString)) + .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).sortBy(_.order)) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) .withFieldConst(_.extraData, JsObject.empty) .transform @@ -30,7 +32,9 @@ object Conversion { ._1 .into[OutputAlert] .withFieldConst(_._type, "Alert") - .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).toSet) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.caseId, _.caseId.map(_.toString)) + .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).sortBy(_.order)) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) .withFieldConst(_.extraData, alertWithExtraData._2) .transform @@ -39,7 +43,7 @@ object Conversion { implicit val auditOutput: Renderer.Aux[RichAudit, OutputAudit] = Renderer.toJson[RichAudit, OutputAudit]( _.into[OutputAudit] .withFieldComputed(_.operation, _.action) - .withFieldComputed(_._id, _._id) + .withFieldComputed(_._id, _._id.toString) .withFieldConst(_._type, "Audit") .withFieldComputed(_._createdAt, _._createdAt) .withFieldComputed(_._createdBy, _._createdBy) @@ -77,7 +81,8 @@ object Conversion { implicit val caseOutput: Renderer.Aux[RichCase, OutputCase] = Renderer.toJson[RichCase, OutputCase]( _.into[OutputCase] .withFieldConst(_._type, "Case") - .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).toSet) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).sortBy(_.order)) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) .withFieldComputed(_.status, _.status.toString) .withFieldConst(_.extraData, JsObject.empty) @@ -90,7 +95,8 @@ object Conversion { ._1 .into[OutputCase] .withFieldConst(_._type, "Case") - .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).toSet) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).sortBy(_.order)) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) .withFieldComputed(_.status, _.status.toString) .withFieldConst(_.extraData, caseWithExtraData._2) @@ -142,7 +148,8 @@ object Conversion { implicit val caseTemplateOutput: Renderer.Aux[RichCaseTemplate, OutputCaseTemplate] = Renderer.toJson[RichCaseTemplate, OutputCaseTemplate]( _.into[OutputCaseTemplate] .withFieldConst(_._type, "CaseTemplate") - .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).toSet) + .withFieldComputed(_._id, _._id.toString) + .withFieldComputed(_.customFields, _.customFields.map(_.toOutput).sortBy(_.order)) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) .withFieldComputed(_.tasks, _.tasks.map(_.toOutput)) .transform @@ -151,6 +158,7 @@ object Conversion { implicit val richCustomFieldOutput: Renderer.Aux[RichCustomField, OutputCustomFieldValue] = Renderer.toJson[RichCustomField, OutputCustomFieldValue]( _.into[OutputCustomFieldValue] + .withFieldComputed(_._id, _.customFieldValue._id.toString) .withFieldComputed(_.value, _.jsValue) .withFieldComputed(_.`type`, _.typeName) .withFieldComputed(_.order, _.order.getOrElse(0)) @@ -178,6 +186,7 @@ object Conversion { Renderer.toJson[RichOrganisation, OutputOrganisation](organisation => organisation .into[OutputOrganisation] + .withFieldComputed(_._id, _._id.toString) .withFieldConst(_._type, "Organisation") .withFieldConst(_.name, organisation.name) .withFieldConst(_.description, organisation.description) @@ -188,7 +197,7 @@ object Conversion { implicit val organiastionRenderer: Renderer.Aux[Organisation with Entity, OutputOrganisation] = Renderer.toJson[Organisation with Entity, OutputOrganisation](organisation => OutputOrganisation( - organisation._id, + organisation._id.toString, "organisation", organisation._createdBy, organisation._updatedBy, @@ -215,6 +224,7 @@ object Conversion { implicit val taskOutput: Renderer.Aux[RichTask, OutputTask] = Renderer.toJson[RichTask, OutputTask]( _.into[OutputTask] .withFieldConst(_._type, "Task") + .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.status, _.status.toString) .withFieldComputed(_.assignee, _.assignee.map(_.login)) .withFieldConst(_.extraData, JsObject.empty) @@ -227,6 +237,7 @@ object Conversion { ._1 .into[OutputTask] .withFieldConst(_._type, "Task") + .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.status, _.status.toString) .withFieldComputed(_.assignee, _.assignee.map(_.login)) .withFieldConst(_.extraData, taskWithExtraData._2) @@ -251,6 +262,7 @@ object Conversion { _.into[OutputUser] .withFieldComputed(_.permissions, _.permissions.asInstanceOf[Set[String]]) .withFieldComputed(_.hasKey, _.apikey.isDefined) + .withFieldComputed(_._id, _._id.toString) .withFieldConst(_.organisations, Nil) .withFieldComputed(_.avatar, user => user.avatar.map(avatar => s"/api/v1/user/${user._id}/avatar/$avatar")) .transform @@ -261,6 +273,7 @@ object Conversion { val (user, organisations) = userWithOrganisations user .into[OutputUser] + .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.permissions, _.permissions.asInstanceOf[Set[String]]) .withFieldComputed(_.hasKey, _.apikey.isDefined) .withFieldConst(_.organisations, organisations.map { case (org, role) => OutputOrganisationProfile(org, role) }) @@ -272,7 +285,7 @@ object Conversion { profile .asInstanceOf[Profile] .into[OutputProfile] - .withFieldConst(_._id, profile._id) + .withFieldConst(_._id, profile._id.toString) .withFieldConst(_._updatedAt, profile._updatedAt) .withFieldConst(_._updatedBy, profile._updatedBy) .withFieldConst(_._createdAt, profile._createdAt) @@ -287,7 +300,7 @@ object Conversion { implicit val dashboardOutput: Renderer.Aux[RichDashboard, OutputDashboard] = Renderer.toJson[RichDashboard, OutputDashboard](dashboard => dashboard .into[OutputDashboard] - .withFieldConst(_._id, dashboard._id) + .withFieldConst(_._id, dashboard._id.toString) .withFieldComputed(_.status, d => if (d.organisationShares.nonEmpty) "Shared" else "Private") .withFieldConst(_._type, "Dashboard") .withFieldConst(_._updatedAt, dashboard._updatedAt) @@ -319,7 +332,7 @@ object Conversion { richObservable .into[OutputObservable] .withFieldConst(_._type, "Observable") - .withFieldComputed(_._id, _.observable._id) + .withFieldComputed(_._id, _.observable._id.toString) .withFieldComputed(_._updatedAt, _.observable._updatedAt) .withFieldComputed(_._updatedBy, _.observable._updatedBy) .withFieldComputed(_._createdAt, _.observable._createdAt) @@ -350,6 +363,7 @@ object Conversion { richObservable .into[OutputObservable] .withFieldConst(_._type, "case_artifact") + .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.dataType, _.`type`.name) .withFieldComputed(_.startDate, _.observable._createdAt) .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) @@ -374,6 +388,7 @@ object Conversion { richLog .into[OutputLog] .withFieldConst(_._type, "Log") + .withFieldComputed(_._id, _._id.toString) .withFieldRenamed(_._createdAt, _.date) .withFieldComputed(_.attachment, _.attachments.headOption.map(_.toOutput)) .withFieldRenamed(_._createdBy, _.owner) @@ -387,6 +402,7 @@ object Conversion { ._1 .into[OutputLog] .withFieldConst(_._type, "Log") + .withFieldComputed(_._id, _._id.toString) .withFieldRenamed(_._createdAt, _.date) .withFieldComputed(_.attachment, _.attachments.headOption.map(_.toOutput)) .withFieldRenamed(_._createdBy, _.owner) diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala index ec3b4454fe..0a4a08dfe3 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -32,15 +33,15 @@ class LogCtrl @Inject() ( override val publicProperties: PublicProperties = properties.log override val initialQuery: Query = Query.init[Traversal.V[Log]]("listLog", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).shares.tasks.logs) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Log]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Log]]( "getLog", - FieldsParser[IdOrName], - (param, graph, authContext) => logSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => logSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Log], IteratorOutput]( "page", FieldsParser[OutputParam], - (range, logSteps, authContext) => + (range, logSteps, _) => logSteps.richPage(range.from, range.to, range.extraData.contains("total"))( _.richLogWithCustomRenderer(logStatsRenderer(range.extraData - "total")) ) @@ -55,7 +56,7 @@ class LogCtrl @Inject() ( for { task <- taskSrv - .getByIds(taskId) + .get(EntityIdOrName(taskId)) .can(Permissions.manageTask) .getOrFail("Task") createdLog <- logSrv.create(inputLog.toLog, task) @@ -71,7 +72,7 @@ class LogCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("log") logSrv .update( - _.getByIds(logId) + _.get(EntityIdOrName(logId)) .can(Permissions.manageTask), propertyUpdaters ) @@ -82,7 +83,7 @@ class LogCtrl @Inject() ( entrypoint("delete log") .authTransaction(db) { implicit req => implicit graph => for { - log <- logSrv.get(logId).can(Permissions.manageTask).getOrFail("Log") + log <- logSrv.get(EntityIdOrName(logId)).can(Permissions.manageTask).getOrFail("Log") _ <- logSrv.cascadeRemove(log) } yield Results.NoContent } diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala index 8fdd2b5cfe..a06de4a596 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala @@ -18,9 +18,9 @@ trait LogRenderer { _.task.richTask.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) def taskParentId: Traversal.V[Log] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = - _.task.fold.domainMap(_.headOption.fold[JsValue](JsNull)(c => JsString(c._id))) + _.task.fold.domainMap(_.headOption.fold[JsValue](JsNull)(c => JsString(c._id.toString))) - def actionCount: Traversal.V[Log] => Traversal[JsNumber, JLong, Converter[JsNumber, JLong]] = + def actionCount: Traversal.V[Log] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = _.in("ActionContext").count.domainMap(JsNumber(_)) def logStatsRenderer(extraData: Set[String]): Traversal.V[Log] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index 0c32f5b2ba..c4d7fe18d6 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -38,10 +38,10 @@ class ObservableCtrl @Inject() ( "listObservable", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).shares.observables ) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Observable]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Observable]]( "getObservable", - FieldsParser[IdOrName], - (param, graph, authContext) => observableSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => observableSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Observable], IteratorOutput]( "page", @@ -75,10 +75,10 @@ class ObservableCtrl @Inject() ( for { case0 <- caseSrv - .get(caseId) + .get(EntityIdOrName(caseId)) .can(Permissions.manageObservable) .getOrFail("Case") - observableType <- observableTypeSrv.getOrFail(inputObservable.dataType) + observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) observablesWithData <- inputObservable .data @@ -100,7 +100,7 @@ class ObservableCtrl @Inject() ( entryPoint("get observable") .authRoTransaction(db) { _ => implicit graph => observableSrv - .getByIds(observableId) + .get(EntityIdOrName(observableId)) // .availableFor(request.organisation) .richObservable .getOrFail("Observable") @@ -116,7 +116,7 @@ class ObservableCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("observable") observableSrv .update( - _.getByIds(observableId).can(Permissions.manageObservable), + _.get(EntityIdOrName(observableId)).can(Permissions.manageObservable), propertyUpdaters ) .map(_ => Results.NoContent) @@ -132,7 +132,7 @@ class ObservableCtrl @Inject() ( ids .toTry { id => observableSrv - .update(_.getByIds(id).can(Permissions.manageObservable), properties) + .update(_.get(EntityIdOrName(id)).can(Permissions.manageObservable), properties) } .map(_ => Results.NoContent) } @@ -143,7 +143,7 @@ class ObservableCtrl @Inject() ( for { observable <- observableSrv - .getByIds(obsId) + .get(EntityIdOrName(obsId)) .can(Permissions.manageObservable) .getOrFail("Observable") _ <- observableSrv.remove(observable) diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala index 9f69cf6185..54b9e3cfd0 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala @@ -12,13 +12,14 @@ import org.thp.thehive.models.Observable import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ +import org.thp.thehive.services.OrganisationOps._ import play.api.libs.json._ trait ObservableRenderer { def seenStats(implicit authContext: AuthContext - ): Traversal.V[Observable] => Traversal[JsObject, JMap[JBoolean, JLong], Converter[JsObject, JMap[JBoolean, JLong]]] = + ): Traversal.V[Observable] => Traversal[JsValue, JMap[JBoolean, JLong], Converter[JsValue, JMap[JBoolean, JLong]]] = _.similar .visible .groupCount(_.byValue(_.ioc)) @@ -39,10 +40,10 @@ trait ObservableRenderer { def isOwner(implicit authContext: AuthContext - ): Traversal.V[Observable] => Traversal[JsBoolean, JList[Vertex], Converter[JsBoolean, JList[Vertex]]] = - _.origin.has("name", authContext.organisation).fold.domainMap(l => JsBoolean(l.nonEmpty)) + ): Traversal.V[Observable] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = + _.origin.get(authContext.organisation).fold.domainMap(l => JsBoolean(l.nonEmpty)) - def observableLinks: Traversal.V[Observable] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = + def observableLinks: Traversal.V[Observable] => Traversal[JsValue, JMap[String, Any], Converter[JsValue, JMap[String, Any]]] = _.coalesceMulti( _.alert.richAlert.domainMap(a => Json.obj("alert" -> a.toJson)), _.`case`.richCaseWithoutPerms.domainMap(c => Json.obj("case" -> c.toJson)) diff --git a/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala index 39050c3590..336ec1458c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -33,10 +34,10 @@ class OrganisationCtrl @Inject() ( (range, organisationSteps, _) => organisationSteps.richPage(range.from, range.to, range.extraData.contains("total"))(_.richOrganisation) ) override val outputQuery: Query = Query.output[RichOrganisation, Traversal.V[Organisation]](_.richOrganisation) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Organisation]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Organisation]]( "getOrganisation", - FieldsParser[IdOrName], - (param, graph, authContext) => organisationSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => organisationSrv.get(idOrName)(graph).visible(authContext) ) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Organisation], Traversal.V[Organisation]]("visible", (organisationSteps, _) => organisationSteps.visibleOrganisationsFrom), @@ -59,14 +60,14 @@ class OrganisationCtrl @Inject() ( def get(organisationId: String): Action[AnyContent] = entrypoint("get organisation") .authRoTransaction(db) { implicit request => implicit graph => - (if (request.organisation == "admin") - organisationSrv.get(organisationId) + (if (organisationSrv.current.isAdmin) + organisationSrv.get(EntityIdOrName(organisationId)) else userSrv .current .organisations .visibleOrganisationsFrom - .get(organisationId)) + .get(EntityIdOrName(organisationId))) .richOrganisation .getOrFail("Organisation") .map(organisation => Results.Ok(organisation.toJson)) @@ -78,7 +79,7 @@ class OrganisationCtrl @Inject() ( .authPermittedTransaction(db, Permissions.manageOrganisation) { implicit request => implicit graph => val propertyUpdaters: Seq[PropertyUpdater] = request.body("organisation") for { - organisation <- organisationSrv.getOrFail(organisationId) + organisation <- organisationSrv.getOrFail(EntityIdOrName(organisationId)) _ <- organisationSrv.update(organisationSrv.get(organisation), propertyUpdaters) } yield Results.NoContent } diff --git a/thehive/app/org/thp/thehive/controllers/v1/ProfileCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ProfileCtrl.scala index 2be85dd50b..f9d4e5dcb2 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ProfileCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ProfileCtrl.scala @@ -1,12 +1,12 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.AuthorizationError import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.{AuthorizationError, EntityIdOrName} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputProfile import org.thp.thehive.models.{Permissions, Profile} @@ -24,10 +24,10 @@ class ProfileCtrl @Inject() ( @Named("with-thehive-schema") implicit val db: Database ) extends QueryableCtrl { - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Profile]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Profile]]( "getProfile", - FieldsParser[IdOrName], - (param, graph, _) => profileSrv.get(param.idOrName)(graph) + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => profileSrv.get(idOrName)(graph) ) val entityName: String = "profile" val publicProperties: PublicProperties = properties.profile @@ -57,7 +57,7 @@ class ProfileCtrl @Inject() ( entrypoint("get profile") .authRoTransaction(db) { _ => implicit graph => profileSrv - .getOrFail(profileId) + .getOrFail(EntityIdOrName(profileId)) .map { profile => Results.Ok(profile.toJson) } @@ -70,7 +70,7 @@ class ProfileCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("profile") if (request.isPermitted(Permissions.manageProfile)) profileSrv - .update(_.get(profileId), propertyUpdaters) + .update(_.get(EntityIdOrName(profileId)), propertyUpdaters) .flatMap { case (profileSteps, _) => profileSteps.getOrFail("Profile") } .map(profile => Results.Ok(profile.toJson)) else @@ -81,7 +81,7 @@ class ProfileCtrl @Inject() ( entrypoint("delete profile") .authPermittedTransaction(db, Permissions.manageProfile) { implicit request => implicit graph => profileSrv - .getOrFail(profileId) + .getOrFail(EntityIdOrName(profileId)) .flatMap(profileSrv.remove) .map(_ => Results.NoContent) } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index fb7652d839..8358cc96d7 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -1,23 +1,27 @@ package org.thp.thehive.controllers.v1 -import javax.inject.{Inject, Singleton} -import org.thp.scalligraph.BadRequestError -import org.thp.scalligraph.controllers.FPathElem -import org.thp.scalligraph.models.{IdMapping, UMapping} +import java.util.Date + +import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.controllers.{FPathElem, FPathEmpty} +import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query.{PublicProperties, PublicPropertyListBuilder} +import org.thp.scalligraph.traversal.Converter import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{BadRequestError, EntityIdOrName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.AuditOps._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CaseTemplateOps._ +import org.thp.thehive.services.CustomFieldOps._ import org.thp.thehive.services.LogOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.TagOps._ import org.thp.thehive.services.TaskOps._ import org.thp.thehive.services.UserOps._ import org.thp.thehive.services._ -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.{JsObject, JsValue, Json} import scala.util.Failure @@ -28,7 +32,9 @@ class Properties @Inject() ( taskSrv: TaskSrv, userSrv: UserSrv, caseTemplateSrv: CaseTemplateSrv, - observableSrv: ObservableSrv + observableSrv: ObservableSrv, + customFieldSrv: CustomFieldSrv, + @Named("with-thehive-schema") db: Database ) { lazy val alert: PublicProperties = @@ -82,7 +88,7 @@ class Properties @Inject() ( .property("base", UMapping.boolean)(_.rename("mainAction").readonly) .property("startDate", UMapping.date)(_.rename("_createdAt").readonly) .property("requestId", UMapping.string)(_.field.readonly) - .property("rootId", IdMapping)(_.select(_.context._id).readonly) + .property("rootId", db.idMapping)(_.select(_.context._id).readonly) .build lazy val `case`: PublicProperties = @@ -110,7 +116,7 @@ class Properties @Inject() ( .property("assignee", UMapping.string.optional)(_.select(_.user.value(_.login)).custom { (_, login, vertex, _, graph, authContext) => for { c <- caseSrv.get(vertex)(graph).getOrFail("Case") - user <- login.map(userSrv.get(_)(graph).getOrFail("User")).flip + user <- login.map(u => userSrv.get(EntityIdOrName(u))(graph).getOrFail("User")).flip _ <- user match { case Some(u) => caseSrv.assign(c, u)(graph, authContext) case None => caseSrv.unassign(c)(graph, authContext) @@ -137,6 +143,57 @@ class Properties @Inject() ( } .map(_ => Json.obj("resolutionStatus" -> value)) }) + .property("customFields", UMapping.jsonNative)(_.subSelect { + case (FPathElem(_, FPathElem(idOrName, _)), caseSteps) => + caseSteps + .customFields(EntityIdOrName(idOrName)) + .jsonValue + case (_, caseSteps) => caseSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) + } + .filter { + case (FPathElem(_, FPathElem(idOrName, _)), caseTraversal) => + db + .roTransaction(implicit graph => customFieldSrv.get(EntityIdOrName(idOrName)).value(_.`type`).getOrFail("CustomField")) + .map { + case CustomFieldType.boolean => caseTraversal.customFields(EntityIdOrName(idOrName)).value(_.booleanValue) + case CustomFieldType.date => caseTraversal.customFields(EntityIdOrName(idOrName)).value(_.dateValue) + case CustomFieldType.float => caseTraversal.customFields(EntityIdOrName(idOrName)).value(_.floatValue) + case CustomFieldType.integer => caseTraversal.customFields(EntityIdOrName(idOrName)).value(_.integerValue) + case CustomFieldType.string => caseTraversal.customFields(EntityIdOrName(idOrName)).value(_.stringValue) + } + .getOrElse(caseTraversal.constant2(null)) + case (_, caseTraversal) => caseTraversal.constant2(null) + } + .converter { + case FPathElem(_, FPathElem(idOrName, _)) => + db + .roTransaction { implicit graph => + customFieldSrv.get(EntityIdOrName(idOrName)).value(_.`type`).getOrFail("CustomField") + } + .map { + case CustomFieldType.boolean => new Converter[Any, JsValue] { def apply(x: JsValue): Any = x.as[Boolean] } + case CustomFieldType.date => new Converter[Any, JsValue] { def apply(x: JsValue): Any = x.as[Date] } + case CustomFieldType.float => new Converter[Any, JsValue] { def apply(x: JsValue): Any = x.as[Double] } + case CustomFieldType.integer => new Converter[Any, JsValue] { def apply(x: JsValue): Any = x.as[Long] } + case CustomFieldType.string => new Converter[Any, JsValue] { def apply(x: JsValue): Any = x.as[String] } + } + .getOrElse(new Converter[Any, JsValue] { def apply(x: JsValue): Any = x }) + case _ => (x: JsValue) => x + } + .custom { + case (FPathElem(_, FPathElem(idOrName, _)), value, vertex, _, graph, authContext) => + for { + c <- caseSrv.get(vertex)(graph).getOrFail("Case") + _ <- caseSrv.setOrCreateCustomField(c, EntityIdOrName(idOrName), Some(value), None)(graph, authContext) + } yield Json.obj(s"customField.$idOrName" -> value) + case (FPathElem(_, FPathEmpty), values: JsObject, vertex, _, graph, authContext) => + for { + c <- caseSrv.get(vertex)(graph).getOrFail("Case") + cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(EntityIdOrName(n))(graph).map(cf => (cf, v, None)) } + _ <- caseSrv.updateCustomField(c, cfv)(graph, authContext) + } yield Json.obj("customFields" -> values) + case _ => Failure(BadRequestError("Invalid custom fields format")) + }) .build lazy val caseTemplate: PublicProperties = @@ -204,7 +261,7 @@ class Properties @Inject() ( .flatMap { task => value.fold(taskSrv.unassign(task)(graph, authContext)) { user => userSrv - .get(user)(graph) + .get(EntityIdOrName(user))(graph) .getOrFail("User") .flatMap(taskSrv.assign(task, _)(graph, authContext)) } @@ -218,7 +275,7 @@ class Properties @Inject() ( .property("message", UMapping.string)(_.field.updatable) .property("deleted", UMapping.boolean)(_.field.updatable) .property("date", UMapping.date)(_.field.readonly) - .property("attachment", IdMapping)(_.select(_.attachments._id).readonly) + .property("attachment", UMapping.string)(_.select(_.attachments.value(_.attachmentId)).readonly) .build lazy val user: PublicProperties = diff --git a/thehive/app/org/thp/thehive/controllers/v1/QueryCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/QueryableCtrl.scala similarity index 79% rename from thehive/app/org/thp/thehive/controllers/v1/QueryCtrl.scala rename to thehive/app/org/thp/thehive/controllers/v1/QueryableCtrl.scala index 5af9b32670..0bc90304f6 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/QueryCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/QueryableCtrl.scala @@ -1,15 +1,14 @@ package org.thp.thehive.controllers.v1 +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.query.{ParamQuery, PublicProperties, Query} -case class IdOrName(idOrName: String) - trait QueryableCtrl { val entityName: String val publicProperties: PublicProperties val initialQuery: Query val pageQuery: ParamQuery[OutputParam] val outputQuery: Query - val getQuery: ParamQuery[IdOrName] + val getQuery: ParamQuery[EntityIdOrName] val extraQueries: Seq[ParamQuery[_]] = Nil } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala index 2f132f8109..6ffdbb1b81 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala @@ -1,6 +1,7 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -9,8 +10,8 @@ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputTask import org.thp.thehive.models._ -import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.CaseOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ import org.thp.thehive.services.{CaseSrv, OrganisationSrv, ShareSrv, TaskSrv} @@ -42,16 +43,16 @@ class TaskCtrl @Inject() ( _.richTaskWithCustomRenderer(taskStatsRenderer(range.extraData)(authContext)) ) ) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[Task]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Task]]( "getTask", - FieldsParser[IdOrName], - (param, graph, authContext) => taskSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => taskSrv.get(idOrName)(graph).visible(authContext) ) override val outputQuery: Query = Query.output[RichTask, Traversal.V[Task]](_.richTask) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query.init[Traversal.V[Task]]( "waitingTask", - (graph, authContext) => taskSrv.startTraversal(graph).has("status", TaskStatus.Waiting).visible(authContext) + (graph, authContext) => taskSrv.startTraversal(graph).has(_.status, TaskStatus.Waiting).visible(authContext) ), Query[Traversal.V[Task], Traversal.V[User]]("assignableUsers", (taskSteps, authContext) => taskSteps.assignableUsers(authContext)), Query[Traversal.V[Task], Traversal.V[Log]]("logs", (taskSteps, _) => taskSteps.logs), @@ -67,7 +68,7 @@ class TaskCtrl @Inject() ( val inputTask: InputTask = request.body("task") val caseId: String = request.body("caseId") for { - case0 <- caseSrv.get(caseId).can(Permissions.manageTask).getOrFail("Case") + case0 <- caseSrv.get(EntityIdOrName(caseId)).can(Permissions.manageTask).getOrFail("Case") createdTask <- taskSrv.create(inputTask.toTask, None) organisation <- organisationSrv.getOrFail(request.organisation) _ <- shareSrv.shareTask(createdTask, case0, organisation) @@ -78,7 +79,7 @@ class TaskCtrl @Inject() ( entrypoint("get task") .authRoTransaction(db) { implicit request => implicit graph => taskSrv - .getByIds(taskId) + .get(EntityIdOrName(taskId)) .visible .richTask .getOrFail("Task") @@ -103,7 +104,7 @@ class TaskCtrl @Inject() ( val propertyUpdaters: Seq[PropertyUpdater] = request.body("task") taskSrv .update( - _.getByIds(taskId) + _.get(EntityIdOrName(taskId)) .can(Permissions.manageTask), propertyUpdaters ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskRenderer.scala index d8ed893823..da24a7ef58 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskRenderer.scala @@ -11,6 +11,7 @@ import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models.Task import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CaseTemplateOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TaskOps._ import play.api.libs.json._ @@ -22,19 +23,19 @@ trait TaskRenderer { _.`case`.richCase.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) def caseParentId: Traversal.V[Task] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = - _.`case`.fold.domainMap(_.headOption.fold[JsValue](JsNull)(c => JsString(c._id))) + _.`case`.fold.domainMap(_.headOption.fold[JsValue](JsNull)(c => JsString(c._id.toString))) def caseTemplateParent: Traversal.V[Task] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = _.caseTemplate.richCaseTemplate.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) def caseTemplateParentId: Traversal.V[Task] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = - _.caseTemplate.fold.domainMap(_.headOption.fold[JsValue](JsNull)(ct => JsString(ct._id))) + _.caseTemplate.fold.domainMap(_.headOption.fold[JsValue](JsNull)(ct => JsString(ct._id.toString))) def shareCount: Traversal.V[Task] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = _.organisations.count.domainMap(count => JsNumber(count - 1)) - def isOwner(implicit authContext: AuthContext): Traversal.V[Task] => Traversal[JsBoolean, JList[Vertex], Converter[JsBoolean, JList[Vertex]]] = - _.origin.has("name", authContext.organisation).fold.domainMap(l => JsBoolean(l.nonEmpty)) + def isOwner(implicit authContext: AuthContext): Traversal.V[Task] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = + _.origin.get(authContext.organisation).fold.domainMap(l => JsBoolean(l.nonEmpty)) def taskStatsRenderer(extraData: Set[String])(implicit authContext: AuthContext diff --git a/thehive/app/org/thp/thehive/controllers/v1/UserCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/UserCtrl.scala index 68c6f48dc0..676094fa54 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/UserCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/UserCtrl.scala @@ -9,7 +9,7 @@ import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PublicProperties, Query} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} -import org.thp.scalligraph.{AuthorizationError, BadRequestError, NotFoundError, RichOptionTry} +import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityIdOrName, NotFoundError, RichOptionTry} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputUser import org.thp.thehive.models._ @@ -43,10 +43,10 @@ class UserCtrl @Inject() ( override val initialQuery: Query = Query.init[Traversal.V[User]]("listUser", (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).users) - override val getQuery: ParamQuery[IdOrName] = Query.initWithParam[IdOrName, Traversal.V[User]]( + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[User]]( "getUser", - FieldsParser[IdOrName], - (param, graph, authContext) => userSrv.get(param.idOrName)(graph).visible(authContext) + FieldsParser[EntityIdOrName], + (idOrName, graph, authContext) => userSrv.get(idOrName)(graph).visible(authContext) ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[User], IteratorOutput]( @@ -72,7 +72,7 @@ class UserCtrl @Inject() ( .map(user => Results .Ok(user.toJson) - .withHeaders("X-Organisation" -> request.organisation) + .withHeaders("X-Organisation" -> request.organisation.toString) .withHeaders("X-Permissions" -> user._1.permissions.mkString(",")) ) } @@ -83,55 +83,55 @@ class UserCtrl @Inject() ( .auth { implicit request => val inputUser: InputUser = request.body("user") db.tryTransaction { implicit graph => - val organisationName = inputUser.organisation.getOrElse(request.organisation) + val organisationName = inputUser.organisation.map(EntityIdOrName(_)).getOrElse(request.organisation) for { - _ <- userSrv.current.organisations(Permissions.manageUser).getByName(organisationName).existsOrFail + _ <- userSrv.current.organisations(Permissions.manageUser).get(organisationName).existsOrFail organisation <- organisationSrv.getOrFail(organisationName) - profile <- profileSrv.getOrFail(inputUser.profile) + profile <- profileSrv.getOrFail(EntityIdOrName(inputUser.profile)) user <- userSrv.addOrCreateUser(inputUser.toUser, inputUser.avatar, organisation, profile) } yield user -> userSrv.canSetPassword(user.user) }.flatMap { case (user, true) => inputUser .password - .map(password => authSrv.setPassword(user._id, password)) + .map(password => authSrv.setPassword(user.login, password)) .flip .map(_ => Results.Created(user.toJson)) case (user, _) => Success(Results.Created(user.toJson)) } } - def lock(userId: String): Action[AnyContent] = + def lock(userIdOrName: String): Action[AnyContent] = entrypoint("lock user") .authTransaction(db) { implicit request => implicit graph => for { - user <- userSrv.current.organisations(Permissions.manageUser).users.get(userId).getOrFail("User") + user <- userSrv.current.organisations(Permissions.manageUser).users.get(EntityIdOrName(userIdOrName)).getOrFail("User") _ <- userSrv.lock(user) } yield Results.NoContent } - def delete(userId: String, organisation: Option[String]): Action[AnyContent] = + def delete(userIdOrName: String, organisation: Option[String]): Action[AnyContent] = entrypoint("delete user") .authTransaction(db) { implicit request => implicit graph => for { - org <- organisationSrv.getOrFail(organisation.getOrElse(request.organisation)) - user <- userSrv.current.organisations(Permissions.manageUser).users.get(userId).getOrFail("User") + org <- organisationSrv.getOrFail(organisation.map(EntityIdOrName(_)).getOrElse(request.organisation)) + user <- userSrv.current.organisations(Permissions.manageUser).users.get(EntityIdOrName(userIdOrName)).getOrFail("User") _ <- userSrv.delete(user, org) } yield Results.NoContent } - def get(userId: String): Action[AnyContent] = + def get(userIdOrName: String): Action[AnyContent] = entrypoint("get user") .authRoTransaction(db) { implicit request => implicit graph => userSrv - .get(userId) + .get(EntityIdOrName(userIdOrName)) .visible - .richUser(request.organisation) + .richUser .getOrFail("User") .map(user => Results.Ok(user.toJson)) } - def update(userId: String): Action[AnyContent] = + def update(userIdOrName: String): Action[AnyContent] = entrypoint("update user") .extract("name", FieldsParser.string.optional.on("name")) .extract("organisation", FieldsParser.string.optional.on("organisation")) @@ -147,7 +147,7 @@ class UserCtrl @Inject() ( val isCurrentUser: Boolean = userSrv .current - .get(userId) + .get(EntityIdOrName(userIdOrName)) .exists val isUserAdmin: Boolean = @@ -155,13 +155,13 @@ class UserCtrl @Inject() ( .current .organisations(Permissions.manageUser) .users - .get(userId) + .get(EntityIdOrName(userIdOrName)) .exists def requireAdmin[A](body: => Try[A]): Try[A] = if (isUserAdmin) body else Failure(AuthorizationError("You are not permitted to update this user")) - userSrv.get(userId).visible.getOrFail("User").flatMap { + userSrv.get(EntityIdOrName(userIdOrName)).visible.getOrFail("User").flatMap { case _ if !isCurrentUser && !isUserAdmin => Failure(AuthorizationError("You are not permitted to update this user")) case user => auditSrv @@ -180,8 +180,8 @@ class UserCtrl @Inject() ( maybeOrganisation.fold[Try[JsObject]](Failure(BadRequestError("Organisation information is required to update user profile"))) { organisationName => for { - profile <- profileSrv.getOrFail(profileName) - organisation <- organisationSrv.getOrFail(organisationName) + profile <- profileSrv.getOrFail(EntityIdOrName(profileName)) + organisation <- organisationSrv.getOrFail(EntityIdOrName(organisationName)) _ <- userSrv.setProfile(user, organisation, profile) } yield Json.obj("organisation" -> organisation.name, "profile" -> profile.name) } @@ -193,7 +193,7 @@ class UserCtrl @Inject() ( Success(Json.obj("avatar" -> JsNull)) case avatar => attachmentSrv - .create(s"$userId.avatar", "image/jpeg", Base64.getDecoder.decode(avatar)) + .create(s"${user.login}.avatar", "image/jpeg", Base64.getDecoder.decode(avatar)) .flatMap(userSrv.setAvatar(user, _)) .map(_ => Json.obj("avatar" -> "[binary data]")) }.flip @@ -206,7 +206,7 @@ class UserCtrl @Inject() ( } } - def setPassword(userId: String): Action[AnyContent] = + def setPassword(userIdOrName: String): Action[AnyContent] = entrypoint("set password") .extract("password", FieldsParser[String].on("password")) .auth { implicit request => @@ -216,27 +216,27 @@ class UserCtrl @Inject() ( .current .organisations(Permissions.manageUser) .users - .get(userId) + .get(EntityIdOrName(userIdOrName)) .getOrFail("User") } - _ <- authSrv.setPassword(userId, request.body("password")) + _ <- authSrv.setPassword(user.login, request.body("password")) _ <- db.tryTransaction(implicit graph => auditSrv.user.update(user, Json.obj("password" -> ""))) } yield Results.NoContent } - def changePassword(userId: String): Action[AnyContent] = + def changePassword(userIdOrName: String): Action[AnyContent] = entrypoint("change password") .extract("password", FieldsParser[String].on("password")) .extract("currentPassword", FieldsParser[String].on("currentPassword")) .auth { implicit request => for { - user <- db.roTransaction(implicit graph => userSrv.current.get(userId).getOrFail("User")) - _ <- authSrv.changePassword(userId, request.body("currentPassword"), request.body("password")) + user <- db.roTransaction(implicit graph => userSrv.current.get(EntityIdOrName(userIdOrName)).getOrFail("User")) + _ <- authSrv.changePassword(user.login, request.body("currentPassword"), request.body("password")) _ <- db.tryTransaction(implicit graph => auditSrv.user.update(user, Json.obj("password" -> ""))) } yield Results.NoContent } - def getKey(userId: String): Action[AnyContent] = + def getKey(userIdOrName: String): Action[AnyContent] = entrypoint("get key") .auth { implicit request => for { @@ -245,14 +245,14 @@ class UserCtrl @Inject() ( .current .organisations(Permissions.manageUser) .users - .get(userId) + .get(EntityIdOrName(userIdOrName)) .getOrFail("User") } - key <- authSrv.getKey(user._id) + key <- authSrv.getKey(user.login) } yield Results.Ok(key) } - def removeKey(userId: String): Action[AnyContent] = + def removeKey(userIdOrName: String): Action[AnyContent] = entrypoint("remove key") .auth { implicit request => for { @@ -261,16 +261,16 @@ class UserCtrl @Inject() ( .current .organisations(Permissions.manageUser) .users - .get(userId) + .get(EntityIdOrName(userIdOrName)) .getOrFail("User") } - _ <- authSrv.removeKey(userId) + _ <- authSrv.removeKey(user.login) _ <- db.tryTransaction(implicit graph => auditSrv.user.update(user, Json.obj("key" -> ""))) } yield Results.NoContent // Failure(AuthorizationError(s"User $userId doesn't exist or permission is insufficient")) } - def renewKey(userId: String): Action[AnyContent] = + def renewKey(userIdOrName: String): Action[AnyContent] = entrypoint("renew key") .auth { implicit request => for { @@ -279,18 +279,18 @@ class UserCtrl @Inject() ( .current .organisations(Permissions.manageUser) .users - .get(userId) + .get(EntityIdOrName(userIdOrName)) .getOrFail("User") } - key <- authSrv.renewKey(userId) + key <- authSrv.renewKey(user.login) _ <- db.tryTransaction(implicit graph => auditSrv.user.update(user, Json.obj("key" -> ""))) } yield Results.Ok(key) } - def avatar(userId: String): Action[AnyContent] = + def avatar(userIdOrName: String): Action[AnyContent] = entrypoint("get user avatar") .authTransaction(db) { implicit request => implicit graph => - userSrv.get(userId).visible.avatar.headOption match { + userSrv.get(EntityIdOrName(userIdOrName)).visible.avatar.headOption match { case Some(avatar) if attachmentSrv.exists(avatar) => Success( Result( @@ -302,7 +302,7 @@ class UserCtrl @Inject() ( ) ) ) - case _ => Failure(NotFoundError(s"user $userId has no avatar")) + case _ => Failure(NotFoundError(s"user $userIdOrName has no avatar")) } } } diff --git a/thehive/app/org/thp/thehive/models/Alert.scala b/thehive/app/org/thp/thehive/models/Alert.scala index e99d6f75e3..98a993bf4c 100644 --- a/thehive/app/org/thp/thehive/models/Alert.scala +++ b/thehive/app/org/thp/thehive/models/Alert.scala @@ -61,11 +61,11 @@ case class RichAlert( organisation: String, tags: Seq[Tag with Entity], customFields: Seq[RichCustomField], - caseId: Option[String], + caseId: Option[EntityId], caseTemplate: Option[String], observableCount: Long ) { - def _id: String = alert._id + def _id: EntityId = alert._id def _createdAt: Date = alert._createdAt def _createdBy: String = alert._createdBy def _updatedAt: Option[Date] = alert._updatedAt @@ -92,7 +92,7 @@ object RichAlert { organisation: String, tags: Seq[Tag with Entity], customFields: Seq[RichCustomField], - caseId: Option[String], + caseId: Option[EntityId], caseTemplate: Option[String], observableCount: Long ): RichAlert = diff --git a/thehive/app/org/thp/thehive/models/Audit.scala b/thehive/app/org/thp/thehive/models/Audit.scala index a3b0d18b18..a97ebad3e8 100644 --- a/thehive/app/org/thp/thehive/models/Audit.scala +++ b/thehive/app/org/thp/thehive/models/Audit.scala @@ -6,7 +6,7 @@ import org.apache.tinkerpop.gremlin.structure.{Edge, Graph, Vertex} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.Converter -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} @BuildEdgeEntity[Audit, User] case class AuditUser() @@ -20,12 +20,14 @@ case class Audit( objectId: Option[String], objectType: Option[String], details: Option[String] -) +) { + def objectEntityId: Option[EntityId] = objectId.map(EntityId.read) +} object Audit { def apply(action: String, entity: Entity, details: Option[String] = None)(implicit authContext: AuthContext): Audit = - Audit(authContext.requestId, action, mainAction = false, Some(entity._id), Some(entity._label), details) + Audit(authContext.requestId, action, mainAction = false, Some(entity._id.toString), Some(entity._label), details) final val create = "create" final val update = "update" @@ -34,7 +36,7 @@ object Audit { } case class RichAudit( - _id: String, + _id: EntityId, _createdAt: Date, _createdBy: String, action: String, @@ -46,7 +48,9 @@ case class RichAudit( context: Entity, visibilityContext: Entity, `object`: Option[Entity] -) +) { + def objectEntityId: Option[EntityId] = objectId.map(EntityId.read) +} object RichAudit { @@ -84,21 +88,22 @@ object Audited { override val fields: Map[String, Mapping[_, _, _]] = Map.empty override val converter: Converter[EEntity, Edge] = (element: Edge) => new Audited with Entity { - override val _id: String = element.id().toString + override val _id: EntityId = EntityId(element.id()) override val _label: String = "Audited" override val _createdBy: String = UMapping.string.getProperty(element, "_createdBy") override val _updatedBy: Option[String] = UMapping.string.optional.getProperty(element, "_updatedBy") override val _createdAt: Date = UMapping.date.getProperty(element, "_createdAt") override val _updatedAt: Option[Date] = UMapping.date.optional.getProperty(element, "_updatedAt") } - override def addEntity(a: Audited, entity: Entity): EEntity = new Audited with Entity { - override def _id: String = entity._id - override def _label: String = entity._label - override def _createdBy: String = entity._createdBy - override def _updatedBy: Option[String] = entity._updatedBy - override def _createdAt: Date = entity._createdAt - override def _updatedAt: Option[Date] = entity._updatedAt - } + override def addEntity(a: Audited, entity: Entity): EEntity = + new Audited with Entity { + override def _id: EntityId = entity._id + override def _label: String = entity._label + override def _createdBy: String = entity._createdBy + override def _updatedBy: Option[String] = entity._updatedBy + override def _createdAt: Date = entity._createdAt + override def _updatedAt: Option[Date] = entity._updatedAt + } override def create(e: Audited, from: Vertex, to: Vertex)(implicit db: Database, graph: Graph): Edge = from.addEdge(label, to) } } @@ -114,21 +119,22 @@ object AuditContext extends HasModel { override val fields: Map[String, Mapping[_, _, _]] = Map.empty override val converter: Converter[EEntity, Edge] = (element: Edge) => new AuditContext with Entity { - override val _id: String = element.id().toString + override val _id: EntityId = EntityId(element.id()) override val _label: String = "AuditContext" override val _createdBy: String = UMapping.string.getProperty(element, "_createdBy") override val _updatedBy: Option[String] = UMapping.string.optional.getProperty(element, "_updatedBy") override val _createdAt: Date = UMapping.date.getProperty(element, "_createdAt") override val _updatedAt: Option[Date] = UMapping.date.optional.getProperty(element, "_updatedAt") } - override def addEntity(a: AuditContext, entity: Entity): EEntity = new AuditContext with Entity { - override def _id: String = entity._id - override def _label: String = entity._label - override def _createdBy: String = entity._createdBy - override def _updatedBy: Option[String] = entity._updatedBy - override def _createdAt: Date = entity._createdAt - override def _updatedAt: Option[Date] = entity._updatedAt - } + override def addEntity(a: AuditContext, entity: Entity): EEntity = + new AuditContext with Entity { + override def _id: EntityId = entity._id + override def _label: String = entity._label + override def _createdBy: String = entity._createdBy + override def _updatedBy: Option[String] = entity._updatedBy + override def _createdAt: Date = entity._createdAt + override def _updatedAt: Option[Date] = entity._updatedAt + } override def create(e: AuditContext, from: Vertex, to: Vertex)(implicit db: Database, graph: Graph): Edge = from.addEdge(label, to) } diff --git a/thehive/app/org/thp/thehive/models/Case.scala b/thehive/app/org/thp/thehive/models/Case.scala index e476b48a85..e15367fc82 100644 --- a/thehive/app/org/thp/thehive/models/Case.scala +++ b/thehive/app/org/thp/thehive/models/Case.scala @@ -106,7 +106,7 @@ case class RichCase( customFields: Seq[RichCustomField], userPermissions: Set[Permission] ) { - def _id: String = `case`._id + def _id: EntityId = `case`._id def _createdBy: String = `case`._createdBy def _updatedBy: Option[String] = `case`._updatedBy def _createdAt: Date = `case`._createdAt @@ -127,7 +127,7 @@ case class RichCase( object RichCase { def apply( - __id: String, + __id: EntityId, __createdBy: String, __updatedBy: Option[String], __createdAt: Date, @@ -151,7 +151,7 @@ object RichCase { userPermissions: Set[Permission] ): RichCase = { val `case`: Case with Entity = new Case(number, title, description, severity, startDate, endDate, flag, tlp, pap, status, summary) with Entity { - override val _id: String = __id + override val _id: EntityId = __id override val _label: String = "Case" override val _createdBy: String = __createdBy override val _updatedBy: Option[String] = __updatedBy diff --git a/thehive/app/org/thp/thehive/models/CaseTemplate.scala b/thehive/app/org/thp/thehive/models/CaseTemplate.scala index f4ac46c21b..dabb30d7a7 100644 --- a/thehive/app/org/thp/thehive/models/CaseTemplate.scala +++ b/thehive/app/org/thp/thehive/models/CaseTemplate.scala @@ -3,7 +3,7 @@ package org.thp.thehive.models import java.util.Date import org.thp.scalligraph.models.Entity -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} @BuildEdgeEntity[CaseTemplate, Organisation] case class CaseTemplateOrganisation() @@ -51,7 +51,7 @@ case class RichCaseTemplate( tasks: Seq[RichTask], customFields: Seq[RichCustomField] ) { - def _id: String = caseTemplate._id + def _id: EntityId = caseTemplate._id def _createdBy: String = caseTemplate._createdBy def _updatedBy: Option[String] = caseTemplate._updatedBy def _createdAt: Date = caseTemplate._createdAt diff --git a/thehive/app/org/thp/thehive/models/CustomField.scala b/thehive/app/org/thp/thehive/models/CustomField.scala index 508d7db7a3..00900c32b0 100644 --- a/thehive/app/org/thp/thehive/models/CustomField.scala +++ b/thehive/app/org/thp/thehive/models/CustomField.scala @@ -61,7 +61,7 @@ class CustomFieldValueEdge(edge: Edge) extends CustomFieldValue[CustomFieldValue override def productArity: Int = 0 override def canEqual(that: Any): Boolean = that.isInstanceOf[CustomFieldValueEdge] - override def _id: String = edge.id().toString + override def _id: EntityId = EntityId(edge.id()) override def _label: String = edge.label() override def _createdBy: String = UMapping.string.getProperty(edge, "_createdBy") override def _updatedBy: Option[String] = UMapping.string.optional.getProperty(edge, "_updatedBy") @@ -105,16 +105,17 @@ object CustomFieldString extends CustomFieldType[String] { override val name: String = "string" override val writes: Writes[String] = Writes.StringWrites - override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = value.getOrElse(JsNull) match { - case v: String => Success(customFieldValue.stringValue = Some(v)) - case JsString(v) => Success(customFieldValue.stringValue = Some(v)) - case JsNull | null => Success(customFieldValue.stringValue = None) - case obj: JsObject => - val stringValue = (obj \ "string").asOpt[String] - val order = (obj \ "order").asOpt[Int] - Success((customFieldValue.stringValue = stringValue).order = order) - case _ => setValueFailure(value) - } + override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = + value.getOrElse(JsNull) match { + case v: String => Success(customFieldValue.stringValue = Some(v)) + case JsString(v) => Success(customFieldValue.stringValue = Some(v)) + case JsNull | null => Success(customFieldValue.stringValue = None) + case obj: JsObject => + val stringValue = (obj \ "string").asOpt[String] + val order = (obj \ "order").asOpt[Int] + Success((customFieldValue.stringValue = stringValue).order = order) + case _ => setValueFailure(value) + } override def getValue(ccf: CustomFieldValue[_]): Option[String] = ccf.stringValue } @@ -123,17 +124,18 @@ object CustomFieldBoolean extends CustomFieldType[Boolean] { override val name: String = "boolean" override val writes: Writes[Boolean] = Writes.BooleanWrites - override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = value.getOrElse(JsNull) match { - case v: Boolean => Success(customFieldValue.booleanValue = Some(v)) - case JsBoolean(v) => Success(customFieldValue.booleanValue = Some(v)) - case JsNull | null => Success(customFieldValue.booleanValue = None) - case obj: JsObject => - val booleanValue = (obj \ "boolean").asOpt[Boolean] - val order = (obj \ "order").asOpt[Int] - Success((customFieldValue.booleanValue = booleanValue).order = order) + override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = + value.getOrElse(JsNull) match { + case v: Boolean => Success(customFieldValue.booleanValue = Some(v)) + case JsBoolean(v) => Success(customFieldValue.booleanValue = Some(v)) + case JsNull | null => Success(customFieldValue.booleanValue = None) + case obj: JsObject => + val booleanValue = (obj \ "boolean").asOpt[Boolean] + val order = (obj \ "order").asOpt[Int] + Success((customFieldValue.booleanValue = booleanValue).order = order) - case _ => setValueFailure(value) - } + case _ => setValueFailure(value) + } override def getValue(ccf: CustomFieldValue[_]): Option[Boolean] = ccf.booleanValue } @@ -142,17 +144,18 @@ object CustomFieldInteger extends CustomFieldType[Int] { override val name: String = "integer" override val writes: Writes[Int] = Writes.IntWrites - override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = value.getOrElse(JsNull) match { - case v: Int => Success(customFieldValue.integerValue = Some(v)) - case JsNumber(n) => Success(customFieldValue.integerValue = Some(n.toInt)) - case JsNull | null => Success(customFieldValue.integerValue = None) - case obj: JsObject => - val integerValue = (obj \ "integer").asOpt[Int] - val order = (obj \ "order").asOpt[Int] - Success((customFieldValue.integerValue = integerValue).order = order) + override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = + value.getOrElse(JsNull) match { + case v: Int => Success(customFieldValue.integerValue = Some(v)) + case JsNumber(n) => Success(customFieldValue.integerValue = Some(n.toInt)) + case JsNull | null => Success(customFieldValue.integerValue = None) + case obj: JsObject => + val integerValue = (obj \ "integer").asOpt[Int] + val order = (obj \ "order").asOpt[Int] + Success((customFieldValue.integerValue = integerValue).order = order) - case _ => setValueFailure(value) - } + case _ => setValueFailure(value) + } override def getValue(ccf: CustomFieldValue[_]): Option[Int] = ccf.integerValue } @@ -161,17 +164,18 @@ object CustomFieldFloat extends CustomFieldType[Double] { override val name: String = "float" override val writes: Writes[Double] = Writes.DoubleWrites - override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = value.getOrElse(JsNull) match { - case n: Number => Success(customFieldValue.floatValue = Some(n.doubleValue())) - case JsNumber(n) => Success(customFieldValue.floatValue = Some(n.toDouble)) - case JsNull | null => Success(customFieldValue.floatValue = None) - case obj: JsObject => - val floatValue = (obj \ "float").asOpt[Double] - val order = (obj \ "order").asOpt[Int] - Success((customFieldValue.floatValue = floatValue).order = order) + override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = + value.getOrElse(JsNull) match { + case n: Number => Success(customFieldValue.floatValue = Some(n.doubleValue())) + case JsNumber(n) => Success(customFieldValue.floatValue = Some(n.toDouble)) + case JsNull | null => Success(customFieldValue.floatValue = None) + case obj: JsObject => + val floatValue = (obj \ "float").asOpt[Double] + val order = (obj \ "order").asOpt[Int] + Success((customFieldValue.floatValue = floatValue).order = order) - case _ => setValueFailure(value) - } + case _ => setValueFailure(value) + } override def getValue(ccf: CustomFieldValue[_]): Option[Double] = ccf.floatValue } @@ -180,18 +184,19 @@ object CustomFieldDate extends CustomFieldType[Date] { override val name: String = "date" override val writes: Writes[Date] = Writes[Date](d => JsNumber(d.getTime)) - override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = value.getOrElse(JsNull) match { - case n: Number => Success(customFieldValue.dateValue = Some(new Date(n.longValue()))) - case JsNumber(n) => Success(customFieldValue.dateValue = Some(new Date(n.toLong))) - case v: Date => Success(customFieldValue.dateValue = Some(v)) - case JsNull | null => Success(customFieldValue.dateValue = None) - case obj: JsObject => - val dateValue = (obj \ "date").asOpt[Long].map(new Date(_)) - val order = (obj \ "order").asOpt[Int] - Success((customFieldValue.dateValue = dateValue).order = order) - - case _ => setValueFailure(value) - } + override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = + value.getOrElse(JsNull) match { + case n: Number => Success(customFieldValue.dateValue = Some(new Date(n.longValue()))) + case JsNumber(n) => Success(customFieldValue.dateValue = Some(new Date(n.toLong))) + case v: Date => Success(customFieldValue.dateValue = Some(v)) + case JsNull | null => Success(customFieldValue.dateValue = None) + case obj: JsObject => + val dateValue = (obj \ "date").asOpt[Long].map(new Date(_)) + val order = (obj \ "order").asOpt[Int] + Success((customFieldValue.dateValue = dateValue).order = order) + + case _ => setValueFailure(value) + } override def getValue(ccf: CustomFieldValue[_]): Option[Date] = ccf.dateValue } diff --git a/thehive/app/org/thp/thehive/models/Dashboard.scala b/thehive/app/org/thp/thehive/models/Dashboard.scala index 9110605f27..25b2941989 100644 --- a/thehive/app/org/thp/thehive/models/Dashboard.scala +++ b/thehive/app/org/thp/thehive/models/Dashboard.scala @@ -3,7 +3,7 @@ package org.thp.thehive.models import java.util.Date import org.thp.scalligraph.models.Entity -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityIdOrName} import play.api.libs.json.JsObject @BuildVertexEntity @@ -19,7 +19,7 @@ case class RichDashboard( dashboard: Dashboard with Entity, organisationShares: Map[String, Boolean] ) { - def _id: String = dashboard._id + def _id: EntityIdOrName = dashboard._id def _createdBy: String = dashboard._createdBy def _updatedBy: Option[String] = dashboard._updatedBy def _createdAt: Date = dashboard._createdAt diff --git a/thehive/app/org/thp/thehive/models/Log.scala b/thehive/app/org/thp/thehive/models/Log.scala index a62f4fbef7..4c5c54bb41 100644 --- a/thehive/app/org/thp/thehive/models/Log.scala +++ b/thehive/app/org/thp/thehive/models/Log.scala @@ -3,7 +3,7 @@ package org.thp.thehive.models import java.util.Date import org.thp.scalligraph.models.Entity -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} @BuildEdgeEntity[Log, Attachment] case class LogAttachment() @@ -12,7 +12,7 @@ case class LogAttachment() case class Log(message: String, date: Date, deleted: Boolean) case class RichLog(log: Log with Entity, attachments: Seq[Attachment with Entity]) { - def _id: String = log._id + def _id: EntityId = log._id def _createdBy: String = log._createdBy def _updatedBy: Option[String] = log._updatedBy def _createdAt: Date = log._createdAt diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index 5ad3524648..31dd2d149a 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -3,7 +3,7 @@ package org.thp.thehive.models import java.util.Date import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType} -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} @BuildEdgeEntity[Observable, KeyValue] case class ObservableKeyValue() @@ -30,7 +30,7 @@ case class RichObservable( extensions: Seq[KeyValue with Entity], reportTags: Seq[ReportTag with Entity] ) { - def _id: String = observable._id + def _id: EntityId = observable._id def _createdBy: String = observable._createdBy def _updatedBy: Option[String] = observable._updatedBy def _createdAt: Date = observable._createdAt diff --git a/thehive/app/org/thp/thehive/models/Organisation.scala b/thehive/app/org/thp/thehive/models/Organisation.scala index b06c516259..41ca8dd5c2 100644 --- a/thehive/app/org/thp/thehive/models/Organisation.scala +++ b/thehive/app/org/thp/thehive/models/Organisation.scala @@ -3,7 +3,7 @@ package org.thp.thehive.models import java.util.Date import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType} -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} @BuildVertexEntity @DefineIndex(IndexType.unique, "name") @@ -23,7 +23,7 @@ case class OrganisationOrganisation() case class RichOrganisation(organisation: Organisation with Entity, links: Seq[Organisation with Entity]) { def name: String = organisation.name def description: String = organisation.description - def _id: String = organisation._id + def _id: EntityId = organisation._id def _createdAt: Date = organisation._createdAt def _createdBy: String = organisation._createdBy def _updatedAt: Option[Date] = organisation._updatedAt diff --git a/thehive/app/org/thp/thehive/models/Share.scala b/thehive/app/org/thp/thehive/models/Share.scala index 2ea59c2aea..0b5fc13646 100644 --- a/thehive/app/org/thp/thehive/models/Share.scala +++ b/thehive/app/org/thp/thehive/models/Share.scala @@ -3,7 +3,7 @@ package org.thp.thehive.models import java.util.Date import org.thp.scalligraph.models.Entity -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} @BuildVertexEntity case class Share(owner: Boolean) @@ -20,8 +20,8 @@ case class ShareTask() @BuildEdgeEntity[Share, Profile] case class ShareProfile() -case class RichShare(share: Share with Entity, caseId: String, organisationName: String, profileName: String) { - def _id: String = share._id +case class RichShare(share: Share with Entity, caseId: EntityId, organisationName: String, profileName: String) { + def _id: EntityId = share._id def _createdBy: String = share._createdBy def _updatedBy: Option[String] = share._updatedBy def _createdAt: Date = share._createdAt diff --git a/thehive/app/org/thp/thehive/models/Task.scala b/thehive/app/org/thp/thehive/models/Task.scala index 9468250450..4ad6480153 100644 --- a/thehive/app/org/thp/thehive/models/Task.scala +++ b/thehive/app/org/thp/thehive/models/Task.scala @@ -36,7 +36,7 @@ case class RichTask( task: Task with Entity, assignee: Option[User with Entity] ) { - def _id: String = task._id + def _id: EntityId = task._id def _createdBy: String = task._createdBy def _updatedBy: Option[String] = task._updatedBy def _createdAt: Date = task._createdAt diff --git a/thehive/app/org/thp/thehive/models/User.scala b/thehive/app/org/thp/thehive/models/User.scala index 670d36b9d5..73a45f5309 100644 --- a/thehive/app/org/thp/thehive/models/User.scala +++ b/thehive/app/org/thp/thehive/models/User.scala @@ -4,7 +4,7 @@ import java.util.Date import org.thp.scalligraph.auth.{Permission, User => ScalligraphUser} import org.thp.scalligraph.models._ -import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity} +import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId} import org.thp.thehive.services.LocalPasswordAuthSrv @BuildEdgeEntity[User, Role] @@ -45,7 +45,7 @@ object User { // preference: JsObject) case class RichUser(user: User with Entity, avatar: Option[String], profile: String, permissions: Set[Permission], organisation: String) { - def _id: String = user._id + def _id: EntityId = user._id def _createdBy: String = user._createdBy def _updatedBy: Option[String] = user._updatedBy def _createdAt: Date = user._createdAt diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 2a330e7e86..a91be8df3d 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -12,7 +12,7 @@ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IdentityConverter, StepLabel, Traversal} -import org.thp.scalligraph.{CreateError, RichOptionTry, RichSeq} +import org.thp.scalligraph.{CreateError, EntityId, EntityIdOrName, RichOptionTry, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ @@ -45,10 +45,10 @@ class AlertSrv @Inject() ( val alertCaseTemplateSrv = new EdgeSrv[AlertCaseTemplate, Alert, CaseTemplate] val alertObservableSrv = new EdgeSrv[AlertObservable, Alert, Observable] - override def get(idOrSource: String)(implicit graph: Graph): Traversal.V[Alert] = - idOrSource.split(';') match { + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Alert] = + name.split(';') match { case Array(tpe, source, sourceRef) => startTraversal.getBySourceId(tpe, source, sourceRef) - case _ => super.getByIds(idOrSource) + case _ => startTraversal.limit(0) } def create( @@ -174,7 +174,7 @@ class AlertSrv @Inject() ( customFieldValue: Option[Any] )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { - cf <- customFieldSrv.getOrFail(customFieldName) + cf <- customFieldSrv.getOrFail(EntityIdOrName(customFieldName)) ccf <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), customFieldValue) ccfe <- alertCustomFieldSrv.create(ccf, alert, cf) } yield RichCustomField(cf, ccfe) @@ -209,25 +209,25 @@ class AlertSrv @Inject() ( .map(_ => ()) } - def markAsUnread(alertId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def markAsUnread(alertId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { alert <- get(alertId).update(_.read, false: Boolean).getOrFail("Alert") _ <- auditSrv.alert.update(alert, Json.obj("read" -> false)) } yield () - def markAsRead(alertId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def markAsRead(alertId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { alert <- get(alertId).update(_.read, true: Boolean).getOrFail("Alert") _ <- auditSrv.alert.update(alert, Json.obj("read" -> true)) } yield () - def followAlert(alertId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def followAlert(alertId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { alert <- get(alertId).update(_.follow, true: Boolean).getOrFail("Alert") _ <- auditSrv.alert.update(alert, Json.obj("follow" -> true)) } yield () - def unfollowAlert(alertId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def unfollowAlert(alertId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { alert <- get(alertId).update(_.follow, false: Boolean).getOrFail("Alert") _ <- auditSrv.alert.update(alert, Json.obj("follow" -> false)) @@ -236,12 +236,12 @@ class AlertSrv @Inject() ( def createCase(alert: RichAlert, user: Option[User with Entity], organisation: Organisation with Entity)(implicit graph: Graph, authContext: AuthContext - ): Try[RichCase] = + ): Try[RichCase] = // FIXME check if alert is already imported for { caseTemplate <- alert .caseTemplate - .map(caseTemplateSrv.get(_).richCaseTemplate.getOrFail("CaseTemplate")) + .map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).richCaseTemplate.getOrFail("CaseTemplate")) .flip customField = alert.customFields.map(f => (f.name, f.value, f.order)) case0 = Case( @@ -264,7 +264,7 @@ class AlertSrv @Inject() ( _ <- markAsRead(alert._id) } yield createdCase - def mergeInCase(alertId: String, caseId: String)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = + def mergeInCase(alertId: EntityIdOrName, caseId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = for { alert <- getOrFail(alertId) case0 <- caseSrv.getOrFail(caseId) @@ -300,7 +300,7 @@ class AlertSrv @Inject() ( .get(`case`) .observables .filter { o => - richObservable.dataOrAttachment.fold(d => o.filterOnData(d.data), a => o.attachments.has("attachmentId", a.attachmentId)) + richObservable.dataOrAttachment.fold(d => o.filterOnData(d.data), a => o.attachments.has(_.attachmentId, a.attachmentId)) } .headOption .foreach { observable => @@ -326,11 +326,14 @@ class AlertSrv @Inject() ( object AlertOps { implicit class AlertOpsDefs(traversal: Traversal.V[Alert]) { - def get(idOrSource: String): Traversal.V[Alert] = - idOrSource.split(';') match { - case Array(tpe, source, sourceRef) => getBySourceId(tpe, source, sourceRef) - case _ => traversal.getByIds(idOrSource) - } + def get(idOrSource: EntityIdOrName): Traversal.V[Alert] = + idOrSource.fold( + traversal.getByIds(_), + _.split(';') match { + case Array(tpe, source, sourceRef) => getBySourceId(tpe, source, sourceRef) + case _ => traversal.limit(0) + } + ) def getBySourceId(`type`: String, source: String, sourceRef: String): Traversal.V[Alert] = traversal @@ -353,24 +356,18 @@ object AlertOps { traversal.outE[AlertTag].filter(_.otherV.hasId(tags.map(_._id).toSeq: _*)).remove() def visible(implicit authContext: AuthContext): Traversal.V[Alert] = - traversal.filter( - _.out[AlertOrganisation] - .has("name", authContext.organisation) - ) + traversal.filter(_.organisation.get(authContext.organisation)) def can(permission: Permission)(implicit authContext: AuthContext): Traversal.V[Alert] = if (authContext.permissions.contains(permission)) - traversal.filter( - _.out[AlertOrganisation] - .has("name", authContext.organisation) - ) + traversal.filter(_.organisation.get(authContext.organisation)) else traversal.limit(0) def imported: Traversal[Boolean, Boolean, IdentityConverter[Boolean]] = traversal - .outE[AlertCase] + .`case` .count - .choose(_.is(P.gt(0)), onTrue = _.constant(true), onFalse = _.constant(false)) + .choose(_.is(P.gt(0)), onTrue = true, onFalse = false) def similarCases(implicit authContext: AuthContext @@ -415,18 +412,17 @@ object AlertOps { JMap[String, Any], Converter[(AlertCustomField with Entity, CustomField with Entity), JMap[String, Any]] ]] - val caseIdLabel = StepLabel[Seq[String], JList[AnyRef], Converter.CList[String, AnyRef, Converter[String, AnyRef]]] + val caseIdLabel = StepLabel[Seq[EntityId], JList[AnyRef], Converter.CList[EntityId, AnyRef, Converter[EntityId, AnyRef]]] val caseTemplateNameLabel = StepLabel[Seq[String], JList[String], Converter.CList[String, String, Converter[String, String]]] val observableCountLabel = StepLabel[Long, JLong, Converter[Long, JLong]] val result = traversal .`match`( - _.as(alertLabel)(_.out("AlertOrganisation").has("name", authContext.organisation).v[Organisation]).as(organisationLabel), - _.as(alertLabel)(_.out("AlertTag").v[Tag].fold).as(tagsLabel), + _.as(alertLabel)(_.organisation.current).as(organisationLabel), + _.as(alertLabel)(_.tags.fold).as(tagsLabel), _.as(alertLabel)( _.outE[AlertCustomField] - .e[AlertCustomField] .as(customFieldValueLabel) .inV .v[CustomField] @@ -434,9 +430,9 @@ object AlertOps { .select((customFieldValueLabel, customFieldLabel)) .fold ).as(customFieldWithValueLabel), - _.as(alertLabel)(_.out[AlertCase]._id.fold).as(caseIdLabel), - _.as(alertLabel)(_.out[AlertCaseTemplate].v[CaseTemplate].value(_.name).fold).as(caseTemplateNameLabel), - _.as(alertLabel)(_.outE[AlertObservable].count).as(observableCountLabel) + _.as(alertLabel)(_.`case`._id.fold).as(caseIdLabel), + _.as(alertLabel)(_.caseTemplate.value(_.name).fold).as(caseTemplateNameLabel), + _.as(alertLabel)(_.observables.count).as(observableCountLabel) ) .select((alertLabel, organisationLabel, tagsLabel, customFieldWithValueLabel, caseIdLabel, caseTemplateNameLabel, observableCountLabel)) .domainMap { @@ -460,13 +456,11 @@ object AlertOps { def customFields(name: String): Traversal.E[AlertCustomField] = traversal.outE[AlertCustomField].filter(_.inV.v[CustomField].has(_.name, name)) - def customFields: Traversal.E[AlertCustomField] = - traversal.outE[AlertCustomField].e[AlertCustomField] + def customFields: Traversal.E[AlertCustomField] = traversal.outE[AlertCustomField] def richCustomFields: Traversal[RichCustomField, JMap[String, Any], Converter[RichCustomField, JMap[String, Any]]] = traversal .outE[AlertCustomField] - .e[AlertCustomField] .project(_.by.by(_.inV.v[CustomField])) .domainMap { case (cfv, cf) => RichCustomField(cf, cfv) @@ -474,18 +468,20 @@ object AlertOps { def observables: Traversal.V[Observable] = traversal.out[AlertObservable].v[Observable] + def caseTemplate: Traversal.V[CaseTemplate] = traversal.out[AlertCaseTemplate].v[CaseTemplate] + def richAlertWithCustomRenderer[D, G, C <: Converter[D, G]]( entityRenderer: Traversal.V[Alert] => Traversal[D, G, C] ): Traversal[(RichAlert, D), JMap[String, Any], Converter[(RichAlert, D), JMap[String, Any]]] = traversal .project( _.by - .by(_.out[AlertOrganisation].v[Organisation].value(_.name)) - .by(_.out[AlertTag].v[Tag].fold) + .by(_.organisation.value(_.name)) + .by(_.tags.fold) .by(_.richCustomFields.fold) - .by(_.out[AlertCase]._id.fold) - .by(_.out[AlertCaseTemplate].v[CaseTemplate].value(_.name).fold) - .by(_.outE[AlertObservable].count) + .by(_.`case`._id.fold) + .by(_.caseTemplate.value(_.name).fold) + .by(_.observables.count) .by(entityRenderer) ) .domainMap { @@ -505,11 +501,11 @@ object AlertOps { traversal .project( _.by - .by(_.out[AlertOrganisation].v[Organisation].value(_.name).fold) - .by(_.out[AlertTag].v[Tag].fold) + .by(_.organisation.value(_.name).fold) + .by(_.tags.fold) .by(_.richCustomFields.fold) - .by(_.out[AlertCase]._id.fold) - .by(_.out[AlertCaseTemplate].v[CaseTemplate].value(_.name).fold) + .by(_.`case`._id.fold) + .by(_.caseTemplate.value(_.name).fold) .by(_.outE[AlertObservable].count) ) .domainMap { diff --git a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala index ddd39f5e75..cc3165c5a3 100644 --- a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala +++ b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala @@ -24,8 +24,8 @@ import scala.concurrent.Future import scala.util.Try @Singleton -class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: StorageSrv)( - implicit @Named("with-thehive-schema") db: Database, +class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: StorageSrv)(implicit + @Named("with-thehive-schema") db: Database, mat: Materializer ) extends VertexSrv[Attachment] { @@ -43,8 +43,8 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage result } - def create(filename: String, contentType: String, data: Array[Byte])( - implicit graph: Graph, + def create(filename: String, contentType: String, data: Array[Byte])(implicit + graph: Graph, authContext: AuthContext ): Try[Attachment with Entity] = { val hs = hashers.fromBinary(data) @@ -52,8 +52,8 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage storageSrv.saveBinary("attachment", id, data).flatMap(_ => createEntity(Attachment(filename, data.length.toLong, contentType, hs, id))) } - def create(filename: String, size: Long, contentType: String, data: Source[ByteString, NotUsed])( - implicit graph: Graph, + def create(filename: String, size: Long, contentType: String, data: Source[ByteString, NotUsed])(implicit + graph: Graph, authContext: AuthContext ): Try[Attachment with Entity] = { val hs = hashers.fromBinary(data) @@ -61,9 +61,8 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage storageSrv.saveBinary("attachment", id, data).flatMap(_ => createEntity(Attachment(filename, size, contentType, hs, id))) } - override def get(idOrAttachmentId: String)(implicit graph: Graph): Traversal.V[Attachment] = - if (db.isValidId(idOrAttachmentId)) getByIds(idOrAttachmentId) - else startTraversal.getByAttachmentId(idOrAttachmentId) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Attachment] = + startTraversal.getByAttachmentId(name) def source(attachment: Attachment with Entity): Source[ByteString, Future[IOResult]] = StreamConverters.fromInputStream(() => stream(attachment)) @@ -80,7 +79,7 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage object AttachmentOps { implicit class AttachmentOpsDefs(traversal: Traversal.V[Attachment]) { - def getByAttachmentId(attachmentId: String): Traversal.V[Attachment] = traversal.has("attachmentId", attachmentId) + def getByAttachmentId(attachmentId: String): Traversal.V[Attachment] = traversal.has(_.attachmentId, attachmentId) def visible(implicit authContext: AuthContext): Traversal.V[Attachment] = traversal // TODO diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index ae4a9d8765..f5fe61c8ff 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -13,8 +13,10 @@ import org.thp.scalligraph.models.{Entity, _} import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IdentityConverter, Traversal} +import org.thp.scalligraph.{EntityId, EntityIdOrName} import org.thp.thehive.models._ import org.thp.thehive.services.AuditOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.notification.AuditNotificationMessage import play.api.libs.json.{JsObject, JsValue, Json} @@ -29,32 +31,32 @@ class AuditSrv @Inject() ( eventSrv: EventSrv )(implicit @Named("with-thehive-schema") db: Database) extends VertexSrv[Audit] { auditSrv => - lazy val userSrv: UserSrv = userSrvProvider.get - val auditUserSrv = new EdgeSrv[AuditUser, Audit, User] - val auditedSrv = new EdgeSrv[Audited, Audit, Product] - val auditContextSrv = new EdgeSrv[AuditContext, Audit, Product] - val `case` = new SelfContextObjectAudit[Case] - val task = new SelfContextObjectAudit[Task] - val observable = new SelfContextObjectAudit[Observable] - val log = new ObjectAudit[Log, Task] - val caseTemplate = new SelfContextObjectAudit[CaseTemplate] - val taskInTemplate = new ObjectAudit[Task, CaseTemplate] - val alert = new SelfContextObjectAudit[Alert] - val alertToCase = new ObjectAudit[Alert, Case] - val share = new ShareAudit - val observableInAlert = new ObjectAudit[Observable, Alert] - val user = new UserAudit - val dashboard = new SelfContextObjectAudit[Dashboard] - val organisation = new SelfContextObjectAudit[Organisation] - val profile = new SelfContextObjectAudit[Profile] - val customField = new SelfContextObjectAudit[CustomField] - val page = new SelfContextObjectAudit[Page] - private val pendingAuditsLock = new Object - private val transactionAuditIdsLock = new Object - private val unauditedTransactionsLock = new Object - private var pendingAudits: Map[AnyRef, PendingAudit] = Map.empty - private var transactionAuditIds: List[(AnyRef, String)] = Nil - private var unauditedTransactions: Set[AnyRef] = Set.empty + lazy val userSrv: UserSrv = userSrvProvider.get + val auditUserSrv = new EdgeSrv[AuditUser, Audit, User] + val auditedSrv = new EdgeSrv[Audited, Audit, Product] + val auditContextSrv = new EdgeSrv[AuditContext, Audit, Product] + val `case` = new SelfContextObjectAudit[Case] + val task = new SelfContextObjectAudit[Task] + val observable = new SelfContextObjectAudit[Observable] + val log = new ObjectAudit[Log, Task] + val caseTemplate = new SelfContextObjectAudit[CaseTemplate] + val taskInTemplate = new ObjectAudit[Task, CaseTemplate] + val alert = new SelfContextObjectAudit[Alert] + val alertToCase = new ObjectAudit[Alert, Case] + val share = new ShareAudit + val observableInAlert = new ObjectAudit[Observable, Alert] + val user = new UserAudit + val dashboard = new SelfContextObjectAudit[Dashboard] + val organisation = new SelfContextObjectAudit[Organisation] + val profile = new SelfContextObjectAudit[Profile] + val customField = new SelfContextObjectAudit[CustomField] + val page = new SelfContextObjectAudit[Page] + private val pendingAuditsLock = new Object + private val transactionAuditIdsLock = new Object + private val unauditedTransactionsLock = new Object + private var pendingAudits: Map[AnyRef, PendingAudit] = Map.empty + private var transactionAuditIds: List[(AnyRef, EntityId)] = Nil + private var unauditedTransactions: Set[AnyRef] = Set.empty /** * Gets the main action Audits by ids sorted by date @@ -63,7 +65,7 @@ class AuditSrv @Inject() ( * @param graph db * @return */ - def getMainByIds(order: Order, ids: String*)(implicit graph: Graph): Traversal.V[Audit] = + def getMainByIds(order: Order, ids: EntityId*)(implicit graph: Graph): Traversal.V[Audit] = getByIds(ids: _*) .has(_.mainAction, true) .sort(_.by("_createdAt", order)) @@ -324,7 +326,7 @@ object AuditOps { // case otherwise // FIXME } - def forCase(caseId: String): Traversal.V[Audit] = traversal.filter(_.`case`.hasId(caseId)) +// def forCase(caseId: String): Traversal.V[Audit] = traversal.filter(_.`case`.hasId(caseId)) def `case`: Traversal.V[Case] = traversal @@ -345,7 +347,7 @@ object AuditOps { def visible(implicit authContext: AuthContext): Traversal.V[Audit] = visible(authContext.organisation) - def visible(organisationName: String): Traversal.V[Audit] = traversal.filter(_.organisation.has("name", organisationName)) + def visible(organisation: EntityIdOrName): Traversal.V[Audit] = traversal.filter(_.organisation.get(organisation)) def `object`: Traversal[Vertex, Vertex, IdentityConverter[Vertex]] = traversal.out[Audited] diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 856d2aff3d..69b4bd66a3 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -13,11 +13,12 @@ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} -import org.thp.scalligraph.{CreateError, RichOptionTry, RichSeq} +import org.thp.scalligraph.{CreateError, EntityIdOrName, EntityName, RichOptionTry, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CustomFieldOps._ +import org.thp.thehive.services.DataOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ @@ -79,7 +80,9 @@ class CaseSrv @Inject() ( caseTemplate .fold[Seq[RichCustomField]](Nil)(_.customFields) .map(cf => (cf.name, cf.value, cf.order)) - cfs <- (caseTemplateCustomFields ++ customFields).toTry { case (name, value, order) => createCustomField(createdCase, name, value, order) } + cfs <- (caseTemplateCustomFields ++ customFields).toTry { + case (name, value, order) => createCustomField(createdCase, EntityIdOrName(name), value, order) + } caseTemplateTags = caseTemplate.fold[Seq[Tag with Entity]](Nil)(_.tags) allTags = tags ++ caseTemplateTags _ <- allTags.toTry(t => caseTagSrv.create(CaseTag(), createdCase, t)) @@ -98,7 +101,7 @@ class CaseSrv @Inject() ( val closeCase = PropertyUpdater(FPathElem("closeCase"), "") { (vertex, _, _, _) => get(vertex) .tasks - .or(_.has("status", "Waiting"), _.has("status", "InProgress")) + .or(_.has(_.status, TaskStatus.Waiting), _.has(_.status, TaskStatus.InProgress)) .toIterator .toTry { case task if task.status == TaskStatus.InProgress => taskSrv.updateStatus(task, null, TaskStatus.Completed) @@ -163,7 +166,7 @@ class CaseSrv @Inject() ( .visible .`case` .hasId(`case`._id) - .exists || get(`case`).observables.filter(_.hasId(richObservable.observable._id)).exists + .exists || get(`case`).observables.filter(_.hasId(richObservable.observable._id)).exists // FIXME if (alreadyExistInThatCase) Failure(CreateError("Observable already exists")) else @@ -182,15 +185,11 @@ class CaseSrv @Inject() ( get(`case`).remove() } - override def get(idOrNumber: String)(implicit graph: Graph): Traversal.V[Case] = - Success(idOrNumber) - .filter(_.headOption.contains('#')) - .map(_.tail.toInt) - .map(startTraversal.getByNumber(_)) - .getOrElse(super.getByIds(idOrNumber)) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Case] = + Try(startTraversal.getByNumber(name.toInt)).getOrElse(startTraversal.limit(0)) - def getCustomField(`case`: Case with Entity, customFieldName: String)(implicit graph: Graph): Option[RichCustomField] = - get(`case`).customFields(customFieldName).richCustomField.headOption + def getCustomField(`case`: Case with Entity, customFieldIdOrName: EntityIdOrName)(implicit graph: Graph): Option[RichCustomField] = + get(`case`).customFields(customFieldIdOrName).richCustomField.headOption def updateCustomField( `case`: Case with Entity, @@ -201,31 +200,31 @@ class CaseSrv @Inject() ( .richCustomFields .toIterator .filterNot(rcf => customFieldNames.contains(rcf.name)) - .foreach(rcf => get(`case`).customFields(rcf.name).remove()) + .foreach(rcf => get(`case`).customFields(EntityName(rcf.name)).remove()) customFieldValues - .toTry { case (cf, v, o) => setOrCreateCustomField(`case`, cf.name, Some(v), o) } + .toTry { case (cf, v, o) => setOrCreateCustomField(`case`, EntityName(cf.name), Some(v), o) } .map(_ => ()) } - def setOrCreateCustomField(`case`: Case with Entity, customFieldName: String, value: Option[Any], order: Option[Int])(implicit + def setOrCreateCustomField(`case`: Case with Entity, customFieldIdOrName: EntityIdOrName, value: Option[Any], order: Option[Int])(implicit graph: Graph, authContext: AuthContext ): Try[Unit] = { - val cfv = get(`case`).customFields(customFieldName) + val cfv = get(`case`).customFields(customFieldIdOrName) if (cfv.clone().exists) cfv.setValue(value) else - createCustomField(`case`, customFieldName, value, order).map(_ => ()) + createCustomField(`case`, customFieldIdOrName, value, order).map(_ => ()) } def createCustomField( `case`: Case with Entity, - customFieldName: String, + customFieldIdOrName: EntityIdOrName, customFieldValue: Option[Any], order: Option[Int] )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { - cf <- customFieldSrv.getOrFail(customFieldName) + cf <- customFieldSrv.getOrFail(customFieldIdOrName) ccf <- CustomFieldType.map(cf.`type`).setValue(CaseCustomField(), customFieldValue).map(_.order_=(order)) ccfe <- caseCustomFieldSrv.create(ccf, `case`, cf) } yield RichCustomField(cf, ccfe) @@ -234,7 +233,7 @@ class CaseSrv @Inject() ( `case`: Case with Entity, impactStatus: String )(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - impactStatusSrv.getOrFail(impactStatus).flatMap(setImpactStatus(`case`, _)) + impactStatusSrv.getOrFail(EntityIdOrName(impactStatus)).flatMap(setImpactStatus(`case`, _)) def setImpactStatus( `case`: Case with Entity, @@ -254,7 +253,7 @@ class CaseSrv @Inject() ( `case`: Case with Entity, resolutionStatus: String )(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - resolutionStatusSrv.getOrFail(resolutionStatus).flatMap(setResolutionStatus(`case`, _)) + resolutionStatusSrv.getOrFail(EntityIdOrName(resolutionStatus)).flatMap(setResolutionStatus(`case`, _)) def setResolutionStatus( `case`: Case with Entity, @@ -342,30 +341,21 @@ object CaseOps { def resolutionStatus: Traversal.V[ResolutionStatus] = traversal.out[CaseResolutionStatus].v[ResolutionStatus] - def get(id: String): Traversal.V[Case] = - Success(id) - .filter(_.headOption.contains('#')) - .map(_.tail.toInt) - .map(getByNumber) - .getOrElse(traversal.getByIds(id)) + def get(idOrName: EntityIdOrName): Traversal.V[Case] = + idOrName.fold(traversal.getByIds(_), n => getByNumber(n.toInt)) - def getByNumber(caseNumber: Int): Traversal.V[Case] = traversal.has("number", caseNumber) + def getByNumber(caseNumber: Int): Traversal.V[Case] = traversal.has(_.number, caseNumber) def visible(implicit authContext: AuthContext): Traversal.V[Case] = visible(authContext.organisation) - def visible(organisationName: String): Traversal.V[Case] = - traversal.filter(_.in[ShareCase].in[OrganisationShare].has("name", organisationName)) + def visible(organisationIdOrName: EntityIdOrName): Traversal.V[Case] = + traversal.filter(_.organisations.get(organisationIdOrName)) def assignee: Traversal.V[User] = traversal.out[CaseUser].v[User] def can(permission: Permission)(implicit authContext: AuthContext): Traversal.V[Case] = if (authContext.permissions.contains(permission)) - traversal.filter( - _.in[ShareCase] - .filter(_.out[ShareProfile].has("permissions", permission)) - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.shares.filter(_.profile.has(_.permissions, permission)).organisation.current) else traversal.limit(0) @@ -399,16 +389,18 @@ object CaseOps { ) -> renderedEntity } - def customFields(name: String): Traversal.E[CaseCustomField] = - traversal.outE[CaseCustomField].filter(_.inV.has("name", name)).e[CaseCustomField] + def customFields(idOrName: EntityIdOrName): Traversal.E[CaseCustomField] = + idOrName + .fold( + id => traversal.outE[CaseCustomField].filter(_.inV.getByIds(id)), + name => traversal.outE[CaseCustomField].filter(_.inV.v[CustomField].has(_.name, name)) + ) - def customFields: Traversal.E[CaseCustomField] = - traversal.outE[CaseCustomField].e[CaseCustomField] + def customFields: Traversal.E[CaseCustomField] = traversal.outE[CaseCustomField] def richCustomFields: Traversal[RichCustomField, JMap[String, Any], Converter[RichCustomField, JMap[String, Any]]] = traversal .outE[CaseCustomField] - .e[CaseCustomField] .project(_.by.by(_.inV.v[CustomField])) .domainMap { case (cfv, cf) => RichCustomField(cf, cfv) @@ -416,15 +408,15 @@ object CaseOps { def share(implicit authContext: AuthContext): Traversal.V[Share] = share(authContext.organisation) - def share(organisationName: String): Traversal.V[Share] = - traversal.in[ShareCase].filter(_.in[OrganisationShare].has("name", organisationName)).v[Share] + def share(organisation: EntityIdOrName): Traversal.V[Share] = + shares.filter(_.organisation.get(organisation)).v[Share] def shares: Traversal.V[Share] = traversal.in[ShareCase].v[Share] def organisations: Traversal.V[Organisation] = traversal.in[ShareCase].in[OrganisationShare].v[Organisation] def organisations(permission: Permission): Traversal.V[Organisation] = - traversal.in[ShareCase].filter(_.out[ShareProfile].has("permissions", permission)).in[OrganisationShare].v[Organisation] + shares.filter(_.profile.has(_.permissions, permission)).organisation def userPermissions(implicit authContext: AuthContext): Traversal[Set[Permission], Vertex, Converter[Set[Permission], Vertex]] = traversal @@ -432,13 +424,13 @@ object CaseOps { .profile .domainMap(profile => profile.permissions & authContext.permissions) - def origin: Traversal.V[Organisation] = traversal.in[ShareCase].has("owner", true).in[OrganisationShare].v[Organisation] + def origin: Traversal.V[Organisation] = shares.has(_.owner, true).organisation def audits(implicit authContext: AuthContext): Traversal.V[Audit] = audits(authContext.organisation) - def audits(organisationName: String): Traversal.V[Audit] = + def audits(organisationIdOrName: EntityIdOrName): Traversal.V[Audit] = traversal - .unionFlat(_.visible(organisationName), _.observables(organisationName), _.tasks(organisationName), _.share(organisationName)) + .unionFlat(_.visible(organisationIdOrName), _.observables(organisationIdOrName), _.tasks(organisationIdOrName), _.share(organisationIdOrName)) .in[AuditContext] .v[Audit] @@ -463,18 +455,15 @@ object CaseOps { .as(originCaseLabel) .observables .as(observableLabel) - .out[ObservableData] - .in[ObservableData] - .in[ShareObservable] - .filter( - _.in[OrganisationShare] - .has("name", authContext.organisation) - ) - .out[ShareCase] + .data + .observables + .shares + .filter(_.organisation.current) + .`case` .where(P.neq(originCaseLabel.name)) .group(_.by, _.by(_.select(observableLabel).richObservable.fold)) .unfold - .project(_.by(_.selectKeys.v[Case].richCase).by(_.selectValues)) + .project(_.by(_.selectKeys.richCase).by(_.selectValues)) .toSeq } @@ -482,7 +471,7 @@ object CaseOps { traversal .project( _.by - .by(_.tags.v[Tag].fold) + .by(_.tags.fold) .by(_.impactStatus.value(_.value).fold) .by(_.resolutionStatus.value(_.value).fold) .by(_.assignee.value(_.login).fold) @@ -533,13 +522,13 @@ object CaseOps { def tasks(implicit authContext: AuthContext): Traversal.V[Task] = tasks(authContext.organisation) - def tasks(organisationName: String): Traversal.V[Task] = - share(organisationName).tasks + def tasks(organisationIdOrName: EntityIdOrName): Traversal.V[Task] = + share(organisationIdOrName).tasks def observables(implicit authContext: AuthContext): Traversal.V[Observable] = observables(authContext.organisation) - def observables(organisationName: String): Traversal.V[Observable] = - share(organisationName).observables + def observables(organisationIdOrName: EntityIdOrName): Traversal.V[Observable] = + share(organisationIdOrName).observables def assignableUsers(implicit authContext: AuthContext): Traversal.V[User] = organisations(Permissions.manageCase) diff --git a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala index 956c48942a..22e4b17ad5 100644 --- a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala @@ -12,7 +12,7 @@ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} -import org.thp.scalligraph.{CreateError, RichSeq} +import org.thp.scalligraph.{CreateError, EntityIdOrName, EntityName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.CaseTemplateOps._ @@ -38,9 +38,8 @@ class CaseTemplateSrv @Inject() ( val caseTemplateOrganisationSrv = new EdgeSrv[CaseTemplateOrganisation, CaseTemplate, Organisation] val caseTemplateTaskSrv = new EdgeSrv[CaseTemplateTask, CaseTemplate, Task] - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[CaseTemplate] = - if (db.isValidId(idOrName)) super.getByIds(idOrName) - else startTraversal.getByName(idOrName) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[CaseTemplate] = + startTraversal.getByName(name) override def createEntity(e: CaseTemplate)(implicit graph: Graph, authContext: AuthContext): Try[CaseTemplate with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("CaseTemplate") @@ -53,8 +52,8 @@ class CaseTemplateSrv @Inject() ( tagNames: Set[String], tasks: Seq[(Task, Option[User with Entity])], customFields: Seq[(String, Option[Any])] - )( - implicit graph: Graph, + )(implicit + graph: Graph, authContext: AuthContext ): Try[RichCaseTemplate] = tagNames.toTry(tagSrv.getOrCreate).flatMap(tags => create(caseTemplate, organisation, tags, tasks, customFields)) @@ -64,11 +63,11 @@ class CaseTemplateSrv @Inject() ( tags: Seq[Tag with Entity], tasks: Seq[(Task, Option[User with Entity])], customFields: Seq[(String, Option[Any])] - )( - implicit graph: Graph, + )(implicit + graph: Graph, authContext: AuthContext ): Try[RichCaseTemplate] = - if (organisationSrv.get(organisation).caseTemplates.has("name", P.eq[String](caseTemplate.name)).exists) + if (organisationSrv.get(organisation).caseTemplates.get(EntityName(caseTemplate.name)).exists) Failure(CreateError(s"""The case template "${caseTemplate.name}" already exists""")) else for { @@ -82,8 +81,8 @@ class CaseTemplateSrv @Inject() ( _ <- auditSrv.caseTemplate.create(createdCaseTemplate, richCaseTemplate.toJson) } yield richCaseTemplate - def addTask(caseTemplate: CaseTemplate with Entity, task: Task with Entity)( - implicit graph: Graph, + def addTask(caseTemplate: CaseTemplate with Entity, task: Task with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = for { @@ -154,8 +153,8 @@ class CaseTemplateSrv @Inject() ( .map(_ => ()) } - def setOrCreateCustomField(caseTemplate: CaseTemplate with Entity, customFieldName: String, value: Option[Any], order: Option[Int])( - implicit graph: Graph, + def setOrCreateCustomField(caseTemplate: CaseTemplate with Entity, customFieldName: String, value: Option[Any], order: Option[Int])(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = { val cfv = get(caseTemplate).customFields(customFieldName) @@ -172,7 +171,7 @@ class CaseTemplateSrv @Inject() ( order: Option[Int] )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { - cf <- customFieldSrv.getOrFail(customFieldName) + cf <- customFieldSrv.getOrFail(EntityIdOrName(customFieldName)) ccf <- CustomFieldType.map(cf.`type`).setValue(CaseTemplateCustomField(order = order), customFieldValue) ccfe <- caseTemplateCustomFieldSrv.create(ccf, caseTemplate, cf) } yield RichCustomField(cf, ccfe) @@ -181,21 +180,17 @@ class CaseTemplateSrv @Inject() ( object CaseTemplateOps { implicit class CaseTemplateOpsDefs(traversal: Traversal.V[CaseTemplate]) { - def get(idOrName: String)(implicit db: Database): Traversal.V[CaseTemplate] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) + def get(idOrName: EntityIdOrName): Traversal.V[CaseTemplate] = + idOrName.fold(traversal.getByIds(_), getByName) - def getByName(name: String): Traversal.V[CaseTemplate] = traversal.has("name", name) + def getByName(name: String): Traversal.V[CaseTemplate] = traversal.has(_.name, name) def visible(implicit authContext: AuthContext): Traversal.V[CaseTemplate] = - traversal.filter(_.out[CaseTemplateOrganisation].has("name", authContext.organisation)) + traversal.filter(_.organisation.current) def can(permission: Permission)(implicit authContext: AuthContext): Traversal.V[CaseTemplate] = if (authContext.permissions.contains(permission)) - traversal.filter( - _.out[CaseTemplateOrganisation] - .has("name", authContext.organisation) - ) + traversal.filter(_.organisation.current) else traversal.limit(0) @@ -205,12 +200,11 @@ object CaseTemplateOps { traversal .project( _.by - .by(_.out[CaseTemplateOrganisation].v[Organisation].value(_.name)) - .by(_.out[CaseTemplateTag].v[Tag].fold) - .by(_.out[CaseTemplateTask].v[Task].richTask.fold) + .by(_.organisation.value(_.name)) + .by(_.tags.fold) + .by(_.tasks.richTask.fold) .by( _.outE[CaseTemplateCustomField] - .e[CaseTemplateCustomField] .as(caseTemplateCustomFieldLabel) .inV .v[CustomField] @@ -231,9 +225,9 @@ object CaseTemplateOps { } } - def organisation = traversal.out[CaseTemplateOrganisation].v[Organisation] + def organisation: Traversal.V[Organisation] = traversal.out[CaseTemplateOrganisation].v[Organisation] - def tasks = traversal.out[CaseTemplateTask].v[Task] + def tasks: Traversal.V[Task] = traversal.out[CaseTemplateTask].v[Task] def tags: Traversal.V[Tag] = traversal.out[CaseTemplateTag].v[Tag] @@ -242,10 +236,10 @@ object CaseTemplateOps { traversal.outE[CaseTemplateTag].filter(_.inV.hasId(tags.map(_._id).toSeq: _*)).remove() def customFields(name: String): Traversal.E[CaseTemplateCustomField] = - traversal.outE[CaseTemplateCustomField].filter(_.inV.has("name", name)).e[CaseTemplateCustomField] + traversal.outE[CaseTemplateCustomField].filter(_.inV.v[CustomField].has(_.name, name)) def customFields: Traversal.E[CaseTemplateCustomField] = - traversal.outE[CaseTemplateCustomField].e[CaseTemplateCustomField] + traversal.outE[CaseTemplateCustomField] } implicit class CaseTemplateCustomFieldsOpsDefs(traversal: Traversal.E[CaseTemplateCustomField]) extends CustomFieldValueOpsDefs(traversal) @@ -273,11 +267,12 @@ class CaseTemplateIntegrityCheckOps @Inject() ( .toSeq } - override def resolve(entities: Seq[CaseTemplate with Entity])(implicit graph: Graph): Try[Unit] = entities match { - case head :: tail => - tail.foreach(copyEdge(_, head, e => e.label() == "CaseCaseTemplate" || e.label() == "AlertCaseTemplate")) - service.getByIds(tail.map(_._id): _*).remove() - Success(()) - case _ => Success(()) - } + override def resolve(entities: Seq[CaseTemplate with Entity])(implicit graph: Graph): Try[Unit] = + entities match { + case head :: tail => + tail.foreach(copyEdge(_, head, e => e.label() == "CaseCaseTemplate" || e.label() == "AlertCaseTemplate")) + service.getByIds(tail.map(_._id): _*).remove() + Success(()) + case _ => Success(()) + } } diff --git a/thehive/app/org/thp/thehive/services/ConfigContext.scala b/thehive/app/org/thp/thehive/services/ConfigContext.scala index 16b5a9c29d..7ef17ea550 100644 --- a/thehive/app/org/thp/thehive/services/ConfigContext.scala +++ b/thehive/app/org/thp/thehive/services/ConfigContext.scala @@ -1,6 +1,7 @@ package org.thp.thehive.services import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.config.ConfigContext @@ -16,7 +17,7 @@ class UserConfigContext @Inject() (@Named("with-thehive-schema") db: Database, c db.roTransaction { implicit graph => configSrv .user - .getConfigValue(context.userId, path) + .getConfigValue(EntityName(context.userId), path) .orElse( configSrv .organisation @@ -29,7 +30,7 @@ class UserConfigContext @Inject() (@Named("with-thehive-schema") db: Database, c db.tryTransaction(graph => configSrv .user - .setConfigValue(context.userId, path, value)(graph, context) + .setConfigValue(EntityName(context.userId), path, value)(graph, context) .map(_ => s"user.${context.userId}.$path") ) } @@ -46,7 +47,7 @@ class OrganisationConfigContext @Inject() (@Named("with-thehive-schema") db: Dat .orElse( configSrv .organisation - .getConfigValue("defaults", path) + .getConfigValue(EntityName("defaults"), path) ) .map(_.value) } diff --git a/thehive/app/org/thp/thehive/services/ConfigSrv.scala b/thehive/app/org/thp/thehive/services/ConfigSrv.scala index 7a9a394d64..f4d7683165 100644 --- a/thehive/app/org/thp/thehive/services/ConfigSrv.scala +++ b/thehive/app/org/thp/thehive/services/ConfigSrv.scala @@ -1,13 +1,13 @@ package org.thp.thehive.services import javax.inject.{Inject, Named, Singleton} -import org.apache.tinkerpop.gremlin.process.traversal.P -import org.apache.tinkerpop.gremlin.structure.{Graph, Vertex} +import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} +import org.thp.scalligraph.{EntityId, EntityIdOrName} import org.thp.thehive.models._ import org.thp.thehive.services.ConfigOps._ import org.thp.thehive.services.OrganisationOps._ @@ -27,12 +27,12 @@ class ConfigSrv @Inject() ( val organisationConfigSrv = new EdgeSrv[OrganisationConfig, Organisation, Config] val userConfigSrv = new EdgeSrv[UserConfig, User, Config] - def triggerMap(notificationSrv: NotificationSrv)(implicit graph: Graph): Map[String, Map[Trigger, (Boolean, Seq[String])]] = + def triggerMap(notificationSrv: NotificationSrv)(implicit graph: Graph): Map[EntityId, Map[Trigger, (Boolean, Seq[EntityId])]] = startTraversal.triggerMap(notificationSrv) object organisation { - def setConfigValue(organisationName: String, name: String, value: JsValue)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def setConfigValue(organisationName: EntityIdOrName, name: String, value: JsValue)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = getConfigValue(organisationName, name) match { case Some(config) => get(config).update(_.value, value).domainMap(_ => ()).getOrFail("Config") case None => @@ -43,17 +43,17 @@ class ConfigSrv @Inject() ( } yield () } - def getConfigValue(organisationName: String, name: String)(implicit graph: Graph): Option[Config with Entity] = + def getConfigValue(organisationName: EntityIdOrName, name: String)(implicit graph: Graph): Option[Config with Entity] = organisationSrv .get(organisationName) .config - .has("name", name) + .has(_.name, name) .headOption } object user { - def setConfigValue(userName: String, name: String, value: JsValue)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def setConfigValue(userName: EntityIdOrName, name: String, value: JsValue)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = getConfigValue(userName, name) match { case Some(config) => get(config).update(_.value, value).domainMap(_ => ()).getOrFail("Config") case None => @@ -64,7 +64,7 @@ class ConfigSrv @Inject() ( } yield () } - def getConfigValue(userName: String, name: String)(implicit graph: Graph): Option[Config with Entity] = + def getConfigValue(userName: EntityIdOrName, name: String)(implicit graph: Graph): Option[Config with Entity] = userSrv .get(userName) .config @@ -76,49 +76,51 @@ class ConfigSrv @Inject() ( object ConfigOps { implicit class ConfigOpsDefs(traversal: Traversal.V[Config]) { - def triggerMap(notificationSrv: NotificationSrv): Map[String, Map[Trigger, (Boolean, Seq[String])]] = { + def triggerMap(notificationSrv: NotificationSrv): Map[EntityId, Map[Trigger, (Boolean, Seq[EntityId])]] = { // Traversal of configuration version of type "notification" - def notificationRaw: Traversal[Config with Entity, Vertex, Converter[Config with Entity, Vertex]] = + def notificationRaw: Traversal.V[Config] = traversal .clone() .has(_.name, "notification") // Retrieve triggers configured for each organisation - val organisationTriggers: Iterator[(String, Trigger, Option[String])] = { + val organisationTriggers: Iterator[(EntityId, Trigger, Option[EntityId])] = { val configLabel = StepLabel.v[Config] - val organisationIdLabel = StepLabel[String, AnyRef, Converter[String, AnyRef]] + val organisationIdLabel = StepLabel[EntityId, AnyRef, Converter[EntityId, AnyRef]] for { - (notifConfig, orgId) <- notificationRaw - .as(configLabel) - .in[OrganisationConfig] - ._id - .as(organisationIdLabel) - .select((configLabel, organisationIdLabel)) - .toIterator + (notifConfig, orgId) <- + notificationRaw + .as(configLabel) + .in[OrganisationConfig] + ._id + .as(organisationIdLabel) + .select((configLabel, organisationIdLabel)) + .toIterator // cfg <- notificationSrv.getConfig(notifConfig.value[String]("value")) // trigger <- notificationSrv.getTrigger(cfg.triggerConfig).toOption trigger <- notificationSrv.getTriggers(notifConfig.value) - } yield (orgId, trigger, None: Option[String]) + } yield (orgId, trigger, None: Option[EntityId]) } // Retrieve triggers configured for each user - val userTriggers: Iterator[(String, Trigger, Option[String])] = { + val userTriggers: Iterator[(EntityId, Trigger, Option[EntityId])] = { val configLabel = StepLabel.v[Config] val userLabel = StepLabel.v[User] - val organisationIdLabel = StepLabel[String, AnyRef, Converter[String, AnyRef]] + val organisationIdLabel = StepLabel[EntityId, AnyRef, Converter[EntityId, AnyRef]] for { - (notifConfig, user, orgId) <- notificationRaw - .as(configLabel) - .in[UserConfig] - .v[User] - .as(userLabel) - .out[UserRole] - .out[RoleOrganisation] - ._id - .as(organisationIdLabel) - .select((configLabel, userLabel, organisationIdLabel)) - .toIterator + (notifConfig, user, orgId) <- + notificationRaw + .as(configLabel) + .in[UserConfig] + .v[User] + .as(userLabel) + .out[UserRole] + .out[RoleOrganisation] + ._id + .as(organisationIdLabel) + .select((configLabel, userLabel, organisationIdLabel)) + .toIterator trigger <- notificationSrv.getTriggers(notifConfig.value) } yield (orgId, trigger, Some(user._id)) } diff --git a/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala b/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala index bae0496762..7a1bffbd75 100644 --- a/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala +++ b/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala @@ -5,13 +5,13 @@ import java.util.{Map => JMap} import akka.actor.ActorRef import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.{Edge, Graph} -import org.thp.scalligraph.RichSeq import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services.{IntegrityCheckOps, VertexSrv} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal._ +import org.thp.scalligraph.{EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.CustomFieldOps._ @@ -63,19 +63,17 @@ class CustomFieldSrv @Inject() (auditSrv: AuditSrv, organisationSrv: Organisatio .flatMap(auditSrv.customField.update(_, updatedFields)) } - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[CustomField] = - if (db.isValidId(idOrName)) super.getByIds(idOrName) - else startTraversal.getByName(idOrName) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[CustomField] = + startTraversal.getByName(name) } object CustomFieldOps { implicit class CustomFieldOpsDefs(traversal: Traversal.V[CustomField]) { - def get(idOrName: String)(implicit db: Database): Traversal.V[CustomField] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) + def get(idOrName: EntityIdOrName): Traversal.V[CustomField] = + idOrName.fold(traversal.getByIds(_), getByName) - def getByName(name: String): Traversal.V[CustomField] = traversal.has("name", name) + def getByName(name: String): Traversal.V[CustomField] = traversal.has(_.name, name) } implicit class CustomFieldValueOpsDefs[C <: CustomFieldValue[_]](traversal: Traversal.E[C]) { @@ -178,11 +176,12 @@ object CustomFieldOps { class CustomFieldIntegrityCheckOps @Inject() (@Named("with-thehive-schema") val db: Database, val service: CustomFieldSrv) extends IntegrityCheckOps[CustomField] { - override def resolve(entities: Seq[CustomField with Entity])(implicit graph: Graph): Try[Unit] = entities match { - case head :: tail => - tail.foreach(copyEdge(_, head)) - service.getByIds(tail.map(_._id): _*).remove() - Success(()) - case _ => Success(()) - } + override def resolve(entities: Seq[CustomField with Entity])(implicit graph: Graph): Try[Unit] = + entities match { + case head :: tail => + tail.foreach(copyEdge(_, head)) + service.getByIds(tail.map(_._id): _*).remove() + Success(()) + case _ => Success(()) + } } diff --git a/thehive/app/org/thp/thehive/services/DashboardSrv.scala b/thehive/app/org/thp/thehive/services/DashboardSrv.scala index 01952eff21..c799d8c106 100644 --- a/thehive/app/org/thp/thehive/services/DashboardSrv.scala +++ b/thehive/app/org/thp/thehive/services/DashboardSrv.scala @@ -4,6 +4,7 @@ import java.util.{List => JList, Map => JMap} import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.PropertyUpdater @@ -12,14 +13,15 @@ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.UserOps._ import play.api.libs.json.{JsObject, Json} import scala.util.{Success, Try} @Singleton -class DashboardSrv @Inject() (organisationSrv: OrganisationSrv, userSrv: UserSrv, auditSrv: AuditSrv)( - implicit @Named("with-thehive-schema") db: Database +class DashboardSrv @Inject() (organisationSrv: OrganisationSrv, userSrv: UserSrv, auditSrv: AuditSrv)(implicit + @Named("with-thehive-schema") db: Database ) extends VertexSrv[Dashboard] { val organisationDashboardSrv = new EdgeSrv[OrganisationDashboard, Organisation, Dashboard] val dashboardUserSrv = new EdgeSrv[DashboardUser, Dashboard, User] @@ -44,30 +46,32 @@ class DashboardSrv @Inject() (organisationSrv: OrganisationSrv, userSrv: UserSrv .flatMap(auditSrv.dashboard.update(_, updatedFields)) } - def share(dashboard: Dashboard with Entity, organisationName: String, writable: Boolean)( - implicit graph: Graph, + def share(dashboard: Dashboard with Entity, organisation: EntityIdOrName, writable: Boolean)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = - get(dashboard) - .inE[OrganisationDashboard] - .e[OrganisationDashboard] - .filter(_.outV.has("name", organisationName)) - .update(_.writable, writable) - .fold - .getOrFail("Dashboard") - .flatMap { - case d if d.isEmpty => - organisationSrv - .getOrFail(organisationName) - .flatMap(organisation => organisationDashboardSrv.create(OrganisationDashboard(writable), organisation, dashboard)) - case _ => Success(()) - } - .flatMap { _ => - auditSrv.dashboard.update(dashboard, Json.obj("share" -> Json.obj("organisation" -> organisationName, "writable" -> writable))) - } - - def unshare(dashboard: Dashboard with Entity, organisationName: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { - get(dashboard).inE[OrganisationDashboard].filter(_.outV.has("name", organisationName)).remove() + organisationSrv.get(organisation).getOrFail("Organisation").flatMap { org => + get(dashboard) + .inE[OrganisationDashboard] + .filter(_.outV.v[Organisation].getEntity(org)) + .update(_.writable, writable) + .fold + .getOrFail("Dashboard") + .flatMap { + case d if d.isEmpty => + organisationSrv + .get(organisation) + .getOrFail("Organisation") + .flatMap(organisation => organisationDashboardSrv.create(OrganisationDashboard(writable), organisation, dashboard)) + case _ => Success(()) + } + .flatMap { _ => + auditSrv.dashboard.update(dashboard, Json.obj("share" -> Json.obj("organisation" -> org.name, "writable" -> writable))) + } + } + + def unshare(dashboard: Dashboard with Entity, organisation: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + get(dashboard).inE[OrganisationDashboard].filter(_.outV.v[Organisation].get(organisation)).remove() Success(()) // TODO add audit } @@ -82,8 +86,11 @@ object DashboardOps { implicit class DashboardOpsDefs(traversal: Traversal.V[Dashboard]) { + def get(idOrName: EntityIdOrName): Traversal.V[Dashboard] = + idOrName.fold(traversal.getByIds(_), _ => traversal.limit(0)) + def visible(implicit authContext: AuthContext): Traversal.V[Dashboard] = - traversal.filter(_.or(_.user.current(authContext), _.in[OrganisationDashboard].has("name", authContext.organisation))) + traversal.filter(_.or(_.user.current, _.organisation.current)) def organisation: Traversal.V[Organisation] = traversal.in[OrganisationDashboard].v[Organisation] @@ -91,13 +98,12 @@ object DashboardOps { def canUpdate(implicit authContext: AuthContext): Traversal.V[Dashboard] = traversal.filter( - _.or(_.user.current(authContext), _.inE[OrganisationDashboard].has("writable", true).outV.has("name", authContext.organisation)) + _.or(_.user.current, _.inE[OrganisationDashboard].has(_.writable, true).outV.v[Organisation].current) ) def organisationShares: Traversal[Seq[(String, Boolean)], JList[JMap[String, Any]], Converter[Seq[(String, Boolean)], JList[JMap[String, Any]]]] = traversal .outE[OrganisationDashboard] - .e[OrganisationDashboard] .project( _.byValue(_.writable) .by(_.inV) diff --git a/thehive/app/org/thp/thehive/services/DataSrv.scala b/thehive/app/org/thp/thehive/services/DataSrv.scala index b6fc1ee2da..b1a75c3a4d 100644 --- a/thehive/app/org/thp/thehive/services/DataSrv.scala +++ b/thehive/app/org/thp/thehive/services/DataSrv.scala @@ -37,7 +37,7 @@ class DataSrv @Inject() (@Named("integrity-check-actor") integrityCheckActor: Ac object DataOps { implicit class DataOpsDefs(traversal: Traversal.V[Data]) { - def observables = traversal.in[ObservableData].v[Observable] + def observables: Traversal.V[Observable] = traversal.in[ObservableData].v[Observable] def notShared(caseId: String): Traversal.V[Data] = traversal.filter( @@ -49,7 +49,7 @@ object DataOps { .is(P.eq(0)) ) - def getByData(data: String): Traversal.V[Data] = traversal.has("data", data) + def getByData(data: String): Traversal.V[Data] = traversal.has(_.data, data) def useCount: Traversal[Long, JLong, Converter[Long, JLong]] = traversal.in[ObservableData].count } @@ -57,11 +57,12 @@ object DataOps { } class DataIntegrityCheckOps @Inject() (@Named("with-thehive-schema") val db: Database, val service: DataSrv) extends IntegrityCheckOps[Data] { - override def resolve(entities: Seq[Data with Entity])(implicit graph: Graph): Try[Unit] = entities match { - case head :: tail => - tail.foreach(copyEdge(_, head)) - service.getByIds(tail.map(_._id): _*).remove() - Success(()) - case _ => Success(()) - } + override def resolve(entities: Seq[Data with Entity])(implicit graph: Graph): Try[Unit] = + entities match { + case head :: tail => + tail.foreach(copyEdge(_, head)) + service.getByIds(tail.map(_._id): _*).remove() + Success(()) + case _ => Success(()) + } } diff --git a/thehive/app/org/thp/thehive/services/FlowActor.scala b/thehive/app/org/thp/thehive/services/FlowActor.scala index 9b6e258bf3..593a5956b3 100644 --- a/thehive/app/org/thp/thehive/services/FlowActor.scala +++ b/thehive/app/org/thp/thehive/services/FlowActor.scala @@ -9,16 +9,17 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.EventSrv import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{EntityId, EntityIdOrName} import org.thp.thehive.GuiceAkkaExtension import org.thp.thehive.services.AuditOps._ import org.thp.thehive.services.CaseOps._ import play.api.cache.SyncCacheApi object FlowActor { - case class FlowId(organisation: String, caseId: Option[String]) { + case class FlowId(organisation: EntityIdOrName, caseId: Option[EntityIdOrName]) { override def toString: String = s"$organisation;${caseId.getOrElse("-")}" } - case class AuditIds(ids: Seq[String]) + case class AuditIds(ids: Seq[EntityId]) } class FlowActor extends Actor { @@ -37,7 +38,7 @@ class FlowActor extends Actor { val auditIds = cache.getOrElseUpdate(flowId.toString) { db.roTransaction { implicit graph => caseId - .fold(auditSrv.startTraversal.has("mainAction", true).visible(organisation))(caseSrv.getByIds(_).audits(organisation)) + .fold(auditSrv.startTraversal.has(_.mainAction, true).visible(organisation))(caseSrv.get(_).audits(organisation)) .sort(_.by("_createdAt", Order.desc)) .range(0, 10) ._id @@ -49,10 +50,10 @@ class FlowActor extends Actor { db.roTransaction { implicit graph => auditSrv .getByIds(ids: _*) - .has("mainAction", true) + .has(_.mainAction, true) .project( _.by(_._id) - .by(_.organisation.value(_.name).fold) + .by(_.organisation._id.fold) .by(_.`case`._id.fold) ) .toIterator diff --git a/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala b/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala index 42d766857e..490ad61f6b 100644 --- a/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala +++ b/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala @@ -3,25 +3,24 @@ package org.thp.thehive.services import akka.actor.ActorRef import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.CreateError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{IntegrityCheckOps, VertexSrv} import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{CreateError, EntityIdOrName} import org.thp.thehive.models.ImpactStatus import org.thp.thehive.services.ImpactStatusOps._ import scala.util.{Failure, Success, Try} @Singleton -class ImpactStatusSrv @Inject() (@Named("integrity-check-actor") integrityCheckActor: ActorRef)( - implicit @Named("with-thehive-schema") db: Database +class ImpactStatusSrv @Inject() (@Named("integrity-check-actor") integrityCheckActor: ActorRef)(implicit + @Named("with-thehive-schema") db: Database ) extends VertexSrv[ImpactStatus] { - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[ImpactStatus] = - if (db.isValidId(idOrName)) getByIds(idOrName) - else startTraversal.getByName(idOrName) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[ImpactStatus] = + startTraversal.getByName(name) override def createEntity(e: ImpactStatus)(implicit graph: Graph, authContext: AuthContext): Try[ImpactStatus with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("ImpactStatus") @@ -39,21 +38,21 @@ class ImpactStatusSrv @Inject() (@Named("integrity-check-actor") integrityCheckA object ImpactStatusOps { implicit class ImpactStatusOpsDefs(traversal: Traversal.V[ImpactStatus]) { - def get(idOrName: String)(implicit db: Database): Traversal.V[ImpactStatus] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) + def get(idOrName: EntityIdOrName): Traversal.V[ImpactStatus] = + idOrName.fold(traversal.getByIds(_), getByName) - def getByName(name: String): Traversal.V[ImpactStatus] = traversal.has("value", name).v[ImpactStatus] + def getByName(name: String): Traversal.V[ImpactStatus] = traversal.has(_.value, name) } } class ImpactStatusIntegrityCheckOps @Inject() (@Named("with-thehive-schema") val db: Database, val service: ImpactStatusSrv) extends IntegrityCheckOps[ImpactStatus] { - override def resolve(entities: Seq[ImpactStatus with Entity])(implicit graph: Graph): Try[Unit] = entities match { - case head :: tail => - tail.foreach(copyEdge(_, head)) - service.getByIds(tail.map(_._id): _*).remove() - Success(()) - case _ => Success(()) - } + override def resolve(entities: Seq[ImpactStatus with Entity])(implicit graph: Graph): Try[Unit] = + entities match { + case head :: tail => + tail.foreach(copyEdge(_, head)) + service.getByIds(tail.map(_._id): _*).remove() + Success(()) + case _ => Success(()) + } } diff --git a/thehive/app/org/thp/thehive/services/LocalKeyAuthSrv.scala b/thehive/app/org/thp/thehive/services/LocalKeyAuthSrv.scala index 621393cd0b..ca3c63febd 100644 --- a/thehive/app/org/thp/thehive/services/LocalKeyAuthSrv.scala +++ b/thehive/app/org/thp/thehive/services/LocalKeyAuthSrv.scala @@ -3,10 +3,10 @@ package org.thp.thehive.services import java.util.Base64 import javax.inject.{Inject, Named, Provider, Singleton} -import org.thp.scalligraph.NotFoundError import org.thp.scalligraph.auth._ import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{EntityIdOrName, NotFoundError} import org.thp.thehive.services.UserOps._ import play.api.Configuration import play.api.mvc.RequestHeader @@ -31,7 +31,7 @@ class LocalKeyAuthSrv( override val capabilities: Set[AuthCapability.Value] = Set(AuthCapability.authByKey) - override def authenticate(key: String, organisation: Option[String])(implicit request: RequestHeader): Try[AuthContext] = + override def authenticate(key: String, organisation: Option[EntityIdOrName])(implicit request: RequestHeader): Try[AuthContext] = db.roTransaction { implicit graph => userSrv .startTraversal @@ -44,7 +44,7 @@ class LocalKeyAuthSrv( db.tryTransaction { implicit graph => val newKey = generateKey() userSrv - .get(username) + .get(EntityIdOrName(username)) .update(_.apikey, Some(newKey)) .domainMap(_ => newKey) .getOrFail("User") @@ -53,14 +53,14 @@ class LocalKeyAuthSrv( override def getKey(username: String)(implicit authContext: AuthContext): Try[String] = db.roTransaction { implicit graph => userSrv - .getOrFail(username) + .getOrFail(EntityIdOrName(username)) .flatMap(_.apikey.fold[Try[String]](Failure(NotFoundError(s"User $username hasn't key")))(Success.apply)) } override def removeKey(username: String)(implicit authContext: AuthContext): Try[Unit] = db.tryTransaction { implicit graph => userSrv - .get(username) + .get(EntityIdOrName(username)) .update(_.apikey, None) .domainMap(_ => ()) .getOrFail("User") diff --git a/thehive/app/org/thp/thehive/services/LocalPasswordAuthSrv.scala b/thehive/app/org/thp/thehive/services/LocalPasswordAuthSrv.scala index 52b8ce2397..adaec1a90b 100644 --- a/thehive/app/org/thp/thehive/services/LocalPasswordAuthSrv.scala +++ b/thehive/app/org/thp/thehive/services/LocalPasswordAuthSrv.scala @@ -6,7 +6,7 @@ import org.thp.scalligraph.auth.{AuthCapability, AuthContext, AuthSrv, AuthSrvPr import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.utils.Hasher -import org.thp.scalligraph.{AuthenticationError, AuthorizationError} +import org.thp.scalligraph.{AuthenticationError, AuthorizationError, EntityIdOrName} import org.thp.thehive.models.User import play.api.mvc.RequestHeader import play.api.{Configuration, Logger} @@ -39,30 +39,28 @@ class LocalPasswordAuthSrv(@Named("with-thehive-schema") db: Database, userSrv: def isValidPassword(user: User, password: String): Boolean = user.password.fold(false)(hash => SecureHash.validatePassword(password, hash) || isValidPasswordLegacy(hash, password)) - override def authenticate(username: String, password: String, organisation: Option[String], code: Option[String])( - implicit request: RequestHeader + override def authenticate(username: String, password: String, organisation: Option[EntityIdOrName], code: Option[String])(implicit + request: RequestHeader ): Try[AuthContext] = db.roTransaction { implicit graph => - userSrv - .getOrFail(username) - } - .filter(user => isValidPassword(user, password)) + userSrv + .getOrFail(EntityIdOrName(username)) + }.filter(user => isValidPassword(user, password)) .map(user => localUserSrv.getAuthContext(request, user.login, organisation)) .getOrElse(Failure(AuthenticationError("Authentication failure"))) override def changePassword(username: String, oldPassword: String, newPassword: String)(implicit authContext: AuthContext): Try[Unit] = db.roTransaction { implicit graph => - userSrv - .getOrFail(username) - } - .filter(user => isValidPassword(user, oldPassword)) + userSrv + .getOrFail(EntityIdOrName(username)) + }.filter(user => isValidPassword(user, oldPassword)) .map(_ => setPassword(username, newPassword)) .getOrElse(Failure(AuthorizationError("Authentication failure"))) override def setPassword(username: String, newPassword: String)(implicit authContext: AuthContext): Try[Unit] = db.tryTransaction { implicit graph => userSrv - .get(username) + .get(EntityIdOrName(username)) .update(_.password, Some(hashPassword(newPassword))) .getOrFail("User") .map(_ => ()) diff --git a/thehive/app/org/thp/thehive/services/LocalUserSrv.scala b/thehive/app/org/thp/thehive/services/LocalUserSrv.scala index 336c78a724..356879d506 100644 --- a/thehive/app/org/thp/thehive/services/LocalUserSrv.scala +++ b/thehive/app/org/thp/thehive/services/LocalUserSrv.scala @@ -5,7 +5,7 @@ import org.thp.scalligraph.auth.{AuthContext, AuthContextImpl, User => Scalligra import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.utils.Instance -import org.thp.scalligraph.{AuthenticationError, CreateError, NotFoundError} +import org.thp.scalligraph.{AuthenticationError, CreateError, EntityIdOrName, EntityName, NotFoundError} import org.thp.thehive.models.{Organisation, Permissions, Profile, User} import org.thp.thehive.services.UserOps._ import play.api.Configuration @@ -23,20 +23,20 @@ class LocalUserSrv @Inject() ( configuration: Configuration ) extends ScalligraphUserSrv { - override def getAuthContext(request: RequestHeader, userId: String, organisationName: Option[String]): Try[AuthContext] = + override def getAuthContext(request: RequestHeader, userId: String, organisationName: Option[EntityIdOrName]): Try[AuthContext] = db.roTransaction { implicit graph => val requestId = Instance.getRequestId(request) - val userSteps = userSrv.get(userId) + val users = userSrv.get(EntityIdOrName(userId)) - if (userSteps.clone().exists) - userSteps + if (users.clone().exists) + users .clone() .getAuthContext(requestId, organisationName) .headOption .orElse { organisationName.flatMap { org => - userSteps - .getAuthContext(requestId, Organisation.administration.name) + users + .getAuthContext(requestId, EntityIdOrName(Organisation.administration.name)) .headOption .map(authContext => authContext.changeOrganisation(org, authContext.permissions)) } @@ -59,10 +59,10 @@ class LocalUserSrv @Inject() ( implicit val defaultAuthContext: AuthContext = getSystemAuthContext for { profileStr <- readData(userInfo, profileFieldName, defaultProfile) - profile <- profileSrv.getOrFail(profileStr) + profile <- profileSrv.getOrFail(EntityName(profileStr)) orgaStr <- readData(userInfo, organisationFieldName, defaultOrg) if orgaStr != Organisation.administration.name || profile.name == Profile.admin.name - organisation <- organisationSrv.getOrFail(orgaStr) + organisation <- organisationSrv.getOrFail(EntityName(orgaStr)) richUser <- userSrv.addOrCreateUser( User(userId, userId, None, locked = false, None, None), None, @@ -82,7 +82,7 @@ object LocalUserSrv { AuthContextImpl( User.system.login, User.system.name, - Organisation.administration.name, + EntityIdOrName(Organisation.administration.name), Instance.getInternalId, Permissions.all ) diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 7af9f11de9..9ad586caba 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -1,7 +1,10 @@ package org.thp.thehive.services +import java.util + import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models.{Database, Entity} @@ -12,6 +15,7 @@ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.LogOps._ +import org.thp.thehive.services.TaskOps._ import play.api.libs.json.{JsObject, Json} import scala.util.Try @@ -51,7 +55,7 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv)(implic for { _ <- get(log).attachments.toIterator.toTry(attachmentSrv.cascadeRemove(_)) task <- get(log).task.getOrFail("Task") - _ = get(log._id).remove() + _ = get(log).remove() _ <- auditSrv.log.delete(log, Some(task)) } yield () @@ -72,42 +76,29 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv)(implic object LogOps { implicit class LogOpsDefs(traversal: Traversal.V[Log]) { - def task = traversal.in("TaskLog").v[Task] + def task: Traversal.V[Task] = traversal.in("TaskLog").v[Task] + + def get(idOrName: EntityIdOrName): Traversal.V[Log] = + idOrName.fold(traversal.getByIds(_), _ => traversal.limit(0)) def visible(implicit authContext: AuthContext): Traversal.V[Log] = - traversal.filter( - _.in[TaskLog] - .in[ShareTask] - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.task.visible) - def attachments = traversal.out[LogAttachment].v[Attachment] + def attachments: Traversal.V[Attachment] = traversal.out[LogAttachment].v[Attachment] - def `case` = - traversal - .in[TaskLog] - .in[ShareTask] - .out[ShareCase] - .v[Case] + def `case`: Traversal.V[Case] = task.`case` def can(permission: Permission)(implicit authContext: AuthContext): Traversal.V[Log] = if (authContext.permissions.contains(permission)) - traversal.filter( - _.in[TaskLog] - .in[ShareTask] - .filter(_.out[ShareProfile].has("permissions", permission)) - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.task.can(permission)) else traversal.limit(0) - def richLog = + def richLog: Traversal[RichLog, util.Map[String, Any], Converter[RichLog, util.Map[String, Any]]] = traversal .project( _.by - .by(_.attachments.v[Attachment].fold) + .by(_.attachments.fold) ) .domainMap { case (log, attachments) => @@ -119,11 +110,11 @@ object LogOps { def richLogWithCustomRenderer[D, G, C <: Converter[D, G]]( entityRenderer: Traversal.V[Log] => Traversal[D, G, C] - ) = + ): Traversal[(RichLog, D), util.Map[String, Any], Converter[(RichLog, D), util.Map[String, Any]]] = traversal .project( _.by - .by(_.attachments.v[Attachment].fold) + .by(_.attachments.fold) .by(entityRenderer) ) .domainMap { diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 779393f97b..6a9ca76fe3 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -5,7 +5,6 @@ import java.util.{Map => JMap} import javax.inject.{Inject, Named, Provider, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.{P => JP} import org.apache.tinkerpop.gremlin.structure.{Graph, Vertex} -import org.thp.scalligraph.RichSeq import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models.{Database, Entity} @@ -13,8 +12,11 @@ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} +import org.thp.scalligraph.utils.Hash +import org.thp.scalligraph.{EntityIdOrName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.ObservableOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import play.api.libs.json.JsObject @@ -191,36 +193,39 @@ class ObservableSrv @Inject() ( object ObservableOps { implicit class ObservableOpsDefs(traversal: Traversal.V[Observable]) { + def get(idOrName: EntityIdOrName): Traversal.V[Observable] = + idOrName.fold(traversal.getByIds(_), _ => traversal.limit(0)) def filterOnType(`type`: String): Traversal.V[Observable] = - traversal.filter(_.out[ObservableObservableType].has("name", `type`)) + traversal.filter(_.observableType.has(_.name, `type`)) def filterOnData(data: String): Traversal.V[Observable] = - traversal.filter(_.out[ObservableData].has("data", data)) + traversal.filter(_.data.has(_.data, data)) def filterOnAttachmentName(name: String): Traversal.V[Observable] = - traversal.filter(_.out[ObservableAttachment].has("name", name)) + traversal.filter(_.attachments.has(_.name, name)) def filterOnAttachmentSize(size: Long): Traversal.V[Observable] = - traversal.filter(_.out[ObservableAttachment].has("size", size)) + traversal.filter(_.attachments.has(_.size, size)) def filterOnAttachmentContentType(contentType: String): Traversal.V[Observable] = - traversal.filter(_.out[ObservableAttachment].has("contentType", contentType)) + traversal.filter(_.attachments.has(_.contentType, contentType)) def filterOnAttachmentHash(hash: String): Traversal.V[Observable] = - traversal.filter(_.out[ObservableAttachment].has("hashes", hash)) + traversal.filter(_.attachments.has(_.hashes, Hash(hash))) + + def filterOnAttachmentId(attachmentId: String): Traversal.V[Observable] = + traversal.filter(_.attachments.has(_.attachmentId, attachmentId)) + + def isIoc: Traversal.V[Observable] = + traversal.has(_.ioc, true) def visible(implicit authContext: AuthContext): Traversal.V[Observable] = - traversal.filter(_.in[ShareObservable].in[OrganisationShare].has("name", authContext.organisation)) + traversal.filter(_.organisations.get(authContext.organisation)) def can(permission: Permission)(implicit authContext: AuthContext): Traversal.V[Observable] = if (authContext.permissions.contains(permission)) - traversal.filter( - _.in[ShareObservable] - .filter(_.out[ShareProfile].has("permissions", permission)) - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.shares.filter(_.filter(_.profile.has(_.permissions, permission))).organisation.current) else traversal.limit(0) @@ -232,18 +237,18 @@ object ObservableOps { def organisations: Traversal.V[Organisation] = traversal.in[ShareObservable].in[OrganisationShare].v[Organisation] - def origin: Traversal.V[Organisation] = traversal.in[ShareObservable].has("owner", true).in[OrganisationShare].v[Organisation] + def origin: Traversal.V[Organisation] = shares.has(_.owner, true).organisation def richObservable: Traversal[RichObservable, JMap[String, Any], Converter[RichObservable, JMap[String, Any]]] = traversal .project( _.by - .by(_.out[ObservableObservableType].v[ObservableType].fold) - .by(_.out[ObservableData].v[Data].fold) - .by(_.out[ObservableAttachment].v[Attachment].fold) - .by(_.out[ObservableTag].v[Tag].fold) - .by(_.out[ObservableKeyValue].v[KeyValue].fold) - .by(_.out[ObservableReportTag].v[ReportTag].fold) + .by(_.observableType.fold) + .by(_.data.fold) + .by(_.attachments.fold) + .by(_.tags.fold) + .by(_.keyValues.fold) + .by(_.reportTags.fold) ) .domainMap { case (observable, tpe, data, attachment, tags, extensions, reportTags) => @@ -265,13 +270,13 @@ object ObservableOps { traversal .project( _.by - .by(_.out[ObservableObservableType].v[ObservableType].fold) - .by(_.out[ObservableData].v[Data].fold) - .by(_.out[ObservableAttachment].v[Attachment].fold) - .by(_.out[ObservableTag].v[Tag].fold) + .by(_.observableType.fold) + .by(_.data.fold) + .by(_.attachments.fold) + .by(_.tags.fold) .by(_.similar.visible.limit(1).count) - .by(_.out[ObservableKeyValue].v[KeyValue].fold) - .by(_.out[ObservableReportTag].v[ReportTag].fold) + .by(_.keyValues.fold) + .by(_.reportTags.fold) ) .domainMap { case (observable, tpe, data, attachment, tags, count, extensions, reportTags) => @@ -293,13 +298,13 @@ object ObservableOps { traversal .project( _.by - .by(_.out[ObservableObservableType].v[ObservableType].fold) - .by(_.out[ObservableData].v[Data].fold) - .by(_.out[ObservableAttachment].v[Attachment].fold) - .by(_.out[ObservableTag].v[Tag].fold) + .by(_.observableType.fold) + .by(_.data.fold) + .by(_.attachments.fold) + .by(_.tags.fold) .by(_.similar.visible.limit(1).count) - .by(_.out[ObservableKeyValue].v[KeyValue].fold) - .by(_.out[ObservableReportTag].v[ReportTag].fold) + .by(_.keyValues.fold) + .by(_.reportTags.fold) .by(entityRenderer) ) .domainMap { @@ -355,11 +360,7 @@ object ObservableOps { def share(implicit authContext: AuthContext): Traversal.V[Share] = share(authContext.organisation) - def share(organistionName: String): Traversal.V[Share] = - traversal - .in[ShareObservable] - .filter(_.in[OrganisationShare].has("name", organistionName)) - .v[Share] + def share(organisationName: EntityIdOrName): Traversal.V[Share] = + shares.filter(_.byOrganisation(organisationName)) } - } diff --git a/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala b/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala index 8a04db480f..6b7fafc470 100644 --- a/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala @@ -8,22 +8,21 @@ import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.{BadRequestError, CreateError} +import org.thp.scalligraph.{BadRequestError, CreateError, EntityIdOrName} import org.thp.thehive.models._ import org.thp.thehive.services.ObservableTypeOps._ import scala.util.{Failure, Success, Try} @Singleton -class ObservableTypeSrv @Inject() (@Named("integrity-check-actor") integrityCheckActor: ActorRef)( - implicit @Named("with-thehive-schema") db: Database +class ObservableTypeSrv @Inject() (@Named("integrity-check-actor") integrityCheckActor: ActorRef)(implicit + @Named("with-thehive-schema") db: Database ) extends VertexSrv[ObservableType] { val observableObservableTypeSrv = new EdgeSrv[ObservableObservableType, Observable, ObservableType] - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[ObservableType] = - if (db.isValidId(idOrName)) getByIds(idOrName) - else startTraversal.getByName(idOrName) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[ObservableType] = + startTraversal.getByName(name) override def exists(e: ObservableType)(implicit graph: Graph): Boolean = startTraversal.getByName(e.name).exists @@ -38,11 +37,11 @@ class ObservableTypeSrv @Inject() (@Named("integrity-check-actor") integrityChec else createEntity(observableType) - def remove(idOrName: String)(implicit graph: Graph): Try[Unit] = + def remove(idOrName: EntityIdOrName)(implicit graph: Graph): Try[Unit] = if (useCount(idOrName) == 0) Success(get(idOrName).remove()) else Failure(BadRequestError(s"Observable type $idOrName is used")) - def useCount(idOrName: String)(implicit graph: Graph): Long = + def useCount(idOrName: EntityIdOrName)(implicit graph: Graph): Long = get(idOrName).in[ObservableObservableType].getCount } @@ -50,21 +49,21 @@ object ObservableTypeOps { implicit class ObservableTypeObs(traversal: Traversal.V[ObservableType]) { - def get(idOrName: String)(implicit db: Database): Traversal.V[ObservableType] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) + def get(idOrName: EntityIdOrName): Traversal.V[ObservableType] = + idOrName.fold(traversal.getByIds(_), getByName) - def getByName(name: String): Traversal.V[ObservableType] = traversal.has("name", name).v[ObservableType] + def getByName(name: String): Traversal.V[ObservableType] = traversal.has(_.name, name) } } class ObservableTypeIntegrityCheckOps @Inject() (@Named("with-thehive-schema") val db: Database, val service: ObservableTypeSrv) extends IntegrityCheckOps[ObservableType] { - override def resolve(entities: Seq[ObservableType with Entity])(implicit graph: Graph): Try[Unit] = entities match { - case head :: tail => - tail.foreach(copyEdge(_, head)) - service.getByIds(tail.map(_._id): _*).remove() - Success(()) - case _ => Success(()) - } + override def resolve(entities: Seq[ObservableType with Entity])(implicit graph: Graph): Try[Unit] = + entities match { + case head :: tail => + tail.foreach(copyEdge(_, head)) + service.getByIds(tail.map(_._id): _*).remove() + Success(()) + case _ => Success(()) + } } diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index e71ec182a2..8038415de6 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -1,5 +1,7 @@ package org.thp.thehive.services +import java.util.{Map => JMap} + import akka.actor.ActorRef import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph @@ -9,12 +11,12 @@ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} -import org.thp.scalligraph.{BadRequestError, RichSeq} +import org.thp.scalligraph.{BadRequestError, EntityId, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ -import org.thp.thehive.services.UserOps._ import org.thp.thehive.services.RoleOps._ +import org.thp.thehive.services.UserOps._ import play.api.libs.json.JsObject import scala.util.{Failure, Success, Try} @@ -38,6 +40,8 @@ class OrganisationSrv @Inject() ( super.createEntity(e) } + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Organisation] = startTraversal.getByName(name) + def create(organisation: Organisation, user: User with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Organisation with Entity] = for { createdOrganisation <- create(organisation) @@ -55,17 +59,13 @@ class OrganisationSrv @Inject() ( def visibleOrganisation(implicit graph: Graph, authContext: AuthContext): Traversal.V[Organisation] = userSrv.current.organisations.visibleOrganisationsFrom - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[Organisation] = - if (db.isValidId(idOrName)) getByIds(idOrName) - else startTraversal.getByName(idOrName) - override def exists(e: Organisation)(implicit graph: Graph): Boolean = startTraversal.getByName(e.name).exists override def update( traversal: Traversal.V[Organisation], propertyUpdaters: Seq[PropertyUpdater] )(implicit graph: Graph, authContext: AuthContext): Try[(Traversal.V[Organisation], JsObject)] = - if (traversal.clone().has("name", Organisation.administration.name).exists) + if (traversal.clone().getByName(Organisation.administration.name).exists) Failure(BadRequestError("Admin organisation is unmodifiable")) else auditSrv.mergeAudits(super.update(traversal, propertyUpdaters)) { @@ -77,7 +77,7 @@ class OrganisationSrv @Inject() ( } def linkExists(fromOrg: Organisation with Entity, toOrg: Organisation with Entity)(implicit graph: Graph): Boolean = - fromOrg._id == toOrg._id || get(fromOrg).links.hasId(toOrg._id).exists + fromOrg._id == toOrg._id || get(fromOrg).links.getEntity(toOrg).exists def link(fromOrg: Organisation with Entity, toOrg: Organisation with Entity)(implicit authContext: AuthContext, graph: Graph): Try[Unit] = if (linkExists(fromOrg, toOrg)) Success(()) @@ -104,12 +104,15 @@ class OrganisationSrv @Inject() ( unlink(org2, org1) } - def updateLink(fromOrg: Organisation with Entity, toOrganisations: Seq[String])(implicit authContext: AuthContext, graph: Graph): Try[Unit] = { + def updateLink(fromOrg: Organisation with Entity, toOrganisations: Seq[EntityIdOrName])(implicit + authContext: AuthContext, + graph: Graph + ): Try[Unit] = { val (orgToAdd, orgToRemove) = get(fromOrg) .links - .property[String, String]("name", identity[String]) + ._id .toIterator - .foldLeft((toOrganisations.toSet, Set.empty[String])) { + .foldLeft((toOrganisations.toSet, Set.empty[EntityId])) { case ((toAdd, toRemove), o) if toAdd.contains(o) => (toAdd - o, toRemove) case ((toAdd, toRemove), o) => (toAdd, toRemove + o) } @@ -124,6 +127,11 @@ object OrganisationOps { implicit class OrganisationOpsDefs(traversal: Traversal.V[Organisation]) { + def get(idOrName: EntityIdOrName): Traversal.V[Organisation] = + idOrName.fold(traversal.getByIds(_), traversal.getByName(_)) + + def current(implicit authContext: AuthContext): Traversal.V[Organisation] = get(authContext.organisation) + def links: Traversal.V[Organisation] = traversal.out[OrganisationOrganisation].v[Organisation] def cases: Traversal.V[Case] = traversal.out[OrganisationShare].out[ShareCase].v[Case] @@ -133,16 +141,12 @@ object OrganisationOps { def caseTemplates: Traversal.V[CaseTemplate] = traversal.in[CaseTemplateOrganisation].v[CaseTemplate] def users(requiredPermission: Permission): Traversal.V[User] = - traversal - .in[RoleOrganisation] - .filter(_.out[RoleProfile].has("permissions", requiredPermission)) - .in[UserRole] - .v[User] + traversal.roles.filter(_.profile.has(_.permissions, requiredPermission)).user - def userPermissions(userId: String): Traversal[Permission, String, Converter[Permission, String]] = + def userPermissions(userId: EntityIdOrName): Traversal[Permission, String, Converter[Permission, String]] = traversal .roles - .filter(_.user.has("login", userId)) + .filter(_.user.get(userId)) .profile .property("permissions", Permission(_: String)) @@ -157,9 +161,9 @@ object OrganisationOps { def visible(implicit authContext: AuthContext): Traversal.V[Organisation] = if (authContext.isPermitted(Permissions.manageOrganisation)) traversal else - traversal.filter(_.visibleOrganisationsTo.users.has("login", authContext.userId)) + traversal.filter(_.visibleOrganisationsTo.users.current) - def richOrganisation = + def richOrganisation: Traversal[RichOrganisation, JMap[String, Any], Converter[RichOrganisation, JMap[String, Any]]] = traversal .project( _.by @@ -170,14 +174,12 @@ object OrganisationOps { RichOrganisation(organisation, linkedOrganisations) } + def isAdmin: Boolean = traversal.has(_.name, Organisation.administration.name).exists + def users: Traversal.V[User] = traversal.in[RoleOrganisation].in[UserRole].v[User] def userProfile(login: String): Traversal.V[Profile] = - traversal - .in[RoleOrganisation] - .filter(_.in[UserRole].has("login", login)) - .out[RoleProfile] - .v[Profile] + roles.filter(_.user.has(_.login, login)).profile def visibleOrganisationsTo: Traversal.V[Organisation] = traversal.unionFlat(identity, _.in[OrganisationOrganisation]).dedup().v[Organisation] @@ -187,11 +189,7 @@ object OrganisationOps { def config: Traversal.V[Config] = traversal.out[OrganisationConfig].v[Config] - def get(idOrName: String)(implicit db: Database): Traversal.V[Organisation] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) - - def getByName(name: String): Traversal.V[Organisation] = traversal.has("name", name) + def getByName(name: String): Traversal.V[Organisation] = traversal.has(_.name, name) } } diff --git a/thehive/app/org/thp/thehive/services/PageSrv.scala b/thehive/app/org/thp/thehive/services/PageSrv.scala index d9152b1030..4e28fbe934 100644 --- a/thehive/app/org/thp/thehive/services/PageSrv.scala +++ b/thehive/app/org/thp/thehive/services/PageSrv.scala @@ -9,6 +9,7 @@ import org.thp.scalligraph.services.{EdgeSrv, VertexSrv} import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.models.{Organisation, OrganisationPage, Page} +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.PageOps._ import play.api.libs.json.Json @@ -20,9 +21,7 @@ class PageSrv @Inject() (implicit @Named("with-thehive-schema") db: Database, or val organisationPageSrv = new EdgeSrv[OrganisationPage, Organisation, Page] - override def get(idOrSlug: String)(implicit graph: Graph): Traversal.V[Page] = - if (db.isValidId(idOrSlug)) getByIds(idOrSlug) - else startTraversal.getBySlug(idOrSlug) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Page] = startTraversal.getBySlug(name) def create(page: Page)(implicit authContext: AuthContext, graph: Graph): Try[Page with Entity] = for { @@ -50,14 +49,14 @@ object PageOps { implicit class PageOpsDefs(traversal: Traversal.V[Page]) { - def getByTitle(title: String): Traversal.V[Page] = traversal.has("title", title) + def getByTitle(title: String): Traversal.V[Page] = traversal.has(_.title, title) - def getBySlug(slug: String): Traversal.V[Page] = traversal.has("slug", slug) + def getBySlug(slug: String): Traversal.V[Page] = traversal.has(_.slug, slug) - def visible(implicit authContext: AuthContext): Traversal.V[Page] = traversal.filter( - _.in[OrganisationPage] - .has("name", authContext.organisation) - ) + def organisation: Traversal.V[Organisation] = traversal.in[OrganisationPage].v[Organisation] + + def visible(implicit authContext: AuthContext): Traversal.V[Page] = + traversal.filter(_.organisation.current) } } diff --git a/thehive/app/org/thp/thehive/services/ProfileSrv.scala b/thehive/app/org/thp/thehive/services/ProfileSrv.scala index 0f57dca461..0a066028b6 100644 --- a/thehive/app/org/thp/thehive/services/ProfileSrv.scala +++ b/thehive/app/org/thp/thehive/services/ProfileSrv.scala @@ -3,13 +3,13 @@ package org.thp.thehive.services import akka.actor.ActorRef import javax.inject.{Inject, Named, Provider, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.BadRequestError import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.models._ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{BadRequestError, EntityIdOrName, EntityName} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.ProfileOps._ @@ -22,11 +22,11 @@ class ProfileSrv @Inject() ( auditSrv: AuditSrv, organisationSrvProvider: Provider[OrganisationSrv], @Named("integrity-check-actor") integrityCheckActor: ActorRef -)( - implicit @Named("with-thehive-schema") val db: Database +)(implicit + @Named("with-thehive-schema") val db: Database ) extends VertexSrv[Profile] { lazy val organisationSrv: OrganisationSrv = organisationSrvProvider.get - lazy val orgAdmin: Profile with Entity = db.roTransaction(graph => getOrFail(Profile.orgAdmin.name)(graph)).get + lazy val orgAdmin: Profile with Entity = db.roTransaction(graph => getOrFail(EntityName(Profile.orgAdmin.name))(graph)).get override def createEntity(e: Profile)(implicit graph: Graph, authContext: AuthContext): Try[Profile with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("Profile") @@ -39,9 +39,8 @@ class ProfileSrv @Inject() ( _ <- auditSrv.profile.create(createdProfile, createdProfile.toJson) } yield createdProfile - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[Profile] = - if (db.isValidId(idOrName)) getByIds(idOrName) - else startTraversal.getByName(idOrName) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Profile] = + startTraversal.getByName(name) override def exists(e: Profile)(implicit graph: Graph): Boolean = startTraversal.getByName(e.name).exists @@ -73,25 +72,25 @@ object ProfileOps { def shares: Traversal.V[Share] = traversal.in[ShareProfile].v[Share] - def get(idOrName: String)(implicit db: Database): Traversal.V[Profile] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) + def get(idOrName: EntityIdOrName): Traversal.V[Profile] = + idOrName.fold(traversal.getByIds(_), getByName) - def getByName(name: String): Traversal.V[Profile] = traversal.has("name", name) + def getByName(name: String): Traversal.V[Profile] = traversal.has(_.name, name) def contains(permission: Permission): Traversal.V[Profile] = - traversal.has("permissions", permission) + traversal.has(_.permissions, permission) } } class ProfileIntegrityCheckOps @Inject() (@Named("with-thehive-schema") val db: Database, val service: ProfileSrv) extends IntegrityCheckOps[Profile] { - override def resolve(entities: Seq[Profile with Entity])(implicit graph: Graph): Try[Unit] = entities match { - case head :: tail => - tail.foreach(copyEdge(_, head)) - service.getByIds(tail.map(_._id): _*).remove() - Success(()) - case _ => Success(()) - } + override def resolve(entities: Seq[Profile with Entity])(implicit graph: Graph): Try[Unit] = + entities match { + case head :: tail => + tail.foreach(copyEdge(_, head)) + service.getByIds(tail.map(_._id): _*).remove() + Success(()) + case _ => Success(()) + } } diff --git a/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala b/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala index 1546f87451..d66d6e1ce4 100644 --- a/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala +++ b/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala @@ -3,25 +3,24 @@ package org.thp.thehive.services import akka.actor.ActorRef import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.CreateError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{IntegrityCheckOps, VertexSrv} import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{CreateError, EntityIdOrName} import org.thp.thehive.models.ResolutionStatus import org.thp.thehive.services.ResolutionStatusOps._ import scala.util.{Failure, Success, Try} @Singleton -class ResolutionStatusSrv @Inject() (@Named("integrity-check-actor") integrityCheckActor: ActorRef)( - implicit @Named("with-thehive-schema") db: Database +class ResolutionStatusSrv @Inject() (@Named("integrity-check-actor") integrityCheckActor: ActorRef)(implicit + @Named("with-thehive-schema") db: Database ) extends VertexSrv[ResolutionStatus] { - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[ResolutionStatus] = - if (db.isValidId(idOrName)) getByIds(idOrName) - else startTraversal.getByName(idOrName) + override def getByName(name: String)(implicit graph: Graph): Traversal.V[ResolutionStatus] = + startTraversal.getByName(name) override def createEntity(e: ResolutionStatus)(implicit graph: Graph, authContext: AuthContext): Try[ResolutionStatus with Entity] = { integrityCheckActor ! IntegrityCheckActor.EntityAdded("Resolution") @@ -39,21 +38,21 @@ class ResolutionStatusSrv @Inject() (@Named("integrity-check-actor") integrityCh object ResolutionStatusOps { implicit class ResolutionStatusOpsDefs(traversal: Traversal.V[ResolutionStatus]) { - def get(idOrName: String)(implicit db: Database): Traversal.V[ResolutionStatus] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) + def get(idOrName: EntityIdOrName): Traversal.V[ResolutionStatus] = + idOrName.fold(traversal.getByIds(_), getByName) - def getByName(name: String): Traversal.V[ResolutionStatus] = traversal.has("value", name).v[ResolutionStatus] + def getByName(name: String): Traversal.V[ResolutionStatus] = traversal.has(_.value, name) } } class ResolutionStatusIntegrityCheckOps @Inject() (@Named("with-thehive-schema") val db: Database, val service: ResolutionStatusSrv) extends IntegrityCheckOps[ResolutionStatus] { - override def resolve(entities: Seq[ResolutionStatus with Entity])(implicit graph: Graph): Try[Unit] = entities match { - case head :: tail => - tail.foreach(copyEdge(_, head)) - service.getByIds(tail.map(_._id): _*).remove() - Success(()) - case _ => Success(()) - } + override def resolve(entities: Seq[ResolutionStatus with Entity])(implicit graph: Graph): Try[Unit] = + entities match { + case head :: tail => + tail.foreach(copyEdge(_, head)) + service.getByIds(tail.map(_._id): _*).remove() + Success(()) + case _ => Success(()) + } } diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index cdcfc6bfb6..e0d1ae4c7d 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -5,16 +5,17 @@ import java.util.{Map => JMap} import javax.inject.{Inject, Named, Provider, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.{Graph, T} -import org.thp.scalligraph.CreateError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} +import org.thp.scalligraph.{CreateError, EntityIdOrName} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ +import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ @@ -47,11 +48,11 @@ class ShareSrv @Inject() ( * @param profile the related share profile * @return */ - def shareCase(owner: Boolean, `case`: Case with Entity, organisation: Organisation with Entity, profile: Profile with Entity)( - implicit graph: Graph, + def shareCase(owner: Boolean, `case`: Case with Entity, organisation: Organisation with Entity, profile: Profile with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Share with Entity] = - get(`case`, organisation.name).headOption match { + get(`case`, organisation._id).headOption match { case Some(_) => Failure(CreateError(s"Case #${`case`.number} is already shared with organisation ${organisation.name}")) case None => for { @@ -63,13 +64,13 @@ class ShareSrv @Inject() ( } yield createdShare } - def get(`case`: Case with Entity, organisationName: String)(implicit graph: Graph): Traversal.V[Share] = + def get(`case`: Case with Entity, organisationName: EntityIdOrName)(implicit graph: Graph): Traversal.V[Share] = caseSrv.get(`case`).share(organisationName) - def get(observable: Observable with Entity, organisationName: String)(implicit graph: Graph): Traversal.V[Share] = + def get(observable: Observable with Entity, organisationName: EntityIdOrName)(implicit graph: Graph): Traversal.V[Share] = observableSrv.get(observable).share(organisationName) - def get(task: Task with Entity, organisationName: String)(implicit graph: Graph): Traversal.V[Share] = + def get(task: Task with Entity, organisationName: EntityIdOrName)(implicit graph: Graph): Traversal.V[Share] = taskSrv.get(task).share(organisationName) def update( @@ -88,7 +89,7 @@ class ShareSrv @Inject() ( // def remove(`case`: Case with Entity, organisationId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = // caseSrv.get(`case`).in[ShareCase].filter(_.in[OrganisationShare])._id.getOrFail().flatMap(remove(_)) // FIXME add organisation ? - def remove(shareId: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def remove(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { case0 <- get(shareId).`case`.getOrFail("Case") organisation <- get(shareId).organisation.getOrFail("Organisation") @@ -106,14 +107,15 @@ class ShareSrv @Inject() ( organisation: Organisation with Entity )(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { - shareTask <- taskSrv - .get(task) - .inE[ShareTask] - .filter(_.outV.v[Share].byOrganisationName(organisation.name)) - .getOrFail("Task") + shareTask <- + taskSrv + .get(task) + .inE[ShareTask] + .filter(_.outV.v[Share].byOrganisation(organisation._id)) + .getOrFail("Task") case0 <- taskSrv.get(task).`case`.getOrFail("Case") _ <- auditSrv.share.unshareTask(task, case0, organisation) - } yield shareTaskSrv.get(shareTask.id().toString).remove() + } yield shareTaskSrv.get(shareTask).remove() /** * Unshare Task for a given Organisation @@ -126,14 +128,15 @@ class ShareSrv @Inject() ( organisation: Organisation with Entity )(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { - shareObservable <- observableSrv - .get(observable) - .inE[ShareObservable] - .filter(_.outV.in[OrganisationShare].hasId(organisation._id)) - .getOrFail("Share") + shareObservable <- + observableSrv + .get(observable) + .inE[ShareObservable] + .filter(_.outV.in[OrganisationShare].hasId(organisation._id)) + .getOrFail("Share") case0 <- observableSrv.get(observable).`case`.getOrFail("Case") _ <- auditSrv.share.unshareObservable(observable, case0, organisation) - } yield shareObservableSrv.get(shareObservable.id().toString).remove() + } yield shareObservableSrv.get(shareObservable).remove() /** * Shares all the tasks for an already shared case @@ -158,12 +161,12 @@ class ShareSrv @Inject() ( richTask: RichTask, `case`: Case with Entity, organisation: Organisation with Entity - )( - implicit graph: Graph, + )(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = for { - share <- get(`case`, organisation.name).getOrFail("Case") + share <- get(`case`, organisation._id).getOrFail("Case") _ <- shareTaskSrv.create(ShareTask(), share, richTask.task) _ <- auditSrv.task.create(richTask.task, richTask.toJson) } yield () @@ -172,12 +175,12 @@ class ShareSrv @Inject() ( * Shares an observable for an already shared case * @return */ - def shareObservable(richObservable: RichObservable, `case`: Case with Entity, organisation: Organisation with Entity)( - implicit graph: Graph, + def shareObservable(richObservable: RichObservable, `case`: Case with Entity, organisation: Organisation with Entity)(implicit + graph: Graph, authContext: AuthContext ): Try[Unit] = for { - share <- get(`case`, organisation.name).getOrFail("Case") + share <- get(`case`, organisation._id).getOrFail("Case") _ <- shareObservableSrv.create(ShareObservable(), share, richObservable.observable) _ <- auditSrv.observable.create(richObservable.observable, richObservable.toJson) } yield () @@ -217,12 +220,12 @@ class ShareSrv @Inject() ( case ((toAdd, toRemove), o) if toAdd.contains(o) => (toAdd - o, toRemove) case ((toAdd, toRemove), o) => (toAdd, toRemove + o) } - orgsToRemove.foreach(o => taskSrv.get(task).share(o.name).remove()) + orgsToRemove.foreach(o => taskSrv.get(task).share(o._id).remove()) orgsToAdd .toTry { organisation => for { case0 <- taskSrv.get(task).`case`.getOrFail("Task") - share <- caseSrv.get(case0).share(organisation.name).getOrFail("Share") + share <- caseSrv.get(case0).share(organisation._id).getOrFail("Share") _ <- shareTaskSrv.create(ShareTask(), share, task) _ <- auditSrv.share.shareTask(task, case0, organisation) } yield () @@ -245,7 +248,7 @@ class ShareSrv @Inject() ( .toTry { organisation => for { case0 <- taskSrv.get(task).`case`.getOrFail("Task") - share <- caseSrv.get(case0).share(organisation.name).getOrFail("Case") + share <- caseSrv.get(case0).share(organisation._id).getOrFail("Case") _ <- shareTaskSrv.create(ShareTask(), share, task) _ <- auditSrv.share.shareTask(task, case0, organisation) } yield () @@ -268,7 +271,7 @@ class ShareSrv @Inject() ( .toTry { organisation => for { case0 <- observableSrv.get(observable).`case`.getOrFail("Observable") - share <- caseSrv.get(case0).share(organisation.name).getOrFail("Case") + share <- caseSrv.get(case0).share(organisation._id).getOrFail("Case") _ <- shareObservableSrv.create(ShareObservable(), share, observable) _ <- auditSrv.share.shareObservable(observable, case0, organisation) } yield () @@ -296,12 +299,12 @@ class ShareSrv @Inject() ( case ((toAdd, toRemove), o) if toAdd.contains(o) => (toAdd - o, toRemove) case ((toAdd, toRemove), o) => (toAdd, toRemove + o) } - orgsToRemove.foreach(o => observableSrv.get(observable).share(o.name).remove()) + orgsToRemove.foreach(o => observableSrv.get(observable).share(o._id).remove()) orgsToAdd .toTry { organisation => for { case0 <- observableSrv.get(observable).`case`.getOrFail("Observable") - share <- caseSrv.get(case0).share(organisation.name).getOrFail("Case") + share <- caseSrv.get(case0).share(organisation._id).getOrFail("Case") _ <- shareObservableSrv.create(ShareObservable(), share, observable) _ <- auditSrv.share.shareObservable(observable, case0, organisation) } yield () @@ -312,6 +315,9 @@ class ShareSrv @Inject() ( object ShareOps { implicit class ShareOpsDefs(traversal: Traversal.V[Share]) { + def get(idOrName: EntityIdOrName): Traversal.V[Share] = + idOrName.fold(traversal.getByIds(_), _ => traversal.limit(0)) + def relatedTo(`case`: Case with Entity): Traversal.V[Share] = traversal.filter(_.`case`.hasId(`case`._id)) def `case`: Traversal.V[Case] = traversal.out[ShareCase].v[Case] @@ -322,17 +328,14 @@ object ShareOps { def tasks: Traversal.V[Task] = traversal.out[ShareTask].v[Task] - def byTask(taskId: String): Traversal.V[Share] = traversal.filter( - _.out[ShareTask].hasId(taskId) - ) + def byTask(taskId: EntityIdOrName): Traversal.V[Share] = + traversal.filter(_.tasks.get(taskId)) - def byObservable(observableId: String): Traversal.V[Share] = traversal.filter( - _.out[ShareObservable].hasId(observableId) - ) + def byObservable(observableId: EntityIdOrName): Traversal.V[Share] = + traversal.filter(_.observables.get(observableId)) - def byOrganisationName(organisationName: String): Traversal.V[Share] = traversal.filter( - _.in[OrganisationShare].has("name", organisationName) - ) + def byOrganisation(organisationName: EntityIdOrName): Traversal.V[Share] = + traversal.filter(_.organisation.get(organisationName)) def observables: Traversal.V[Observable] = traversal.out[ShareObservable].v[Observable] diff --git a/thehive/app/org/thp/thehive/services/StreamSrv.scala b/thehive/app/org/thp/thehive/services/StreamSrv.scala index ba87268f76..e3a89c3e22 100644 --- a/thehive/app/org/thp/thehive/services/StreamSrv.scala +++ b/thehive/app/org/thp/thehive/services/StreamSrv.scala @@ -7,13 +7,13 @@ import akka.pattern.{ask, AskTimeoutException} import akka.serialization.Serializer import akka.util.Timeout import javax.inject.{Inject, Named, Singleton} -import org.thp.scalligraph.NotFoundError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.EventSrv import org.thp.scalligraph.services.config.ApplicationConfig.finiteDurationFormat import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{EntityId, NotFoundError} import org.thp.thehive.services.AuditOps._ import play.api.Logger import play.api.libs.json.Json @@ -29,7 +29,7 @@ object StreamTopic { def apply(streamId: String = ""): String = if (streamId.isEmpty) "stream" else s"stream-$streamId" } -case class AuditStreamMessage(id: String*) extends StreamMessage +case class AuditStreamMessage(id: EntityId*) extends StreamMessage /* Ask messages, wait if there is no ready messages */ case object GetStreamMessages extends StreamMessage case object Commit extends StreamMessage @@ -56,7 +56,7 @@ class StreamActor( receive(Nil, keepAliveTimer) } - def receive(messages: Seq[String], keepAliveTimer: Cancellable): Receive = { + def receive(messages: Seq[EntityId], keepAliveTimer: Cancellable): Receive = { case GetStreamMessages => logger.debug(s"[$self] GetStreamMessages") // rearm keepalive @@ -82,7 +82,7 @@ class StreamActor( } def receive( - messages: Seq[String], + messages: Seq[EntityId], requestActor: ActorRef, keepAliveTimer: Cancellable, commitTimer: Cancellable, @@ -178,7 +178,7 @@ class StreamSrv @Inject() ( streamId } - def get(streamId: String): Future[Seq[String]] = { + def get(streamId: String): Future[Seq[EntityId]] = { implicit val timeout: Timeout = Timeout(refresh + 1.second) // Check if stream actor exists eventSrv @@ -225,6 +225,6 @@ class StreamSerializer extends Serializer { new String(bytes) match { case "GetStreamMessages" => GetStreamMessages case "Commit" => Commit - case s => Try(AuditStreamMessage(Json.parse(s).as[Seq[String]]: _*)).getOrElse(throw new NotSerializableException) + case s => Try(AuditStreamMessage(Json.parse(s).as[Seq[String]].map(EntityId.read): _*)).getOrElse(throw new NotSerializableException) } } diff --git a/thehive/app/org/thp/thehive/services/TOTPAuthSrv.scala b/thehive/app/org/thp/thehive/services/TOTPAuthSrv.scala index a2828307ee..94441d57ba 100644 --- a/thehive/app/org/thp/thehive/services/TOTPAuthSrv.scala +++ b/thehive/app/org/thp/thehive/services/TOTPAuthSrv.scala @@ -12,7 +12,7 @@ import org.thp.scalligraph.auth._ import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.{AuthenticationError, MultiFactorCodeRequired} +import org.thp.scalligraph.{AuthenticationError, EntityIdOrName, MultiFactorCodeRequired} import play.api.Configuration import play.api.mvc.RequestHeader @@ -45,8 +45,8 @@ class TOTPAuthSrv( (timestamp - 1 to timestamp + 1).exists { ts => val data = (56 to 0 by -8).map(i => (ts >> i).toByte).toArray val hash = mac.doFinal(data) - val offset = hash(hash.length - 1) & 0xF - (BigInt(hash.slice(offset, offset + 4)).toInt & 0x7FFFFFFF) % 1000000 == code + val offset = hash(hash.length - 1) & 0xf + (BigInt(hash.slice(offset, offset + 4)).toInt & 0x7fffffff) % 1000000 == code } } @@ -59,8 +59,8 @@ class TOTPAuthSrv( } .getOrElse(Failure(MultiFactorCodeRequired("MFA code is required"))) - override def authenticate(username: String, password: String, organisation: Option[String], code: Option[String])( - implicit request: RequestHeader + override def authenticate(username: String, password: String, organisation: Option[EntityIdOrName], code: Option[String])(implicit + request: RequestHeader ): Try[AuthContext] = super.authenticate(username, password, organisation, code).flatMap { case authContext if !enabled => Success(authContext) @@ -72,10 +72,10 @@ class TOTPAuthSrv( } def getSecret(username: String)(implicit graph: Graph): Option[String] = - userSrv.get(username).headOption.flatMap(_.totpSecret) + userSrv.get(EntityIdOrName(username)).headOption.flatMap(_.totpSecret) def unsetSecret(username: String)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - userSrv.get(username).update(_.totpSecret, None).domainMap(_ => ()).getOrFail("User") + userSrv.get(EntityIdOrName(username)).update(_.totpSecret, None).domainMap(_ => ()).getOrFail("User") def generateSecret(): String = { val key = Array.ofDim[Byte](20) @@ -88,7 +88,7 @@ class TOTPAuthSrv( def setSecret(username: String, secret: String)(implicit graph: Graph, authContext: AuthContext): Try[String] = userSrv - .get(username) + .get(EntityIdOrName(username)) .update(_.totpSecret, Some(secret)) .domainMap(_ => secret) .getOrFail("User") diff --git a/thehive/app/org/thp/thehive/services/TagSrv.scala b/thehive/app/org/thp/thehive/services/TagSrv.scala index 7f888bd22c..7b78029f4f 100644 --- a/thehive/app/org/thp/thehive/services/TagSrv.scala +++ b/thehive/app/org/thp/thehive/services/TagSrv.scala @@ -15,8 +15,8 @@ import org.thp.thehive.services.TagOps._ import scala.util.{Success, Try} @Singleton -class TagSrv @Inject() (appConfig: ApplicationConfig, @Named("integrity-check-actor") integrityCheckActor: ActorRef)( - implicit @Named("with-thehive-schema") db: Database +class TagSrv @Inject() (appConfig: ApplicationConfig, @Named("integrity-check-actor") integrityCheckActor: ActorRef)(implicit + @Named("with-thehive-schema") db: Database ) extends VertexSrv[Tag] { val autoCreateConfig: ConfigItem[Boolean, Boolean] = @@ -32,7 +32,8 @@ class TagSrv @Inject() (appConfig: ApplicationConfig, @Named("integrity-check-ac val defaultColourConfig: ConfigItem[String, Int] = appConfig.mapItem[String, Int]( "tags.defaultColour", - "Default colour of the automatically created tags", { + "Default colour of the automatically created tags", + { case s if s(0) == '#' => Try(Integer.parseUnsignedInt(s.tail, 16)).getOrElse(defaultColour) case _ => defaultColour } @@ -42,9 +43,6 @@ class TagSrv @Inject() (appConfig: ApplicationConfig, @Named("integrity-check-ac def parseString(tagName: String): Tag = Tag.fromString(tagName, defaultNamespace, defaultColour) - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[Tag] = - getByIds(idOrName) - def getTag(tag: Tag)(implicit graph: Graph): Traversal.V[Tag] = startTraversal.getTag(tag) def getOrCreate(tagName: String)(implicit graph: Graph, authContext: AuthContext): Try[Tag with Entity] = { @@ -71,10 +69,10 @@ object TagOps { def getTag(tag: Tag): Traversal.V[Tag] = getByName(tag.namespace, tag.predicate, tag.value) def getByName(namespace: String, predicate: String, value: Option[String]): Traversal.V[Tag] = { - val step = traversal - .has("namespace", namespace) - .has("predicate", predicate) - value.fold(step.hasNot("value"))(v => step.has("value", v)) + val t = traversal + .has(_.namespace, namespace) + .has(_.predicate, predicate) + value.fold(t.hasNot(_.value))(v => t.has(_.value, v)) } def displayName: Traversal[String, Vertex, Converter[String, Vertex]] = traversal.domainMap(_.toString) diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index e962ce220d..574d074392 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -5,6 +5,7 @@ import java.util.Date import javax.inject.{Inject, Named, Provider, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.PropertyUpdater @@ -13,6 +14,7 @@ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.models.{TaskStatus, _} import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ import play.api.libs.json.{JsNull, JsObject, Json} @@ -34,8 +36,8 @@ class TaskSrv @Inject() (caseSrvProvider: Provider[CaseSrv], auditSrv: AuditSrv) _ <- owner.map(taskUserSrv.create(TaskUser(), task, _)).flip } yield RichTask(task, owner) - def isAvailableFor(taskId: String)(implicit graph: Graph, authContext: AuthContext): Boolean = - getByIds(taskId).visible(authContext).exists + def isAvailableFor(taskId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Boolean = + get(taskId).visible(authContext).exists def unassign(task: Task with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { get(task).unassign() @@ -117,23 +119,17 @@ class TaskSrv @Inject() (caseSrvProvider: Provider[CaseSrv], auditSrv: AuditSrv) object TaskOps { implicit class TaskOpsDefs(traversal: Traversal.V[Task]) { + def get(idOrName: EntityIdOrName): Traversal.V[Task] = + idOrName.fold(traversal.getByIds(_), _ => traversal.limit(0)) + def visible(implicit authContext: AuthContext): Traversal.V[Task] = - traversal.filter( - _.in[ShareTask] - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.organisations.current) - def active: Traversal.V[Task] = traversal.filterNot(_.has("status", "Cancel")) + def active: Traversal.V[Task] = traversal.filterNot(_.has(_.status, TaskStatus.Cancel)) def can(permission: Permission)(implicit authContext: AuthContext): Traversal.V[Task] = if (authContext.permissions.contains(permission)) - traversal.filter( - _.in[ShareTask] - .filter(_.out[ShareProfile].has("permissions", permission)) - .in[OrganisationShare] - .has("name", authContext.organisation) - ) + traversal.filter(_.shares.filter(_.profile.has(_.permissions, permission)).organisation.current) else traversal.limit(0) @@ -153,9 +149,9 @@ object TaskOps { def organisations: Traversal.V[Organisation] = traversal.in[ShareTask].in[OrganisationShare].v[Organisation] def organisations(permission: Permission): Traversal.V[Organisation] = - traversal.in[ShareTask].filter(_.out[ShareProfile].has("permissions", permission)).in[OrganisationShare].v[Organisation] + shares.filter(_.profile.has(_.permissions, permission)).organisation - def origin: Traversal.V[Organisation] = traversal.in[ShareTask].has("owner", true).in[OrganisationShare].v[Organisation] + def origin: Traversal.V[Organisation] = shares.has(_.owner, true).organisation def assignableUsers(implicit authContext: AuthContext): Traversal.V[User] = organisations(Permissions.manageTask) @@ -179,7 +175,7 @@ object TaskOps { traversal .project( _.by - .by(_.out[TaskUser].v[User].fold) + .by(_.assignee.fold) .by(entityRenderer) ) .domainMap { @@ -193,7 +189,7 @@ object TaskOps { def share(implicit authContext: AuthContext): Traversal.V[Share] = share(authContext.organisation) - def share(organistionName: String): Traversal.V[Share] = - traversal.in[ShareTask].filter(_.in[OrganisationShare].has("name", organistionName)).v[Share] + def share(organisation: EntityIdOrName): Traversal.V[Share] = + traversal.in[ShareTask].filter(_.in[OrganisationShare].v[Organisation].get(organisation)).v[Share] } } diff --git a/thehive/app/org/thp/thehive/services/UserSrv.scala b/thehive/app/org/thp/thehive/services/UserSrv.scala index cb20d6aeb4..f1d0cb059d 100644 --- a/thehive/app/org/thp/thehive/services/UserSrv.scala +++ b/thehive/app/org/thp/thehive/services/UserSrv.scala @@ -15,7 +15,7 @@ import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.Converter.CList import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} -import org.thp.scalligraph.{AuthorizationError, BadRequestError, InternalError, RichOptionTry} +import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityIdOrName, EntityName, InternalError, RichOptionTry} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.OrganisationOps._ @@ -65,7 +65,7 @@ class UserSrv @Inject() ( else Success(())).flatMap { _ => for { - richUser <- get(user).richUser(organisation.name).getOrFail("User") + richUser <- get(user).richUser(authContext, organisation._id).getOrFail("User") _ <- auditSrv.user.create(user, richUser.toJson) } yield richUser } @@ -74,7 +74,7 @@ class UserSrv @Inject() ( graph: Graph, authContext: AuthContext ): Try[RichUser] = - get(user.login) + getByName(user.login) .getOrFail("User") .orElse { for { @@ -92,8 +92,8 @@ class UserSrv @Inject() ( } def delete(user: User with Entity, organisation: Organisation with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { - if (get(user).organisations.hasNot("name", organisation.name).exists) - get(user).role.filter(_.organisation.has("name", organisation.name)).remove() + if (get(user).organisations.filterNot(_.get(organisation._id)).exists) + get(user).role.filterNot(_.organisation.get(organisation._id)).remove() else { get(user).role.remove() get(user).remove() @@ -118,15 +118,13 @@ class UserSrv @Inject() ( _ <- auditSrv.user.update(updatedUser, Json.obj("locked" -> false)) } yield updatedUser - def current(implicit graph: Graph, authContext: AuthContext): Traversal.V[User] = get(authContext.userId) + def current(implicit graph: Graph, authContext: AuthContext): Traversal.V[User] = get(EntityName(authContext.userId)) - override def get(idOrName: String)(implicit graph: Graph): Traversal.V[User] = - if (db.isValidId(idOrName)) getByIds(idOrName) - else - defaultUserDomain.fold(startTraversal.getByName(idOrName)) { - case d if !idOrName.contains('@') => startTraversal.getByName(s"$idOrName@$d") - case _ => startTraversal.getByName(idOrName) - } + override def getByName(name: String)(implicit graph: Graph): Traversal.V[User] = + defaultUserDomain.fold(startTraversal.getByName(name)) { + case d if !name.contains('@') => startTraversal.getByName(s"$name@$d") + case _ => startTraversal.getByName(name) + } override def update( traversal: Traversal.V[User], @@ -156,7 +154,7 @@ class UserSrv @Inject() ( authContext: AuthContext ): Try[Unit] = for { - role <- get(user).role.filter(_.organisation.getByIds(organisation._id)).getOrFail("User") + role <- get(user).role.filter(_.organisation.getEntity(organisation)).getOrFail("User") _ = roleSrv.updateProfile(role, profile) _ <- auditSrv.user.changeProfile(user, organisation, profile) } yield () @@ -165,34 +163,31 @@ class UserSrv @Inject() ( object UserOps { implicit class UserOpsDefs(traversal: Traversal.V[User]) { - def current(authContext: AuthContext): Traversal.V[User] = getByName(authContext.userId) + def get(idOrName: EntityIdOrName): Traversal.V[User] = + idOrName.fold(traversal.getByIds(_), getByName) - def get(idOrName: String)(implicit db: Database): Traversal.V[User] = - if (db.isValidId(idOrName)) traversal.getByIds(idOrName) - else getByName(idOrName) + def current(implicit authContext: AuthContext): Traversal.V[User] = getByName(authContext.userId) - def getByName(login: String): Traversal.V[User] = traversal.has("login", login.toLowerCase) + def getByName(login: String): Traversal.V[User] = traversal.has(_.login, login.toLowerCase) def visible(implicit authContext: AuthContext): Traversal.V[User] = if (authContext.isPermitted(Permissions.manageOrganisation.permission)) traversal else - traversal.filter(_.or(_.organisations.visibleOrganisationsTo.getByName(authContext.organisation), _.systemUser)) + traversal.filter(_.or(_.organisations.visibleOrganisationsTo.get(authContext.organisation), _.systemUser)) + + def isNotLocked: Traversal.V[User] = traversal.has(_.locked, false) def can(requiredPermission: Permission)(implicit authContext: AuthContext, db: Database): Traversal.V[User] = - traversal.filter(_.organisations(requiredPermission).getByName(authContext.organisation)) + traversal.filter(_.organisations(requiredPermission).get(authContext.organisation)) - def getByAPIKey(key: String): Traversal.V[User] = traversal.has("apikey", key).v[User] + def getByAPIKey(key: String): Traversal.V[User] = traversal.has(_.apikey, key).v[User] def organisations: Traversal.V[Organisation] = traversal.out[UserRole].out[RoleOrganisation].v[Organisation] - protected def organisations0(requiredPermission: String): Traversal.V[Organisation] = - traversal - .out[UserRole] - .filter(_.out[RoleProfile].has("permissions", requiredPermission)) - .out[RoleOrganisation] - .v[Organisation] + protected def organisations0(requiredPermission: Permission): Traversal.V[Organisation] = + role.filter(_.profile.has(_.permissions, requiredPermission)).organisation - def organisations(requiredPermission: String)(implicit db: Database): Traversal.V[Organisation] = { + def organisations(requiredPermission: Permission)(implicit db: Database): Traversal.V[Organisation] = { val isInAdminOrganisation = traversal.clone().organisations0(requiredPermission).getByName(Organisation.administration.name).exists if (isInAdminOrganisation) db.labelFilter("Organisation")(traversal.V()).v[Organisation] else organisations0(requiredPermission) @@ -202,11 +197,10 @@ object UserOps { (String, String), JMap[String, Any] ]]] = - traversal - .out[UserRole] + role .project( - _.by(_.out[RoleOrganisation].v[Organisation].value(_.name)) - .by(_.out[RoleProfile].v[Profile].value(_.name)) + _.by(_.organisation.value(_.name)) + .by(_.profile.value(_.name)) ) .fold @@ -214,67 +208,73 @@ object UserOps { def getAuthContext( requestId: String, - organisation: Option[String] + organisation: Option[EntityIdOrName] ): Traversal[AuthContext, JMap[String, Any], Converter[AuthContext, JMap[String, Any]]] = { val organisationName = organisation .orElse( traversal .clone() - .out[UserRole] + .role .sort(_.by("_createdAt", Order.asc)) - .out[RoleOrganisation] - .v[Organisation] - .value(_.name) + .organisation + ._id .headOption ) - .getOrElse(Organisation.administration.name) + .getOrElse(EntityName(Organisation.administration.name)) getAuthContext(requestId, organisationName) } def getAuthContext( requestId: String, - organisationName: String + organisationName: EntityIdOrName ): Traversal[AuthContext, JMap[String, Any], Converter[AuthContext, JMap[String, Any]]] = traversal - .filter(_.organisations.getByName(organisationName)) - .has("locked", false) + .isNotLocked .project( _.byValue(_.login) .byValue(_.name) - .by(_.out[UserRole].filter(_.out[RoleOrganisation].has("name", organisationName)).out[RoleProfile].v[Profile]) + .by(_.role.filter(_.organisation.get(organisationName)).profile) + .by(_.organisations.get(organisationName).value(_.name).limit(1).fold) ) .domainMap { - case (userId, userName, profile) => + case (userId, userName, profile, org) => val scope = - if (organisationName == Organisation.administration.name) "admin" + if (org.contains(Organisation.administration.name)) "admin" else "organisation" val permissions = Permissions.forScope(scope) & profile.permissions AuthContextImpl(userId, userName, organisationName, requestId, permissions) } - def profile(organisation: String): Traversal.V[Profile] = - traversal.out[UserRole].filter(_.out[RoleOrganisation].has("name", organisation)).out[RoleProfile].v[Profile] + def profile(organisation: EntityIdOrName): Traversal.V[Profile] = + role.filter(_.organisation.get(organisation)).profile def richUser(implicit authContext: AuthContext): Traversal[RichUser, JMap[String, Any], Converter[RichUser, JMap[String, Any]]] = + richUser(authContext, authContext.organisation) + + def richUser( + authContext: AuthContext, + organisation: EntityIdOrName + ): Traversal[RichUser, JMap[String, Any], Converter[RichUser, JMap[String, Any]]] = traversal .project( _.by .by(_.avatar.fold) - .by(_.role.project(_.by(_.profile).by(_.organisation.visible.value(_.name).fold)).fold) + .by(_.role.project(_.by(_.profile).by(_.organisation.visible(authContext).project(_.by(_._id).byValue(_.name)).fold)).fold) ) .domainMap { case (user, attachment, profileOrganisations) => - profileOrganisations - .find(_._2.contains(authContext.organisation)) + organisation + .fold(id => profileOrganisations.find(_._2.exists(_._1 == id)), name => profileOrganisations.find(_._2.exists(_._2 == name))) .orElse(profileOrganisations.headOption) .fold(throw InternalError(s"")) { // FIXME - case (profile, organisationName) => + case (profile, organisationIdAndName) => val avatar = attachment.headOption.map(_.attachmentId) - RichUser(user, avatar, profile.name, profile.permissions, organisationName.headOption.getOrElse("***")) + RichUser(user, avatar, profile.name, profile.permissions, organisationIdAndName.headOption.fold("***")(_._2)) } } - def richUser(organisation: String): Traversal[RichUser, JMap[String, Any], Converter[RichUser, JMap[String, Any]]] = + /* + def richUser(organisationId: EntityId): Traversal[RichUser, JMap[String, Any], Converter[RichUser, JMap[String, Any]]] = traversal .project( _.by @@ -292,36 +292,39 @@ object UserOps { ) } + */ + def richUserWithCustomRenderer[D, G, C <: Converter[D, G]]( - organisation: String, + organisation: EntityIdOrName, entityRenderer: Traversal.V[User] => Traversal[D, G, C] - ): Traversal[(RichUser, D), JMap[String, Any], Converter[(RichUser, D), JMap[String, Any]]] = + )(implicit authContext: AuthContext): Traversal[(RichUser, D), JMap[String, Any], Converter[(RichUser, D), JMap[String, Any]]] = traversal .project( _.by - .by(_.profile(organisation).fold) .by(_.avatar.fold) + .by(_.role.project(_.by(_.profile).by(_.organisation.visible.project(_.by(_._id).byValue(_.name)).fold)).fold) .by(entityRenderer) ) .domainMap { - case (user, profiles, attachment, renderedEntity) => - RichUser( - user, - attachment.headOption.map(_.attachmentId), - profiles.headOption.fold("")(_.name), - profiles.headOption.fold(Set.empty[Permission])(_.permissions), - organisation - ) -> renderedEntity + case (user, attachment, profileOrganisations, renderedEntity) => + organisation + .fold(id => profileOrganisations.find(_._2.exists(_._1 == id)), name => profileOrganisations.find(_._2.exists(_._2 == name))) + .orElse(profileOrganisations.headOption) + .fold(throw InternalError(s"")) { // FIXME + case (profile, organisationIdAndName) => + val avatar = attachment.headOption.map(_.attachmentId) + RichUser(user, avatar, profile.name, profile.permissions, organisationIdAndName.headOption.fold("***")(_._2)) -> renderedEntity + } } def config(configName: String): Traversal.V[Config] = - traversal.out[UserConfig].has("name", configName).v[Config] + traversal.out[UserConfig].v[Config].has(_.name, configName) def role: Traversal.V[Role] = traversal.out[UserRole].v[Role] def avatar: Traversal.V[Attachment] = traversal.out[UserAttachment].v[Attachment] - def systemUser: Traversal.V[User] = traversal.has("login", User.system.login) + def systemUser: Traversal.V[User] = traversal.has(_.login, User.system.login) def dashboards: Traversal.V[Dashboard] = traversal.in[DashboardUser].v[Dashboard] @@ -343,7 +346,7 @@ class UserIntegrityCheckOps @Inject() ( override def initialCheck()(implicit graph: Graph, authContext: AuthContext): Unit = { super.initialCheck() val adminUserIsCreated = service - .get(User.init.login) + .getByName(User.init.login) .role .filter(_.profile.getByName(Profile.admin.name)) .organisation @@ -351,9 +354,9 @@ class UserIntegrityCheckOps @Inject() ( .exists if (!adminUserIsCreated) for { - adminUser <- service.getOrFail(User.init.login) - adminProfile <- profileSrv.getOrFail(Profile.admin.name) - adminOrganisation <- organisationSrv.getOrFail(Organisation.administration.name) + adminUser <- service.getByName(User.init.login).getOrFail("User") + adminProfile <- profileSrv.getByName(Profile.admin.name).getOrFail("Profile") + adminOrganisation <- organisationSrv.getByName(Organisation.administration.name).getOrFail("Organisation") _ <- roleSrv.create(adminUser, adminOrganisation, adminProfile) } yield () () diff --git a/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala b/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala index dead9d9464..a49724452e 100644 --- a/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala +++ b/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala @@ -4,10 +4,10 @@ import akka.actor.{Actor, ActorIdentity, Identify} import akka.util.Timeout import javax.inject.{Inject, Named} import org.apache.tinkerpop.gremlin.structure.Graph -import org.thp.scalligraph.BadConfigurationError import org.thp.scalligraph.models.{Database, Entity, Schema} import org.thp.scalligraph.services.EventSrv import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{BadConfigurationError, EntityId} import org.thp.thehive.models.{Audit, Organisation, User} import org.thp.thehive.services.AuditOps._ import org.thp.thehive.services.OrganisationOps._ @@ -29,12 +29,12 @@ object NotificationTopic { } sealed trait NotificationMessage -case class NotificationExecution(userId: Option[String], auditId: String, notificationConfig: NotificationConfig) extends NotificationMessage +case class NotificationExecution(userId: Option[EntityId], auditId: EntityId, notificationConfig: NotificationConfig) extends NotificationMessage object NotificationExecution { implicit val format: Format[NotificationExecution] = Json.format[NotificationExecution] } -case class AuditNotificationMessage(id: String*) extends NotificationMessage +case class AuditNotificationMessage(id: EntityId*) extends NotificationMessage object AuditNotificationMessage { implicit val format: Format[AuditNotificationMessage] = Json.format[AuditNotificationMessage] @@ -90,7 +90,7 @@ class NotificationActor @Inject() ( val roles: Set[String] = configuration.get[Seq[String]]("roles").toSet // Map of OrganisationId -> Trigger -> (present in org, list of UserId) */ - def triggerMap: Map[String, Map[Trigger, (Boolean, Seq[String])]] = + def triggerMap: Map[EntityId, Map[Trigger, (Boolean, Seq[EntityId])]] = cache.getOrElseUpdate("notification-triggers", 5.minutes)(db.roTransaction(graph => configSrv.triggerMap(notificationSrv)(graph))) override def preStart(): Unit = { @@ -117,8 +117,8 @@ class NotificationActor @Inject() ( context: Option[Entity], `object`: Option[Entity], organisation: Organisation with Entity - )( - implicit graph: Graph + )(implicit + graph: Graph ): Unit = notificationConfigs .foreach { @@ -176,7 +176,7 @@ class NotificationActor @Inject() ( val config = notificationConfig.flatMap(_.asOpt[NotificationConfig]) executeNotification(Some(user), config, audit, context, obj, organisation) } - if (inOrg) { + if (inOrg) organisationSrv .get(organisation) .config @@ -198,7 +198,6 @@ class NotificationActor @Inject() ( } executeNotification(None, orgConfig, audit, context, obj, organisation) } - } case (trigger, _) => logger.debug(s"Notification trigger ${trigger.name} is NOT applicable") } case _ => diff --git a/thehive/app/org/thp/thehive/services/notification/notifiers/Template.scala b/thehive/app/org/thp/thehive/services/notification/notifiers/Template.scala index ee9d65f483..1ad397e682 100644 --- a/thehive/app/org/thp/thehive/services/notification/notifiers/Template.scala +++ b/thehive/app/org/thp/thehive/services/notification/notifiers/Template.scala @@ -29,14 +29,14 @@ trait Template { .flatMap { f => cc.getClass.getSuperclass.getDeclaredMethod(f).invoke(cc) match { case option: Option[_] => option.map(f -> _.toString) - case list: Seq[_] => Some(f -> list.mkString("[", ",", "]")) - case set: Set[_] => Some(f -> set.mkString("[", ",", "]")) - case other => Some(f -> other.toString) + case list: Seq[_] => Some(f -> list.mkString("[", ",", "]")) + case set: Set[_] => Some(f -> set.mkString("[", ",", "]")) + case other => Some(f -> other.toString) } } .toMap } + - ("_id" -> cc._id) + + ("_id" -> cc._id.toString) + ("_type" -> cc._label) + ("_createdAt" -> cc._createdAt.toString) + ("_createdBy" -> cc._createdBy) + diff --git a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala index ad21bf9974..d7b65a3448 100644 --- a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala +++ b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala @@ -6,11 +6,11 @@ import akka.stream.Materializer import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.structure.{Graph, Vertex} import org.thp.client.{ProxyWS, ProxyWSConfig} -import org.thp.scalligraph.BadConfigurationError -import org.thp.scalligraph.models.{Entity, Schema, UMapping} +import org.thp.scalligraph.models.{Entity, UMapping} import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IdentityConverter, Traversal} +import org.thp.scalligraph.{BadConfigurationError, EntityIdOrName} import org.thp.thehive.controllers.v0.AuditRenderer import org.thp.thehive.controllers.v0.Conversion.fromObjectType import org.thp.thehive.models._ @@ -52,7 +52,6 @@ class WebhookProvider @Inject() ( appConfig: ApplicationConfig, auditSrv: AuditSrv, customFieldSrv: CustomFieldSrv, - schema: Schema, ec: ExecutionContext, mat: Materializer ) extends NotifierProvider { @@ -182,7 +181,7 @@ class Webhook( case keyValue @ (key, value) if key.startsWith("customField.") => val fieldName = key.drop(12) customFieldSrv - .getOrFail(fieldName) + .getOrFail(EntityIdOrName(fieldName)) .fold(_ => keyValue, cf => "customFields" -> Json.obj(fieldName -> Json.obj(cf.`type`.toString -> value))) case keyValue => keyValue }) diff --git a/thehive/app/org/thp/thehive/services/notification/triggers/LogInMyTask.scala b/thehive/app/org/thp/thehive/services/notification/triggers/LogInMyTask.scala index 792e978746..c98751d464 100644 --- a/thehive/app/org/thp/thehive/services/notification/triggers/LogInMyTask.scala +++ b/thehive/app/org/thp/thehive/services/notification/triggers/LogInMyTask.scala @@ -2,6 +2,7 @@ package org.thp.thehive.services.notification.triggers import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.EntityId import org.thp.scalligraph.models.Entity import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.models.{Audit, Organisation, User} @@ -31,8 +32,9 @@ class LogInMyTask(logSrv: LogSrv) extends Trigger { super.filter(audit, context, organisation, user) && preFilter(audit, context, organisation) && u.login != audit._createdBy && - audit.objectId.fold(false)(taskAssignee(_).fold(false)(_ == u.login)) + audit.objectEntityId.fold(false)(o => taskAssignee(o).fold(false)(_ == u.login)) } - def taskAssignee(logId: String)(implicit graph: Graph): Option[String] = logSrv.getByIds(logId).task.assignee.value(_.login).headOption + def taskAssignee(logId: EntityId)(implicit graph: Graph): Option[String] = + logSrv.getByIds(logId).task.assignee.value(_.login).headOption } diff --git a/thehive/app/org/thp/thehive/services/notification/triggers/TaskAssigned.scala b/thehive/app/org/thp/thehive/services/notification/triggers/TaskAssigned.scala index 7dcf2dd331..1ff4598baa 100644 --- a/thehive/app/org/thp/thehive/services/notification/triggers/TaskAssigned.scala +++ b/thehive/app/org/thp/thehive/services/notification/triggers/TaskAssigned.scala @@ -2,11 +2,13 @@ package org.thp.thehive.services.notification.triggers import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.EntityId import org.thp.scalligraph.models.Entity import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.models.{Audit, Organisation, User} import org.thp.thehive.services.TaskOps._ import org.thp.thehive.services.TaskSrv +import org.thp.thehive.services.UserOps._ import play.api.Configuration import scala.util.{Success, Try} @@ -25,13 +27,14 @@ class TaskAssigned(taskSrv: TaskSrv) extends Trigger { override def filter(audit: Audit with Entity, context: Option[Entity], organisation: Organisation with Entity, user: Option[User with Entity])( implicit graph: Graph - ): Boolean = user.fold(false) { u => - preFilter(audit, context, organisation) && - super.filter(audit, context, organisation, user) && - u.login != audit._createdBy && - audit.objectId.fold(false)(taskAssignee(_, u.login).isDefined) - } + ): Boolean = + user.fold(false) { u => + preFilter(audit, context, organisation) && + super.filter(audit, context, organisation, user) && + u.login != audit._createdBy && + audit.objectEntityId.fold(false)(taskAssignee(_, u._id).isDefined) + } - def taskAssignee(taskId: String, login: String)(implicit graph: Graph): Option[User with Entity] = - taskSrv.getByIds(taskId).assignee.has("login", login).headOption + def taskAssignee(taskId: EntityId, userId: EntityId)(implicit graph: Graph): Option[User with Entity] = + taskSrv.getByIds(taskId).assignee.get(userId).headOption } diff --git a/thehive/test/org/thp/thehive/DatabaseBuilder.scala b/thehive/test/org/thp/thehive/DatabaseBuilder.scala index 516d41913b..51767a822f 100644 --- a/thehive/test/org/thp/thehive/DatabaseBuilder.scala +++ b/thehive/test/org/thp/thehive/DatabaseBuilder.scala @@ -5,11 +5,11 @@ import java.io.File import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.scalactic.Or -import org.thp.scalligraph.RichOption import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, Entity, Schema} import org.thp.scalligraph.services.{EdgeSrv, GenIntegrityCheckOps, VertexSrv} +import org.thp.scalligraph.{EntityId, EntityName, RichOption} import org.thp.thehive.models._ import org.thp.thehive.services._ import play.api.Logger @@ -166,16 +166,17 @@ class DatabaseBuilder @Inject() ( try { val data = readFile(path) for { - json <- Json - .parse(data) - .asOpt[JsValue] - .orElse(warn(s"File $data has invalid format")) - .flatMap { - case arr: JsArray => arr.asOpt[Seq[JsObject]].orElse(warn("Array must contain only object")) - case o: JsObject => Some(Seq(o)) - case _ => warn(s"File $data contains data that is not an object nor an array") - } - .getOrElse(Nil) + json <- + Json + .parse(data) + .asOpt[JsValue] + .orElse(warn(s"File $data has invalid format")) + .flatMap { + case arr: JsArray => arr.asOpt[Seq[JsObject]].orElse(warn("Array must contain only object")) + case o: JsObject => Some(Seq(o)) + case _ => warn(s"File $data contains data that is not an object nor an array") + } + .getOrElse(Nil) } yield FObject(json) } catch { case error: Throwable => @@ -185,16 +186,17 @@ class DatabaseBuilder @Inject() ( implicit class RichField(field: Field) { - def getString(path: String): Option[String] = field.get(path) match { - case FString(value) => Some(value) - case _ => None - } + def getString(path: String): Option[String] = + field.get(path) match { + case FString(value) => Some(value) + case _ => None + } } def createVertex[V <: Product]( srv: VertexSrv[V], parser: FieldsParser[V] - )(implicit graph: Graph, authContext: AuthContext): Map[String, String] = + )(implicit graph: Graph, authContext: AuthContext): Map[String, EntityId] = readJsonFile(s"data/${srv.model.label}.json").flatMap { fields => parser(fields - "id") .flatMap(e => Or.from(srv.createEntity(e))) @@ -208,16 +210,16 @@ class DatabaseBuilder @Inject() ( fromSrv: VertexSrv[FROM], toSrv: VertexSrv[TO], parser: FieldsParser[E], - idMap: Map[String, String] + idMap: Map[String, EntityId] )(implicit graph: Graph, authContext: AuthContext): Seq[E with Entity] = readJsonFile(s"data/${srv.model.label}.json") .flatMap { fields => (for { fromExtId <- fields.getString("from").toTry(Failure(new Exception("Edge has no from vertex"))) - fromId = idMap.getOrElse(fromExtId, fromExtId) + fromId = idMap.getOrElse(fromExtId, EntityName(fromExtId)) from <- fromSrv.getOrFail(fromId) toExtId <- fields.getString("to").toTry(Failure(new Exception("Edge has no to vertex"))) - toId = idMap.getOrElse(toExtId, toExtId) + toId = idMap.getOrElse(toExtId, EntityName(toExtId)) to <- toSrv.getOrFail(toId) e <- parser(fields - "from" - "to").fold(e => srv.create(e, from, to), _ => Failure(new Exception("XX"))) } yield e) diff --git a/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala index 2a27b5b513..14a594cb2f 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala @@ -3,6 +3,7 @@ package org.thp.thehive.controllers.v0 import java.util.Date import io.scalaland.chimney.dsl._ +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -274,8 +275,8 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { summary = None, owner = Some("certuser@thehive.local"), customFields = Json.obj( - "boolean1" -> Json.obj("boolean" -> JsNull, "order" -> JsNull), - "string1" -> Json.obj("string" -> "string1 custom field", "order" -> JsNull) + "boolean1" -> Json.obj("boolean" -> JsNull, "order" -> 2), + "string1" -> Json.obj("string" -> "string1 custom field", "order" -> 1) ), stats = Json.obj() ) @@ -283,7 +284,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { TestCase(resultCaseOutput) must_=== expected val observables = app[Database].roTransaction { implicit graph => val authContext = DummyUserSrv(organisation = "cert").authContext - app[CaseSrv].get(resultCaseOutput._id).observables(authContext).richObservable.toList + app[CaseSrv].get(EntityIdOrName(resultCaseOutput._id)).observables(authContext).richObservable.toList } observables must contain( exactly( @@ -300,7 +301,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { "merge an alert with a case" in testApp { app => val request1 = FakeRequest("POST", "/api/v0/alert/testType;testSource;ref5/merge/#1") .withHeaders("user" -> "certuser@thehive.local") - val result1 = app[AlertCtrl].mergeWithCase("testType;testSource;ref5", "#1")(request1) + val result1 = app[AlertCtrl].mergeWithCase("testType;testSource;ref5", "1")(request1) status(result1) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result1)}") val resultCase = contentAsJson(result1) @@ -311,7 +312,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { app[Database].roTransaction { implicit graph => val observables = app .apply[CaseSrv] - .get("#1") + .get(EntityIdOrName("1")) .observables(DummyUserSrv(userId = "certuser@thehive.local", organisation = "cert").getSystemAuthContext) .toList diff --git a/thehive/test/org/thp/thehive/controllers/v0/AuditCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/AuditCtrlTest.scala index 14e7ea923e..82abfb9c27 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/AuditCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/AuditCtrlTest.scala @@ -2,9 +2,9 @@ package org.thp.thehive.controllers.v0 import java.util.Date -import org.thp.scalligraph.AppBuilder import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv} +import org.thp.scalligraph.{AppBuilder, EntityIdOrName} import org.thp.thehive.TestAppBuilder import org.thp.thehive.models.{Case, CaseStatus, Permissions} import org.thp.thehive.services.{CaseSrv, FlowActor, OrganisationSrv} @@ -29,14 +29,14 @@ class AuditCtrlTest extends PlaySpecification with TestAppBuilder { } // Check for no parasite audit - getFlow("#1") must beEmpty + getFlow("1") must beEmpty // Create an event first val `case` = app[Database].tryTransaction { implicit graph => app[CaseSrv].create( Case(0, "case audit", "desc audit", 1, new Date(), None, flag = false, 1, 1, CaseStatus.Open, None), None, - app[OrganisationSrv].getOrFail("admin").get, + app[OrganisationSrv].getOrFail(EntityIdOrName("admin")).get, Set.empty, Seq.empty, None, @@ -45,7 +45,7 @@ class AuditCtrlTest extends PlaySpecification with TestAppBuilder { }.get // Get the actual data - val l = getFlow(`case`._id) + val l = getFlow(`case`._id.toString) // l must not(beEmpty) pending diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala index d5b4a91a86..e36be809b7 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala @@ -4,6 +4,7 @@ import java.util.Date import akka.stream.Materializer import io.scalaland.chimney.dsl._ +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -90,7 +91,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { owner = Some("certuser@thehive.local"), customFields = Json.obj( "boolean1" -> Json.obj("boolean" -> true, "order" -> JsNull), - "string1" -> Json.obj("string" -> "string1 custom field", "order" -> JsNull), + "string1" -> Json.obj("string" -> "string1 custom field", "order" -> 1), "date1" -> Json.obj("date" -> now.getTime, "order" -> JsNull) ), stats = Json.obj() @@ -152,7 +153,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { val tasksList = contentAsJson(resultList)(defaultAwaitTimeout, app[Materializer]).as[Seq[OutputTask]] tasksList.find(_.title == "task x") must beSome - val assignee = app[Database].roTransaction(implicit graph => app[CaseSrv].get(outputCase._id).assignee.getOrFail("Case")) + val assignee = app[Database].roTransaction(implicit graph => app[CaseSrv].get(EntityIdOrName(outputCase._id)).assignee.getOrFail("Case")) assignee must beSuccessfulTry assignee.get.login shouldEqual "certuser@thehive.local" @@ -161,11 +162,11 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { "try to get a case" in testApp { app => val request = FakeRequest("GET", s"/api/v0/case/#2") .withHeaders("user" -> "certuser@thehive.local") - val result = app[CaseCtrl].get("#145")(request) + val result = app[CaseCtrl].get("145")(request) status(result) shouldEqual 404 - val result2 = app[CaseCtrl].get("#2")(request) + val result2 = app[CaseCtrl].get("2")(request) status(result2) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result2)}") val resultCase = contentAsJson(result2) val resultCaseOutput = resultCase.as[OutputCase] @@ -192,7 +193,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { } "update a case properly" in testApp { app => - val request = FakeRequest("PATCH", s"/api/v0/case/#1") + val request = FakeRequest("PATCH", s"/api/v0/case/1") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody( Json.obj( @@ -200,7 +201,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { "flag" -> true ) ) - val result = app[CaseCtrl].update("#1")(request) + val result = app[CaseCtrl].update("1")(request) status(result) must_=== 200 val resultCase = contentAsJson(result).as[OutputCase] @@ -213,7 +214,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { .withHeaders("user" -> "certuser@thehive.local") .withJsonBody( Json.obj( - "ids" -> List("#1", "#2"), + "ids" -> List("1", "2"), "description" -> "new description", "tlp" -> 1, "pap" -> 1 @@ -228,15 +229,15 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { resultCases.map(_.tlp) must contain(be_==(1)).forall resultCases.map(_.pap) must contain(be_==(1)).forall - val requestGet1 = FakeRequest("GET", s"/api/v0/case/#1") + val requestGet1 = FakeRequest("GET", s"/api/v0/case/1") .withHeaders("user" -> "certuser@thehive.local") - val resultGet1 = app[CaseCtrl].get("#1")(requestGet1) + val resultGet1 = app[CaseCtrl].get("1")(requestGet1) status(resultGet1) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(resultGet1)}") val case1 = contentAsJson(resultGet1).as[OutputCase] - val requestGet2 = FakeRequest("GET", s"/api/v0/case/#2") + val requestGet2 = FakeRequest("GET", s"/api/v0/case/2") .withHeaders("user" -> "certuser@thehive.local") - val resultGet2 = app[CaseCtrl].get("#2")(requestGet2) + val resultGet2 = app[CaseCtrl].get("2")(requestGet2) status(resultGet2) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(resultGet2)}") val case3 = contentAsJson(resultGet2).as[OutputCase] @@ -333,10 +334,10 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { } "assign a case to an user" in testApp { app => - val request = FakeRequest("PATCH", s"/api/v0/case/#4") + val request = FakeRequest("PATCH", s"/api/v0/case/4") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.obj("owner" -> "certro@thehive.local")) - val result = app[CaseCtrl].update("#1")(request) + val result = app[CaseCtrl].update("1")(request) status(result) must beEqualTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") val resultCase = contentAsJson(result) val resultCaseOutput = resultCase.as[OutputCase] @@ -347,17 +348,17 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { "force delete a case" in testApp { app => val tasks = app[Database].roTransaction { implicit graph => val authContext = DummyUserSrv(organisation = "cert").authContext - app[CaseSrv].get("#1").tasks(authContext).toSeq + app[CaseSrv].get(EntityIdOrName("1")).tasks(authContext).toSeq } tasks must have size 2 val requestDel = FakeRequest("DELETE", s"/api/v0/case/#1/force") .withHeaders("user" -> "certuser@thehive.local") - val resultDel = app[CaseCtrl].realDelete("#1")(requestDel) + val resultDel = app[CaseCtrl].realDelete("1")(requestDel) status(resultDel) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(resultDel)}") app[Database].roTransaction { implicit graph => - app[CaseSrv].get("#1").headOption must beNone + app[CaseSrv].get(EntityIdOrName("1")).headOption must beNone // tasks.flatMap(task => app[TaskSrv].get(task).headOption) must beEmpty } } diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala index deb4d29c5b..a2bba30c28 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.controllers.v0 +import org.thp.scalligraph.EntityName import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -104,7 +105,7 @@ class CaseTemplateCtrlTest extends PlaySpecification with TestAppBuilder { status(result) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") app[Database].roTransaction { implicit graph => - app[CaseTemplateSrv].get("spam").headOption must beNone + app[CaseTemplateSrv].get(EntityName("spam")).headOption must beNone } } @@ -133,7 +134,7 @@ class CaseTemplateCtrlTest extends PlaySpecification with TestAppBuilder { status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") val updatedOutput = app[Database].roTransaction { implicit graph => - app[CaseTemplateSrv].get("spam").richCaseTemplate.head + app[CaseTemplateSrv].get(EntityName("spam")).richCaseTemplate.head } updatedOutput.displayName shouldEqual "patched" diff --git a/thehive/test/org/thp/thehive/controllers/v0/CustomFieldCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CustomFieldCtrlTest.scala index 31d532daa1..9989b9de2b 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CustomFieldCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CustomFieldCtrlTest.scala @@ -162,7 +162,7 @@ class CustomFieldCtrlTest extends PlaySpecification with TestAppBuilder { val request = FakeRequest("DELETE", s"/api/customField/${cf._id}") .withHeaders("user" -> "admin@thehive.local") - val result = app[CustomFieldCtrl].delete(cf._id)(request) + val result = app[CustomFieldCtrl].delete(cf._id.toString)(request) status(result) shouldEqual 204 @@ -192,7 +192,7 @@ class CustomFieldCtrlTest extends PlaySpecification with TestAppBuilder { } """.stripMargin)) - val result = app[CustomFieldCtrl].update(cf.get._id)(request) + val result = app[CustomFieldCtrl].update(cf.get._id.toString)(request) status(result) shouldEqual 200 @@ -224,7 +224,7 @@ class CustomFieldCtrlTest extends PlaySpecification with TestAppBuilder { } """.stripMargin)) - val result = app[CustomFieldCtrl].update(cf.get._id)(request) + val result = app[CustomFieldCtrl].update(cf.get._id.toString)(request) status(result) shouldEqual 200 diff --git a/thehive/test/org/thp/thehive/controllers/v0/DashboardCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/DashboardCtrlTest.scala index 360db804df..145aa1fdd9 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/DashboardCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/DashboardCtrlTest.scala @@ -29,31 +29,31 @@ class DashboardCtrlTest extends PlaySpecification with TestAppBuilder { "get a dashboard if visible" in testApp { app => val dashboard = app[Database].roTransaction { implicit graph => - app[DashboardSrv].startTraversal.has("title", "dashboard cert").getOrFail("Dashboard").get + app[DashboardSrv].startTraversal.has(_.title, "dashboard cert").getOrFail("Dashboard").get } val request = FakeRequest("GET", s"/api/dashboard/${dashboard._id}") .withHeaders("user" -> "certuser@thehive.local") - val result = app[DashboardCtrl].get(dashboard._id)(request) + val result = app[DashboardCtrl].get(dashboard._id.toString)(request) status(result) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") val requestFailed = FakeRequest("GET", s"/api/dashboard/${dashboard._id}") .withHeaders("user" -> "socuser@thehive.local") - val resultFailed = app[DashboardCtrl].get(dashboard._id)(requestFailed) + val resultFailed = app[DashboardCtrl].get(dashboard._id.toString)(requestFailed) status(resultFailed) must equalTo(404).updateMessage(s => s"$s\n${contentAsString(resultFailed)}") } "update a dashboard" in testApp { app => val dashboard = app[Database].roTransaction { implicit graph => - app[DashboardSrv].startTraversal.has("title", "dashboard cert").getOrFail("Dashboard").get + app[DashboardSrv].startTraversal.has(_.title, "dashboard cert").getOrFail("Dashboard").get } val request = FakeRequest("PATCH", s"/api/dashboard/${dashboard._id}") .withHeaders("user" -> "certadmin@thehive.local") .withJsonBody(Json.parse("""{"title": "updated", "description": "updated", "status": "Private", "definition": "{}"}""")) - val result = app[DashboardCtrl].update(dashboard._id)(request) + val result = app[DashboardCtrl].update(dashboard._id.toString)(request) status(result) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") @@ -67,17 +67,17 @@ class DashboardCtrlTest extends PlaySpecification with TestAppBuilder { "delete a dashboard" in testApp { app => val dashboard = app[Database].roTransaction { implicit graph => - app[DashboardSrv].startTraversal.has("title", "dashboard cert").getOrFail("Dashboard").get + app[DashboardSrv].startTraversal.has(_.title, "dashboard cert").getOrFail("Dashboard").get } val request = FakeRequest("DELETE", s"/api/dashboard/${dashboard._id}") .withHeaders("user" -> "certadmin@thehive.local") - val result = app[DashboardCtrl].delete(dashboard._id)(request) + val result = app[DashboardCtrl].delete(dashboard._id.toString)(request) status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") app[Database].roTransaction { implicit graph => - app[DashboardSrv].startTraversal.has("title", "dashboard cert").exists must beFalse + app[DashboardSrv].startTraversal.has(_.title, "dashboard cert").exists must beFalse } } } diff --git a/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala index 308f9e65cd..c76dc02c1a 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/ObservableCtrlTest.scala @@ -44,7 +44,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "observable controller" should { "be able to create an observable with string data" in testApp { app => - val request = FakeRequest("POST", s"/api/case/#1/artifact") + val request = FakeRequest("POST", s"/api/case/1/artifact") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.parse(""" { @@ -57,7 +57,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "data":["multi","line","test"] } """.stripMargin)) - val result = app[ObservableCtrl].create("#1")(request) + val result = app[ObservableCtrl].create("1")(request) status(result) must equalTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") val createdObservables = contentAsJson(result).as[Seq[OutputObservable]] @@ -71,7 +71,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { } "be able to create and search 2 observables with data array" in testApp { app => - val request = FakeRequest("POST", s"/api/case/#1/artifact") + val request = FakeRequest("POST", s"/api/case/1/artifact") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.parse(""" { @@ -84,7 +84,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "data":["observable", "in", "array"] } """.stripMargin)) - val result = app[ObservableCtrl].create("#1")(request) + val result = app[ObservableCtrl].create("1")(request) status(result) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") @@ -98,8 +98,8 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { createdObservables.map(_.tags) must contain(be_==(Set("lol", "tagfile"))).forall val requestCase = - FakeRequest("GET", s"/api/v0/case/#1").withHeaders("user" -> "certuser@thehive.local") - val resultCaseGet = app[CaseCtrl].get("#1")(requestCase) + FakeRequest("GET", s"/api/v0/case/1").withHeaders("user" -> "certuser@thehive.local") + val resultCaseGet = app[CaseCtrl].get("1")(requestCase) status(resultCaseGet) shouldEqual 200 @@ -160,7 +160,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { Headers("user" -> "certuser@thehive.local"), body = AnyContentAsMultipartFormData(MultipartFormData(dataParts, files, Nil)) ) - val result = app[ObservableCtrl].create("#1")(request) + val result = app[ObservableCtrl].create("1")(request) status(result) must equalTo(201).updateMessage(s => s"$s\n${contentAsString(result)}") val createdObservables = contentAsJson(result).as[Seq[OutputObservable]] @@ -210,7 +210,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { } "create 2 observables with the same data" in testApp { app => - val request1 = FakeRequest("POST", s"/api/case/#1/artifact") + val request1 = FakeRequest("POST", s"/api/case/1/artifact") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.parse(""" { @@ -219,12 +219,12 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "data":"localhost" } """)) - val result1 = app[ObservableCtrl].create("#1")(request1) + val result1 = app[ObservableCtrl].create("1")(request1) status(result1) must beEqualTo(201).updateMessage(s => s"$s\n${contentAsString(result1)}") getData("localhost", app) must have size 1 - val request2 = FakeRequest("POST", s"/api/case/#2/artifact") + val request2 = FakeRequest("POST", s"/api/case/2/artifact") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.parse(""" { @@ -233,7 +233,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "data":"localhost" } """)) - val result2 = app[ObservableCtrl].create("#2")(request2) + val result2 = app[ObservableCtrl].create("2")(request2) status(result2) must equalTo(201).updateMessage(s => s"$s\n${contentAsString(result2)}") getData("localhost", app) must have size 1 @@ -260,7 +260,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { } def createDummyObservable(observableCtrl: ObservableCtrl): Seq[OutputObservable] = { - val request = FakeRequest("POST", s"/api/case/#1/artifact") + val request = FakeRequest("POST", s"/api/case/1/artifact") .withHeaders("user" -> "certuser@thehive.local") .withJsonBody(Json.parse(s""" { @@ -273,7 +273,7 @@ class ObservableCtrlTest extends PlaySpecification with TestAppBuilder { "data":"${UUID.randomUUID()}\\n${UUID.randomUUID()}" } """)) - val result = observableCtrl.create("#1")(request) + val result = observableCtrl.create("1")(request) status(result) shouldEqual 201 contentAsJson(result).as[Seq[OutputObservable]] diff --git a/thehive/test/org/thp/thehive/controllers/v0/OrganisationCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/OrganisationCtrlTest.scala index 579028de13..4df0d83472 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/OrganisationCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/OrganisationCtrlTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.controllers.v0 +import org.thp.scalligraph.EntityName import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -121,7 +122,7 @@ class OrganisationCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[OrganisationCtrl].update("cert")(request) status(result) must beEqualTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") app[Database].roTransaction { implicit graph => - app[OrganisationSrv].get("cert2").exists must beTrue + app[OrganisationSrv].get(EntityName("cert2")).exists must beTrue } } diff --git a/thehive/test/org/thp/thehive/controllers/v0/PageCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/PageCtrlTest.scala index 902e495484..fe6783469a 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/PageCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/PageCtrlTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.controllers.v0 +import org.thp.scalligraph.EntityName import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -69,7 +70,7 @@ class PageCtrlTest extends PlaySpecification with TestAppBuilder { status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") app[Database].roTransaction { implicit graph => - app[PageSrv].get("how_to_create_a_case").exists + app[PageSrv].get(EntityName("how_to_create_a_case")).exists } must beFalse } } diff --git a/thehive/test/org/thp/thehive/controllers/v0/ProfileCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/ProfileCtrlTest.scala index c01c0941dd..eed5872234 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/ProfileCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/ProfileCtrlTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.controllers.v0 +import org.thp.scalligraph.EntityName import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -58,7 +59,7 @@ class ProfileCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[ProfileCtrl].delete("testProfile")(request) status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") app[Database].roTransaction { implicit graph => - app[ProfileSrv].get("testProfile").exists must beFalse + app[ProfileSrv].get(EntityName("testProfile")).exists must beFalse } } diff --git a/thehive/test/org/thp/thehive/controllers/v0/ShareCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/ShareCtrlTest.scala index 0f8b75e3ae..df32702057 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/ShareCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/ShareCtrlTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.controllers.v0 +import org.thp.scalligraph.EntityName import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -12,45 +13,45 @@ import play.api.test.{FakeRequest, PlaySpecification} class ShareCtrlTest extends PlaySpecification with TestAppBuilder { "share a case" in testApp { app => - val request = FakeRequest("POST", "/api/case/#1/shares") + val request = FakeRequest("POST", "/api/case/1/shares") .withJsonBody(Json.obj("shares" -> List(Json.toJson(InputShare("soc", Profile.orgAdmin.name, TasksFilter.all, ObservablesFilter.all))))) .withHeaders("user" -> "certuser@thehive.local") - val result = app[ShareCtrl].shareCase("#1")(request) + val result = app[ShareCtrl].shareCase("1")(request) status(result) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") app[Database].roTransaction { implicit graph => - app[CaseSrv].get("#1").visible(DummyUserSrv(organisation = "soc").authContext).exists + app[CaseSrv].get(EntityName("1")).visible(DummyUserSrv(organisation = "soc").authContext).exists } must beTrue } "fail to share a already share case" in testApp { app => - val request = FakeRequest("POST", "/api/case/#2/shares") + val request = FakeRequest("POST", "/api/case/2/shares") .withJsonBody(Json.obj("shares" -> Seq(Json.toJson(InputShare("soc", Profile.orgAdmin.name, TasksFilter.all, ObservablesFilter.all))))) .withHeaders("user" -> "certuser@thehive.local") - val result = app[ShareCtrl].shareCase("#2")(request) + val result = app[ShareCtrl].shareCase("2")(request) status(result) must equalTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") } "remove a share" in testApp { app => - val request = FakeRequest("DELETE", s"/api/case/#2") + val request = FakeRequest("DELETE", s"/api/case/2") .withJsonBody(Json.obj("organisations" -> Seq("soc"))) .withHeaders("user" -> "certuser@thehive.local") - val result = app[ShareCtrl].removeShares("#2")(request) + val result = app[ShareCtrl].removeShares("2")(request) status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") app[Database].roTransaction { implicit graph => - app[CaseSrv].get("#2").visible(DummyUserSrv(userId = "socro@thehive.local").authContext).exists + app[CaseSrv].get(EntityName("2")).visible(DummyUserSrv(userId = "socro@thehive.local").authContext).exists } must beFalse } "refuse to remove owner share" in testApp { app => - val request = FakeRequest("DELETE", s"/api/case/#2") + val request = FakeRequest("DELETE", s"/api/case/2") .withJsonBody(Json.obj("organisations" -> Seq("cert"))) .withHeaders("user" -> "certuser@thehive.local") - val result = app[ShareCtrl].removeShares("#2")(request) + val result = app[ShareCtrl].removeShares("2")(request) status(result) must equalTo(400).updateMessage(s => s"$s\n${contentAsString(result)}") } diff --git a/thehive/test/org/thp/thehive/controllers/v0/StreamCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/StreamCtrlTest.scala index ed4350db4f..3fc3885aa5 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/StreamCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/StreamCtrlTest.scala @@ -2,6 +2,7 @@ package org.thp.thehive.controllers.v0 import java.util.Date +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.thehive.TestAppBuilder @@ -35,7 +36,7 @@ class StreamCtrlTest extends PlaySpecification with TestAppBuilder { app[CaseSrv].create( Case(0, s"case audit", s"desc audit", 1, new Date(), None, flag = false, 1, 1, CaseStatus.Open, None), None, - app[OrganisationSrv].getOrFail("cert").get, + app[OrganisationSrv].getOrFail(EntityName("cert")).get, Set.empty, Seq.empty, None, diff --git a/thehive/test/org/thp/thehive/controllers/v0/TagCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/TagCtrlTest.scala index 5c9eee3a40..3d143dee34 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/TagCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/TagCtrlTest.scala @@ -126,7 +126,7 @@ class TagCtrlTest extends PlaySpecification with TestAppBuilder { val request = FakeRequest("GET", s"/api/tag/${tag._id}") .withHeaders("user" -> "certuser@thehive.local") - val result = app[TagCtrl].get(tag._id)(request) + val result = app[TagCtrl].get(tag._id.toString)(request) status(result) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") } diff --git a/thehive/test/org/thp/thehive/controllers/v0/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/UserCtrlTest.scala index a9dc3a8c2d..af62fcae55 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/UserCtrlTest.scala @@ -1,9 +1,9 @@ package org.thp.thehive.controllers.v0 import akka.stream.Materializer -import org.thp.scalligraph.AuthenticationError import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{AuthenticationError, EntityName} import org.thp.thehive.TestAppBuilder import org.thp.thehive.dto.v0.OutputUser import org.thp.thehive.services.UserSrv @@ -144,7 +144,7 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { status(result) must beEqualTo(204) app[Database].roTransaction { implicit graph => - app[UserSrv].get("certro@thehive.local").exists + app[UserSrv].get(EntityName("certro@thehive.local")).exists } must beFalse } } diff --git a/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala index d0f65a2559..d12bc94044 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/AlertCtrlTest.scala @@ -75,7 +75,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { pap = 2, read = false, follow = true, - customFields = Set.empty, + customFields = Seq.empty, caseTemplate = None, observableCount = 0L, caseId = None, @@ -123,7 +123,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { pap = 2, read = false, follow = true, - customFields = Set.empty, + customFields = Seq.empty, caseTemplate = Some("spam"), observableCount = 0L, caseId = None, @@ -139,7 +139,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { alertSrv.startTraversal.has(_.sourceRef, "ref1").getOrFail("Alert") } must beSuccessfulTry.which { alert: Alert with Entity => val request = FakeRequest("GET", s"/api/v1/alert/${alert._id}").withHeaders("user" -> "socuser@thehive.local") - val result = app[AlertCtrl].get(alert._id)(request) + val result = app[AlertCtrl].get(alert._id.toString)(request) status(result) must_=== 200 val resultAlert = contentAsJson(result).as[OutputAlert] val expected = TestAlert( diff --git a/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala index 14003736b9..be0ab07cb3 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala @@ -4,7 +4,7 @@ import java.util.Date import org.thp.thehive.TestAppBuilder import org.thp.thehive.dto.v1.{InputCase, OutputCase, OutputCustomFieldValue} -import play.api.libs.json.{JsNull, JsString, Json} +import play.api.libs.json.{JsNull, JsString, JsValue, Json} import play.api.test.{FakeRequest, PlaySpecification} case class TestCase( @@ -20,11 +20,10 @@ case class TestCase( status: String, summary: Option[String] = None, user: Option[String], - customFields: Set[OutputCustomFieldValue] = Set.empty + customFields: Seq[TestCustomFieldValue] = Seq.empty ) object TestCase { - def apply(outputCase: OutputCase): TestCase = TestCase( outputCase.title, @@ -39,7 +38,20 @@ object TestCase { outputCase.status, outputCase.summary, outputCase.assignee, - outputCase.customFields + outputCase.customFields.map(TestCustomFieldValue.apply).sortBy(_.order) + ) +} + +case class TestCustomFieldValue(name: String, description: String, `type`: String, value: JsValue, order: Int) + +object TestCustomFieldValue { + def apply(outputCustomFieldValue: OutputCustomFieldValue): TestCustomFieldValue = + TestCustomFieldValue( + outputCustomFieldValue.name, + outputCustomFieldValue.description, + outputCustomFieldValue.`type`, + outputCustomFieldValue.value, + outputCustomFieldValue.order ) } @@ -79,7 +91,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { status = "Open", summary = None, user = Some("certuser@thehive.local"), - customFields = Set.empty + customFields = Seq.empty ) TestCase(resultCase) must_=== expected @@ -119,9 +131,9 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { status = "Open", summary = None, user = Some("certuser@thehive.local"), - customFields = Set( - OutputCustomFieldValue("boolean1", "boolean custom field", "boolean", JsNull, 0), - OutputCustomFieldValue("string1", "string custom field", "string", JsString("string1 custom field"), 0) + customFields = Seq( + TestCustomFieldValue("string1", "string custom field", "string", JsString("string1 custom field"), 1), + TestCustomFieldValue("boolean1", "boolean custom field", "boolean", JsNull, 2) ) ) @@ -129,9 +141,9 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { } "get a case" in testApp { app => - val request = FakeRequest("GET", s"/api/v1/case/#1") + val request = FakeRequest("GET", s"/api/v1/case/1") .withHeaders("user" -> "certuser@thehive.local") - val result = app[CaseCtrl].get("#1")(request) + val result = app[CaseCtrl].get("1")(request) val resultCase = contentAsJson(result).as[OutputCase] val expected = TestCase( title = "case#1", diff --git a/thehive/test/org/thp/thehive/controllers/v1/OrganisationCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/OrganisationCtrlTest.scala index 3e7b31df63..6fa3027b93 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/OrganisationCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/OrganisationCtrlTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.controllers.v1 +import org.thp.scalligraph.EntityName import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder @@ -58,7 +59,7 @@ class OrganisationCtrlTest extends PlaySpecification with TestAppBuilder { val result = app[OrganisationCtrl].update("cert")(request) status(result) must_=== 204 app[Database].roTransaction { implicit graph => - app[OrganisationSrv].get("cert2").exists must beTrue + app[OrganisationSrv].get(EntityName("cert2")).exists must beTrue } } diff --git a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala index 0cf0ef0f10..e7ac8f762c 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala @@ -126,9 +126,9 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { val expected = TestUser( login = "socuser@thehive.local", name = "socuser", - profile = "", - permissions = Set.empty, - organisation = "cert" + profile = "analyst", + permissions = Profile.analyst.permissions.map(_.toString), + organisation = "soc" ) TestUser(resultCase) must_=== expected diff --git a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala index a5a1a3ed6a..f63bc0e3e6 100644 --- a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala @@ -2,7 +2,7 @@ package org.thp.thehive.services import java.util.Date -import org.thp.scalligraph.CreateError +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -37,10 +37,10 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { read = false, follow = false ), - app[OrganisationSrv].getOrFail("cert").get, + app[OrganisationSrv].getOrFail(EntityName("cert")).get, Set("tag1", "tag2"), Map("string1" -> Some("lol")), - Some(app[CaseTemplateSrv].getOrFail("spam").get) + Some(app[CaseTemplateSrv].getOrFail(EntityName("spam")).get) ) } a must beSuccessfulTry.which { a => @@ -55,7 +55,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { } app[Database].roTransaction { implicit graph => - app[OrganisationSrv].get("cert").alerts.toList must contain(a.get.alert) + app[OrganisationSrv].get(EntityName("cert")).alerts.toList must contain(a.get.alert) val tags = app[TagSrv].startTraversal.toSeq.filter(t => t.predicate == "tag1" || t.predicate == "tag2") @@ -66,11 +66,11 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "update tags" in testApp { app => val newTags = app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) tag3 <- app[TagSrv].getOrCreate("tag3") tag5 <- app[TagSrv].getOrCreate("tag5") _ <- app[AlertSrv].updateTags(alert, Set(tag3, tag5)) - } yield app[AlertSrv].get("testType;testSource;ref1").tags.toSeq + } yield app[AlertSrv].get(EntityName("testType;testSource;ref1")).tags.toSeq } newTags must beSuccessfulTry.which(t => t.map(_.toString) must contain(exactly("tag3", "tag5"))) } @@ -78,9 +78,9 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "update tag names" in testApp { app => val tags = app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) _ <- app[AlertSrv].updateTagNames(alert, Set("tag3", "tag5")) - } yield app[AlertSrv].get("testType;testSource;ref1").tags.toSeq + } yield app[AlertSrv].get(EntityName("testType;testSource;ref1")).tags.toSeq } tags must beSuccessfulTry.which(t => t.map(_.toString) must contain(exactly("tag3", "tag5"))) } @@ -88,9 +88,9 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "add tags" in testApp { app => val tags = app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) _ <- app[AlertSrv].addTags(alert, Set("tag7")) - } yield app[AlertSrv].get("testType;testSource;ref1").tags.toSeq + } yield app[AlertSrv].get(EntityName("testType;testSource;ref1")).tags.toSeq } tags must beSuccessfulTry.which(t => @@ -101,7 +101,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "add an observable if not existing" in testApp { app => val similarObs = app[Database].tryTransaction { implicit graph => for { - observableType <- app[ObservableTypeSrv].getOrFail("domain") + observableType <- app[ObservableTypeSrv].getOrFail(EntityName("domain")) observable <- app[ObservableSrv].create( observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true), `type` = observableType, @@ -114,42 +114,49 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].getOrFail("testType;testSource;ref4") + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref4")) _ <- app[AlertSrv].addObservable(alert, similarObs) } yield () - } must beASuccessfulTry() + } must beASuccessfulTry app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) _ <- app[AlertSrv].addObservable(alert, similarObs) } yield () } must beASuccessfulTry app[Database].roTransaction { implicit graph => - app[AlertSrv].get("testType;testSource;ref1").observables.filterOnData("perdu.com").filterOnType("domain").tags.toSeq.map(_.toString) + app[AlertSrv] + .get(EntityName("testType;testSource;ref1")) + .observables + .filterOnData("perdu.com") + .filterOnType("domain") + .tags + .toSeq + .map(_.toString) } must contain("tag10") } "update custom fields" in testApp { app => app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") - cfv <- app[CustomFieldSrv].getOrFail("string1") + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) + cfv <- app[CustomFieldSrv].getOrFail(EntityName("string1")) _ <- app[AlertSrv].updateCustomField(alert, Seq((cfv, JsString("sad")))) } yield () } must beSuccessfulTry app[Database].roTransaction { implicit graph => - app[AlertSrv].get("testType;testSource;ref1").customFields("string1").nameJsonValue.headOption + app[AlertSrv].get(EntityName("testType;testSource;ref1")).customFields("string1").nameJsonValue.headOption } must beSome("string1" -> JsString("sad")) } "mark as read an alert" in testApp { app => app[Database].tryTransaction { implicit graph => for { - _ <- app[AlertSrv].markAsRead("testType;testSource;ref1") - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + _ <- app[AlertSrv].markAsRead(EntityName("testType;testSource;ref1")) + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) } yield alert.read } must beASuccessfulTry(true) } @@ -157,8 +164,8 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "mark as unread an alert" in testApp { app => app[Database].tryTransaction { implicit graph => for { - _ <- app[AlertSrv].markAsUnread("testType;testSource;ref1") - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + _ <- app[AlertSrv].markAsUnread(EntityName("testType;testSource;ref1")) + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) } yield alert.read } must beASuccessfulTry(false) } @@ -166,8 +173,8 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "mark as follow an alert" in testApp { app => app[Database].tryTransaction { implicit graph => for { - _ <- app[AlertSrv].followAlert("testType;testSource;ref1") - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + _ <- app[AlertSrv].followAlert(EntityName("testType;testSource;ref1")) + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) } yield alert.follow } must beASuccessfulTry(true) } @@ -175,8 +182,8 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "mark as unfollow an alert" in testApp { app => app[Database].tryTransaction { implicit graph => for { - _ <- app[AlertSrv].unfollowAlert("testType;testSource;ref1") - alert <- app[AlertSrv].getOrFail("testType;testSource;ref1") + _ <- app[AlertSrv].unfollowAlert(EntityName("testType;testSource;ref1")) + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) } yield alert.follow } must beASuccessfulTry(false) } @@ -184,11 +191,11 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "create a case" in testApp { app => app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].get("testType;testSource;ref1").richAlert.getOrFail("Alert") - organisation <- app[OrganisationSrv].getOrFail("cert") + alert <- app[AlertSrv].get(EntityName("testType;testSource;ref1")).richAlert.getOrFail("Alert") + organisation <- app[OrganisationSrv].getOrFail(EntityName("cert")) c <- app[AlertSrv].createCase(alert, None, organisation) _ = c.title must beEqualTo("[SPAM] alert#1") - _ <- app[CaseSrv].startTraversal.has("title", "[SPAM] alert#1").getOrFail("Alert") + _ <- app[CaseSrv].startTraversal.has(_.title, "[SPAM] alert#1").getOrFail("Alert") } yield () } must beASuccessfulTry(()) } @@ -196,11 +203,11 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "merge into an existing case" in testApp { app => app[Database] .tryTransaction { implicit graph => - app[AlertSrv].mergeInCase("testType;testSource;ref1", "#1") + app[AlertSrv].mergeInCase(EntityName("testType;testSource;ref1"), EntityName("1")) } must beASuccessfulTry app[Database].roTransaction { implicit graph => - val observables = app[CaseSrv].get("#1").observables.richObservable.toList + val observables = app[CaseSrv].get(EntityName("1")).observables.richObservable.toList observables must have size 1 observables must contain { (o: RichObservable) => o.data must beSome.which((_: Data).data must beEqualTo("h.fr")) @@ -212,13 +219,13 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { "remove totally an alert" in testApp { app => app[Database].tryTransaction { implicit graph => for { - alert <- app[AlertSrv].getOrFail("testType;testSource;ref4") + alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref4")) _ <- app[AlertSrv].remove(alert) } yield () } must beSuccessfulTry app[Database].roTransaction { implicit graph => // app[ObservableSrv].initSteps.filterOnType("domain").filterOnData("perdu.com").exists must beFalse - app[AlertSrv].startTraversal.get("testType;testSource;ref4").exists must beFalse + app[AlertSrv].startTraversal.get(EntityName("testType;testSource;ref4")).exists must beFalse } } } diff --git a/thehive/test/org/thp/thehive/services/AttachmentSrvTest.scala b/thehive/test/org/thp/thehive/services/AttachmentSrvTest.scala index 15cc6cf4c4..26bd3bf412 100644 --- a/thehive/test/org/thp/thehive/services/AttachmentSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AttachmentSrvTest.scala @@ -4,6 +4,7 @@ import java.io.{File, InputStream} import java.nio.file.{Path, Files => JFiles} import java.util.UUID +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models._ @@ -59,7 +60,7 @@ class AttachmentSrvTest extends PlaySpecification with TestAppBuilder { allAttachments must not(beEmpty) app[Database].roTransaction { implicit graph => - app[AttachmentSrv].get(allAttachments.head.attachmentId).exists must beTrue + app[AttachmentSrv].get(EntityName(allAttachments.head.attachmentId)).exists must beTrue } } } diff --git a/thehive/test/org/thp/thehive/services/AuditSrvTest.scala b/thehive/test/org/thp/thehive/services/AuditSrvTest.scala index 65c47fb0a1..5ff0a6089e 100644 --- a/thehive/test/org/thp/thehive/services/AuditSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AuditSrvTest.scala @@ -3,6 +3,7 @@ package org.thp.thehive.services import java.util.Date import org.apache.tinkerpop.gremlin.process.traversal.Order +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -17,7 +18,7 @@ class AuditSrvTest extends PlaySpecification with TestAppBuilder { "get main audits by ids and sorted" in testApp { app => app[Database].roTransaction { implicit graph => // Create 3 case events first - val orgAdmin = app[OrganisationSrv].getOrFail("admin").get + val orgAdmin = app[OrganisationSrv].getOrFail(EntityName("admin")).get val c1 = app[Database] .tryTransaction(implicit graph => app[CaseSrv].create( diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index cccad04499..2843c291ac 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -3,12 +3,12 @@ package org.thp.thehive.services import java.util.Date import org.specs2.matcher.Matcher -import org.thp.scalligraph.CreateError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.FPathElem import org.thp.scalligraph.models._ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{CreateError, EntityName} import org.thp.thehive.TestAppBuilder import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ @@ -31,7 +31,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "get a case without impact status" in testApp { app => app[Database].roTransaction { implicit graph => - val richCase = app[CaseSrv].get("#1").richCase.head + val richCase = app[CaseSrv].get(EntityName("1")).richCase.head richCase must_== RichCase( richCase._id, authContext.userId, @@ -71,7 +71,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "get a case with impact status" in testApp { app => app[Database].roTransaction { implicit graph => - val richCase = app[CaseSrv].get("#2").richCase.head + val richCase = app[CaseSrv].get(EntityName("2")).richCase.head richCase must_== RichCase( richCase._id, authContext.userId, @@ -112,7 +112,8 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "get a case with custom fields" in testApp { app => app[Database].roTransaction { implicit graph => - val richCase = app[CaseSrv].get("#3").richCase(DummyUserSrv(userId = "socuser@thehive.local", organisation = "soc").authContext).head + val richCase = + app[CaseSrv].get(EntityName("3")).richCase(DummyUserSrv(userId = "socuser@thehive.local", organisation = "soc").authContext).head richCase.number must_=== 3 richCase.title must_=== "case#3" richCase.description must_=== "description of case #3" @@ -169,34 +170,34 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "add custom field with wrong type" in testApp { app => app[Database].transaction { implicit graph => - app[CaseSrv].getOrFail("#3") must beSuccessfulTry.which { `case`: Case with Entity => - app[CaseSrv].setOrCreateCustomField(`case`, "boolean1", Some("plop"), None) must beFailedTry + app[CaseSrv].getOrFail(EntityName("3")) must beSuccessfulTry.which { `case`: Case with Entity => + app[CaseSrv].setOrCreateCustomField(`case`, EntityName("boolean1"), Some("plop"), None) must beFailedTry } } } "add custom field" in testApp { app => app[Database].transaction { implicit graph => - app[CaseSrv].getOrFail("#3") must beSuccessfulTry.which { `case`: Case with Entity => - app[CaseSrv].setOrCreateCustomField(`case`, "boolean1", Some(true), None) must beSuccessfulTry - app[CaseSrv].getCustomField(`case`, "boolean1").flatMap(_.value) must beSome.which(_ == true) + app[CaseSrv].getOrFail(EntityName("3")) must beSuccessfulTry.which { `case`: Case with Entity => + app[CaseSrv].setOrCreateCustomField(`case`, EntityName("boolean1"), Some(true), None) must beSuccessfulTry + app[CaseSrv].getCustomField(`case`, EntityName("boolean1")).flatMap(_.value) must beSome.which(_ == true) } } } "update custom field" in testApp { app => app[Database].transaction { implicit graph => - app[CaseSrv].getOrFail("#3") must beSuccessfulTry.which { `case`: Case with Entity => - app[CaseSrv].setOrCreateCustomField(`case`, "boolean1", Some(false), None) must beSuccessfulTry - app[CaseSrv].getCustomField(`case`, "boolean1").flatMap(_.value) must beSome.which(_ == false) + app[CaseSrv].getOrFail(EntityName("3")) must beSuccessfulTry.which { `case`: Case with Entity => + app[CaseSrv].setOrCreateCustomField(`case`, EntityName("boolean1"), Some(false), None) must beSuccessfulTry + app[CaseSrv].getCustomField(`case`, EntityName("boolean1")).flatMap(_.value) must beSome.which(_ == false) } } } "update case title" in testApp { app => app[Database].transaction { implicit graph => - app[CaseSrv].get("#3").update(_.title, "new title").getOrFail("Case") - app[CaseSrv].getOrFail("#3") must beSuccessfulTry.which { `case`: Case with Entity => + app[CaseSrv].get(EntityName("3")).update(_.title, "new title").getOrFail("Case") + app[CaseSrv].getOrFail(EntityName("3")) must beSuccessfulTry.which { `case`: Case with Entity => `case`.title must_=== "new title" } } @@ -214,11 +215,11 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { Success(Json.obj("status" -> CaseStatus.Resolved)) }) - val r = app[Database].tryTransaction(implicit graph => app[CaseSrv].update(app[CaseSrv].get("#1"), updates)) + val r = app[Database].tryTransaction(implicit graph => app[CaseSrv].update(app[CaseSrv].get(EntityName("1")), updates)) r must beSuccessfulTry - val updatedCase = app[Database].roTransaction(implicit graph => app[CaseSrv].get("#1").getOrFail("Case").get) + val updatedCase = app[Database].roTransaction(implicit graph => app[CaseSrv].get(EntityName("1")).getOrFail("Case").get) updatedCase.status shouldEqual CaseStatus.Resolved updatedCase.endDate must beSome } @@ -226,7 +227,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "upsert case tags" in testApp { app => app[Database].tryTransaction { implicit graph => for { - c3 <- app[CaseSrv].get("#3").getOrFail("Case") + c3 <- app[CaseSrv].get(EntityName("3")).getOrFail("Case") _ <- app[CaseSrv].updateTagNames(c3, Set("""testNamespace:testPredicate="t2"""", """testNamespace:testPredicate="yolo"""")) } yield app[CaseSrv].get(c3).tags.toList.map(_.toString) } must beASuccessfulTry.which { tags => @@ -241,7 +242,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[CaseSrv].create( Case(0, "case 5", "desc 5", 1, new Date(), None, flag = false, 2, 3, CaseStatus.Open, None), None, - app[OrganisationSrv].getOrFail("cert").get, + app[OrganisationSrv].getOrFail(EntityName("cert")).get, app[TagSrv].startTraversal.toSeq.toSet, Seq.empty, None, @@ -259,13 +260,13 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { ) must beSuccessfulTry app[Database].roTransaction { implicit graph => - app[CaseSrv].startTraversal.has("title", "case 5").tags.toList.length shouldEqual currentLen + 1 + app[CaseSrv].startTraversal.has(_.title, "case 5").tags.toList.length shouldEqual currentLen + 1 } } "add an observable if not existing" in testApp { app => app[Database].roTransaction { implicit graph => - val c1 = app[CaseSrv].get("#1").getOrFail("Case").get + val c1 = app[CaseSrv].get(EntityName("1")).getOrFail("Case").get val observables = app[ObservableSrv].startTraversal.richObservable.toList observables must not(beEmpty) @@ -279,7 +280,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { val newObs = app[Database].tryTransaction { implicit graph => app[ObservableSrv].create( Observable(Some("if you feel lost"), 1, ioc = false, sighted = true), - app[ObservableTypeSrv].get("domain").getOrFail("Case").get, + app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Case").get, "lost.com", Set[String](), Nil @@ -298,7 +299,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[CaseSrv].create( Case(0, "case 9", "desc 9", 1, new Date(), None, flag = false, 2, 3, CaseStatus.Open, None), None, - app[OrganisationSrv].getOrFail("cert").get, + app[OrganisationSrv].getOrFail(EntityName("cert")).get, Set[Tag with Entity](), Seq.empty, None, @@ -320,7 +321,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { case0 <- app[CaseSrv].create( Case(0, "case 6", "desc 6", 1, new Date(), None, flag = false, 2, 3, CaseStatus.Open, None), None, - app[OrganisationSrv].getOrFail("cert").get, + app[OrganisationSrv].getOrFail(EntityName("cert")).get, app[TagSrv].startTraversal.toSeq.toSet, Seq.empty, None, @@ -343,7 +344,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[CaseSrv].create( Case(0, "case 7", "desc 7", 1, new Date(), None, flag = false, 2, 3, CaseStatus.Open, None), None, - app[OrganisationSrv].getOrFail("cert").get, + app[OrganisationSrv].getOrFail(EntityName("cert")).get, app[TagSrv].startTraversal.toSeq.toSet, Seq.empty, None, @@ -365,8 +366,8 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { .tryTransaction(implicit graph => app[CaseSrv].create( Case(0, "case 8", "desc 8", 2, new Date(), None, flag = false, 2, 3, CaseStatus.Open, None), - Some(app[UserSrv].get("certuser@thehive.local").getOrFail("Case").get), - app[OrganisationSrv].getOrFail("cert").get, + Some(app[UserSrv].get(EntityName("certuser@thehive.local")).getOrFail("Case").get), + app[OrganisationSrv].getOrFail(EntityName("cert")).get, app[TagSrv].startTraversal.toSeq.toSet, Seq.empty, None, @@ -383,21 +384,21 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { app[Database].tryTransaction(implicit graph => app[CaseSrv].unassign(c8)) must beSuccessfulTry checkAssignee(beFalse) app[Database].tryTransaction(implicit graph => - app[CaseSrv].assign(c8, app[UserSrv].get("certuser@thehive.local").getOrFail("Case").get) + app[CaseSrv].assign(c8, app[UserSrv].get(EntityName("certuser@thehive.local")).getOrFail("Case").get) ) must beSuccessfulTry checkAssignee(beTrue) } "show only visible cases" in testApp { app => app[Database].roTransaction { implicit graph => - app[CaseSrv].get("#3").visible.getOrFail("Case") must beFailedTry + app[CaseSrv].get(EntityName("3")).visible.getOrFail("Case") must beFailedTry } } "forbid correctly case access" in testApp { app => app[Database].roTransaction { implicit graph => app[CaseSrv] - .get("#1") + .get(EntityName("1")) .can(Permissions.manageCase)(DummyUserSrv(userId = "certro@thehive.local", organisation = "cert").authContext) .exists must beFalse } @@ -405,15 +406,15 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "show linked cases" in testApp { app => app[Database].roTransaction { implicit graph => - app[CaseSrv].get("#1").linkedCases must beEmpty + app[CaseSrv].get(EntityName("1")).linkedCases must beEmpty val observables = app[ObservableSrv].startTraversal.richObservable.toList val hfr = observables.find(_.message.contains("Some weird domain")).get app[Database].tryTransaction { implicit graph => - app[CaseSrv].addObservable(app[CaseSrv].get("#2").getOrFail("Case").get, hfr) + app[CaseSrv].addObservable(app[CaseSrv].get(EntityName("2")).getOrFail("Case").get, hfr) } - app[Database].roTransaction(implicit graph => app[CaseSrv].get("#1").linkedCases must not(beEmpty)) + app[Database].roTransaction(implicit graph => app[CaseSrv].get(EntityName("1")).linkedCases must not(beEmpty)) } } } diff --git a/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala index ac6e13058e..5e4b3cb102 100644 --- a/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -28,12 +29,12 @@ class CaseTemplateSrvTest extends PlaySpecification with TestAppBuilder { pap = Some(3), summary = Some("summary case template test 1") ), - organisation = app[OrganisationSrv].getOrFail("cert").get, + organisation = app[OrganisationSrv].getOrFail(EntityName("cert")).get, tagNames = Set("""testNamespace:testPredicate="t2"""", """testNamespace:testPredicate="newOne""""), tasks = Seq( ( Task("task case template case template test 1", "group1", None, TaskStatus.Waiting, flag = false, None, None, 0, None), - app[UserSrv].get("certuser@thehive.local").headOption + app[UserSrv].get(EntityName("certuser@thehive.local")).headOption ) ), customFields = Seq(("string1", Some("love")), ("boolean1", Some(false))) @@ -42,8 +43,8 @@ class CaseTemplateSrvTest extends PlaySpecification with TestAppBuilder { app[Database].roTransaction { implicit graph => app[TagSrv].startTraversal.getByName("testNamespace", "testPredicate", Some("newOne")).exists must beTrue - app[TaskSrv].startTraversal.has("title", "task case template case template test 1").exists must beTrue - val richCT = app[CaseTemplateSrv].startTraversal.has("name", "case template test 1").richCaseTemplate.getOrFail("CaseTemplate").get + app[TaskSrv].startTraversal.has(_.title, "task case template case template test 1").exists must beTrue + val richCT = app[CaseTemplateSrv].startTraversal.getByName("case template test 1").richCaseTemplate.getOrFail("CaseTemplate").get richCT.customFields.length shouldEqual 2 } } @@ -52,20 +53,20 @@ class CaseTemplateSrvTest extends PlaySpecification with TestAppBuilder { app[Database].tryTransaction { implicit graph => for { richTask <- app[TaskSrv].create(Task("t1", "default", None, TaskStatus.Waiting, flag = false, None, None, 1, None), None) - caseTemplate <- app[CaseTemplateSrv].getOrFail("spam") + caseTemplate <- app[CaseTemplateSrv].getOrFail(EntityName("spam")) _ <- app[CaseTemplateSrv].addTask(caseTemplate, richTask.task) } yield () } must beSuccessfulTry app[Database].roTransaction { implicit graph => - app[CaseTemplateSrv].get("spam").tasks.has("title", "t1").exists + app[CaseTemplateSrv].get(EntityName("spam")).tasks.has(_.title, "t1").exists } must beTrue } "update case template tags" in testApp { app => app[Database].tryTransaction { implicit graph => for { - caseTemplate <- app[CaseTemplateSrv].getOrFail("spam") + caseTemplate <- app[CaseTemplateSrv].getOrFail(EntityName("spam")) _ <- app[CaseTemplateSrv].updateTagNames( caseTemplate, Set("""testNamespace:testPredicate="t2"""", """testNamespace:testPredicate="newOne2"""", """newNspc.newPred="newOne3"""") @@ -73,7 +74,7 @@ class CaseTemplateSrvTest extends PlaySpecification with TestAppBuilder { } yield () } must beSuccessfulTry app[Database].roTransaction { implicit graph => - app[CaseTemplateSrv].get("spam").tags.toList.map(_.toString) + app[CaseTemplateSrv].get(EntityName("spam")).tags.toList.map(_.toString) } must containTheSameElementsAs( Seq("testNamespace:testPredicate=\"t2\"", "testNamespace:testPredicate=\"newOne2\"", "newNspc:newPred=\"newOne3\"") ) @@ -82,12 +83,12 @@ class CaseTemplateSrvTest extends PlaySpecification with TestAppBuilder { "add tags to a case template" in testApp { app => app[Database].tryTransaction { implicit graph => for { - caseTemplate <- app[CaseTemplateSrv].getOrFail("spam") + caseTemplate <- app[CaseTemplateSrv].getOrFail(EntityName("spam")) _ <- app[CaseTemplateSrv].addTags(caseTemplate, Set("""testNamespace:testPredicate="t2"""", """testNamespace:testPredicate="newOne2"""")) } yield () } must beSuccessfulTry app[Database].roTransaction { implicit graph => - app[CaseTemplateSrv].get("spam").tags.toList.map(_.toString) + app[CaseTemplateSrv].get(EntityName("spam")).tags.toList.map(_.toString) } must containTheSameElementsAs( Seq( "testNamespace:testPredicate=\"t2\"", @@ -101,17 +102,17 @@ class CaseTemplateSrvTest extends PlaySpecification with TestAppBuilder { "update/create case template custom fields" in testApp { app => app[Database].tryTransaction { implicit graph => for { - string1 <- app[CustomFieldSrv].getOrFail("string1") - bool1 <- app[CustomFieldSrv].getOrFail("boolean1") - integer1 <- app[CustomFieldSrv].getOrFail("integer1") - caseTemplate <- app[CaseTemplateSrv].getOrFail("spam") + string1 <- app[CustomFieldSrv].getOrFail(EntityName("string1")) + bool1 <- app[CustomFieldSrv].getOrFail(EntityName("boolean1")) + integer1 <- app[CustomFieldSrv].getOrFail(EntityName("integer1")) + caseTemplate <- app[CaseTemplateSrv].getOrFail(EntityName("spam")) _ <- app[CaseTemplateSrv].updateCustomField(caseTemplate, Seq((string1, JsString("hate")), (bool1, JsTrue), (integer1, JsNumber(1)))) } yield () } must beSuccessfulTry val expected: Seq[(String, JsValue)] = Seq("string1" -> JsString("hate"), "boolean1" -> JsTrue, "integer1" -> JsNumber(1)) app[Database].roTransaction { implicit graph => - app[CaseTemplateSrv].get("spam").customFields.nameJsonValue.toSeq + app[CaseTemplateSrv].get(EntityName("spam")).customFields.nameJsonValue.toSeq } must contain(exactly(expected: _*)) } } diff --git a/thehive/test/org/thp/thehive/services/ConfigSrvTest.scala b/thehive/test/org/thp/thehive/services/ConfigSrvTest.scala index 61b2cbc794..b02f47084c 100644 --- a/thehive/test/org/thp/thehive/services/ConfigSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/ConfigSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.thehive.TestAppBuilder @@ -12,13 +13,13 @@ class ConfigSrvTest extends PlaySpecification with TestAppBuilder { "config service" should { "set/get values" in testApp { app => app[Database].tryTransaction { implicit graph => - app[ConfigSrv].organisation.setConfigValue("cert", "test", JsBoolean(true)) - app[ConfigSrv].user.setConfigValue("certuser@thehive.local", "test2", JsString("lol")) + app[ConfigSrv].organisation.setConfigValue(EntityName("cert"), "test", JsBoolean(true)) + app[ConfigSrv].user.setConfigValue(EntityName("certuser@thehive.local"), "test2", JsString("lol")) } app[Database].roTransaction { implicit graph => - app[ConfigSrv].organisation.getConfigValue("cert", "test") must beSome.which(c => c.value.as[Boolean] must beTrue) - app[ConfigSrv].user.getConfigValue("certuser@thehive.local", "test2") must beSome.which(c => c.value.as[String] shouldEqual "lol") + app[ConfigSrv].organisation.getConfigValue(EntityName("cert"), "test") must beSome.which(c => c.value.as[Boolean] must beTrue) + app[ConfigSrv].user.getConfigValue(EntityName("certuser@thehive.local"), "test2") must beSome.which(c => c.value.as[String] shouldEqual "lol") } } } diff --git a/thehive/test/org/thp/thehive/services/CustomFieldSrvTest.scala b/thehive/test/org/thp/thehive/services/CustomFieldSrvTest.scala index ea44ab8f82..5a40bffd1c 100644 --- a/thehive/test/org/thp/thehive/services/CustomFieldSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CustomFieldSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.thehive.TestAppBuilder @@ -33,7 +34,7 @@ class CustomFieldSrvTest extends PlaySpecification with TestAppBuilder { "delete custom fields" in testApp { app => app[Database].tryTransaction { implicit graph => for { - cf <- app[CustomFieldSrv].getOrFail("boolean1") + cf <- app[CustomFieldSrv].getOrFail(EntityName("boolean1")) _ <- app[CustomFieldSrv].delete(cf, force = true) } yield () } must beSuccessfulTry @@ -41,7 +42,7 @@ class CustomFieldSrvTest extends PlaySpecification with TestAppBuilder { "count use of custom fields" in testApp { app => app[Database].roTransaction { implicit graph => - app[CustomFieldSrv].useCount(app[CustomFieldSrv].getOrFail("boolean1").get) + app[CustomFieldSrv].useCount(app[CustomFieldSrv].getOrFail(EntityName("boolean1")).get) } shouldEqual Map("Case" -> 1, "CaseTemplate" -> 1) } } diff --git a/thehive/test/org/thp/thehive/services/DashboardSrvTest.scala b/thehive/test/org/thp/thehive/services/DashboardSrvTest.scala index a701964170..af939cf759 100644 --- a/thehive/test/org/thp/thehive/services/DashboardSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/DashboardSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -15,7 +16,8 @@ class DashboardSrvTest extends PlaySpecification with TestAppBuilder { s" dashboard service" should { "create dashboards" in testApp { app => val definition = - Json.parse("""{ + Json + .parse("""{ "period":"custom", "items":[ { @@ -49,7 +51,8 @@ class DashboardSrvTest extends PlaySpecification with TestAppBuilder { "fromDate":"2019-07-08T22:00:00.000Z", "toDate":"2019-11-27T23:00:00.000Z" } - }""").as[JsObject] + }""") + .as[JsObject] app[Database].tryTransaction { implicit graph => app[DashboardSrv].create(Dashboard("dashboard test 1", "desc dashboard test 1", definition)) } must beASuccessfulTry.which { d => @@ -62,9 +65,9 @@ class DashboardSrvTest extends PlaySpecification with TestAppBuilder { "share a dashboard" in testApp { app => app[Database].tryTransaction { implicit graph => for { - dashboard <- app[DashboardSrv].startTraversal.has("title", "dashboard soc").getOrFail("Dashboard") + dashboard <- app[DashboardSrv].startTraversal.has(_.title, "dashboard soc").getOrFail("Dashboard") _ = app[DashboardSrv].get(dashboard).visible.headOption must beNone - _ <- app[DashboardSrv].share(dashboard, "cert", writable = false) + _ <- app[DashboardSrv].share(dashboard, EntityName("cert"), writable = false) _ = app[DashboardSrv].get(dashboard).visible.headOption must beSome } yield () } must beASuccessfulTry @@ -81,9 +84,9 @@ class DashboardSrvTest extends PlaySpecification with TestAppBuilder { "remove a dashboard" in testApp { app => app[Database].tryTransaction { implicit graph => for { - dashboard <- app[DashboardSrv].startTraversal.has("title", "dashboard soc").getOrFail("Dashboard") + dashboard <- app[DashboardSrv].startTraversal.has(_.title, "dashboard soc").getOrFail("Dashboard") _ <- app[DashboardSrv].remove(dashboard) - } yield app[DashboardSrv].startTraversal.has("title", "dashboard soc").exists + } yield app[DashboardSrv].startTraversal.has(_.title, "dashboard soc").exists } must beASuccessfulTry(false) } } diff --git a/thehive/test/org/thp/thehive/services/DataSrvTest.scala b/thehive/test/org/thp/thehive/services/DataSrvTest.scala index a0736bc89e..054b578e89 100644 --- a/thehive/test/org/thp/thehive/services/DataSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/DataSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -13,7 +14,7 @@ class DataSrvTest extends PlaySpecification with TestAppBuilder { "data service" should { "create not existing data" in testApp { app => - val existingData = app[Database].roTransaction(implicit graph => app[DataSrv].startTraversal.has("data", "h.fr").getOrFail("Data")).get + val existingData = app[Database].roTransaction(implicit graph => app[DataSrv].startTraversal.getByData("h.fr").getOrFail("Data")).get val newData = app[Database].tryTransaction(implicit graph => app[DataSrv].create(existingData)) newData must beSuccessfulTry.which(data => data._id shouldEqual existingData._id) } @@ -22,7 +23,7 @@ class DataSrvTest extends PlaySpecification with TestAppBuilder { app[Database].tryTransaction { implicit graph => app[ObservableSrv].create( Observable(Some("love"), 1, ioc = false, sighted = true), - app[ObservableTypeSrv].get("domain").getOrFail("Observable").get, + app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Observable").get, "love.com", Set("tagX"), Nil diff --git a/thehive/test/org/thp/thehive/services/LocalPasswordAuthSrvTest.scala b/thehive/test/org/thp/thehive/services/LocalPasswordAuthSrvTest.scala index 6244b871da..0a5427c6a0 100644 --- a/thehive/test/org/thp/thehive/services/LocalPasswordAuthSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/LocalPasswordAuthSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.models.Database import org.thp.thehive.TestAppBuilder import play.api.Configuration @@ -10,7 +11,7 @@ class LocalPasswordAuthSrvTest extends PlaySpecification with TestAppBuilder { "localPasswordAuth service" should { "be able to verify passwords" in testApp { app => app[Database].roTransaction { implicit graph => - val certuser = app[UserSrv].getOrFail("certuser@thehive.local").get + val certuser = app[UserSrv].getOrFail(EntityName("certuser@thehive.local")).get val localPasswordAuthSrv = app[LocalPasswordAuthProvider].apply(app[Configuration]).get.asInstanceOf[LocalPasswordAuthSrv] val request = FakeRequest("POST", "/api/v0/login") .withJsonBody( diff --git a/thehive/test/org/thp/thehive/services/OrganisationSrvTest.scala b/thehive/test/org/thp/thehive/services/OrganisationSrvTest.scala index c8c3056afb..c341b60a09 100644 --- a/thehive/test/org/thp/thehive/services/OrganisationSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/OrganisationSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.thehive.TestAppBuilder @@ -18,7 +19,7 @@ class OrganisationSrvTest extends PlaySpecification with TestAppBuilder { "get an organisation by its name" in testApp { app => app[Database].tryTransaction { implicit graph => - app[OrganisationSrv].getOrFail("cert") + app[OrganisationSrv].getOrFail(EntityName("cert")) } must beSuccessfulTry } } diff --git a/thehive/test/org/thp/thehive/services/UserSrvTest.scala b/thehive/test/org/thp/thehive/services/UserSrvTest.scala index 49c9b790d7..fc121bcac8 100644 --- a/thehive/test/org/thp/thehive/services/UserSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/UserSrvTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.scalligraph.traversal.TraversalOps._ @@ -40,29 +41,29 @@ class UserSrvTest extends PlaySpecification with TestAppBuilder { ) ) must beSuccessfulTry .which { user => - app[UserSrv].getOrFail(user.login) must beSuccessfulTry(user) + app[UserSrv].getOrFail(user._id) must beSuccessfulTry(user) } } } "deduplicate users in an organisation" in testApp { app => - implicit val db = app[Database] - val userSrv = app[UserSrv] - val organisationSrv = app[OrganisationSrv] - val profileSrv = app[ProfileSrv] - val roleSrv = app[RoleSrv] + implicit val db: Database = app[Database] + val userSrv = app[UserSrv] + val organisationSrv = app[OrganisationSrv] + val profileSrv = app[ProfileSrv] + val roleSrv = app[RoleSrv] db.tryTransaction { implicit graph => - val certadmin = userSrv.get("certadmin@thehive.local").head - val cert = organisationSrv.get("cert").head - val analyst = profileSrv.get("analyst").head + val certadmin = userSrv.get(EntityName("certadmin@thehive.local")).head + val cert = organisationSrv.get(EntityName("cert")).head + val analyst = profileSrv.get(EntityName("analyst")).head roleSrv.create(certadmin, cert, analyst).get - val userCount = userSrv.get("certadmin@thehive.local").organisations.get("cert").getCount + val userCount = userSrv.get(EntityName("certadmin@thehive.local")).organisations.get(EntityName("cert")).getCount if (userCount == 2) Success(()) else Failure(new Exception(s"User certadmin is not in cert organisation twice ($userCount)")) } new UserIntegrityCheckOps(db, userSrv, profileSrv, organisationSrv, roleSrv).check() db.roTransaction { implicit graph => - val userCount = userSrv.get("certadmin@thehive.local").organisations.get("cert").getCount + val userCount = userSrv.get(EntityName("certadmin@thehive.local")).organisations.get(EntityName("cert")).getCount userCount must beEqualTo(1) } } diff --git a/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala b/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala index 790b847308..882f1f1491 100644 --- a/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala +++ b/thehive/test/org/thp/thehive/services/notification/notifiers/NotificationTemplateTest.scala @@ -2,6 +2,7 @@ package org.thp.thehive.services.notification.notifiers import java.util.{HashMap => JHashMap} +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv, Schema} import org.thp.scalligraph.traversal.TraversalOps._ @@ -13,9 +14,10 @@ import scala.collection.JavaConverters._ class NotificationTemplateTest extends PlaySpecification with TestAppBuilder { implicit val authContext: AuthContext = DummyUserSrv(userId = "certuser@thehive.local").authContext - def templateEngine(testSchema: Schema): Template = new Object with Template { - val schema = testSchema - } + def templateEngine(testSchema: Schema): Template = + new Object with Template { + override val schema: Schema = testSchema + } "template engine" should { "format message" in testApp { app => @@ -39,11 +41,17 @@ class NotificationTemplateTest extends PlaySpecification with TestAppBuilder { val model = new JHashMap[String, AnyRef] model.put( "audit", - Map("objectType" -> "Case", "objectId" -> "2231", "action" -> "create", "_createdBy" -> "admin@thehive.local", "requestId" -> "testRequest").asJava + Map( + "objectType" -> "Case", + "objectId" -> "2231", + "action" -> "create", + "_createdBy" -> "admin@thehive.local", + "requestId" -> "testRequest" + ).asJava ) model.put("object", Map("_type" -> "Case", "title" -> "case title").asJava) - model.put("user", Map("name" -> "Thomas").asJava) - model.put("context", Map("_id" -> "2231").asJava) + model.put("user", Map("name" -> "Thomas").asJava) + model.put("context", Map("_id" -> "2231").asJava) val message = templateEngine(app[Schema]).handlebars.compileInline(template).apply(model) message must beEqualTo("""Dear Thomas, |you have a new notification: @@ -77,7 +85,7 @@ class NotificationTemplateTest extends PlaySpecification with TestAppBuilder { val message = app[Database].tryTransaction { implicit graph => for { - case4 <- app[CaseSrv].get("#1").getOrFail("Case") + case4 <- app[CaseSrv].get(EntityName("1")).getOrFail("Case") _ <- app[CaseSrv].addTags(case4, Set("emailer test")) _ <- app[CaseSrv].addTags(case4, Set("emailer test")) // this is needed to make AuditSrv write Audit in DB audit <- app[AuditSrv].startTraversal.has(_.objectId, case4._id.toString).getOrFail("Audit") @@ -89,13 +97,13 @@ class NotificationTemplateTest extends PlaySpecification with TestAppBuilder { m must beMatching("""Dear certuser, |you have a new notification: | - |The Case \d+ has been updated by certuser@thehive.local + |The Case ~\d+ has been updated by certuser@thehive.local | |case#1 | | - |Audit \(testRequest\): update Case \d+ by certuser@thehive.local - |Context \d+""".stripMargin) + |Audit \(testRequest\): update Case ~\d+ by certuser@thehive.local + |Context ~\d+""".stripMargin) } } } diff --git a/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala b/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala index 84eb7a9611..62e1a15285 100644 --- a/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala +++ b/thehive/test/org/thp/thehive/services/notification/triggers/AlertCreatedTest.scala @@ -4,6 +4,7 @@ import java.util.Date import org.thp.scalligraph.models.Database import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{EntityIdOrName, EntityName} import org.thp.thehive.TestAppBuilder import org.thp.thehive.controllers.v0.AlertCtrl import org.thp.thehive.dto.v0.{InputAlert, OutputAlert} @@ -44,7 +45,7 @@ class AlertCreatedTest extends PlaySpecification with TestAppBuilder { status(result) should equalTo(201) val alertOutput = contentAsJson(result).as[OutputAlert] - val alert = app[AlertSrv].get(alertOutput.id).getOrFail("Alert") + val alert = app[AlertSrv].get(EntityIdOrName(alertOutput.id)).getOrFail("Alert") alert must beSuccessfulTry @@ -52,12 +53,12 @@ class AlertCreatedTest extends PlaySpecification with TestAppBuilder { audit must beSuccessfulTry - val organisation = app[OrganisationSrv].get("cert").getOrFail("Organisation") + val organisation = app[OrganisationSrv].get(EntityName("cert")).getOrFail("Organisation") organisation must beSuccessfulTry - val user2 = app[UserSrv].getOrFail("certadmin@thehive.local") - val user1 = app[UserSrv].getOrFail("certuser@thehive.local") + val user2 = app[UserSrv].getOrFail(EntityName("certadmin@thehive.local")) + val user1 = app[UserSrv].getOrFail(EntityName("certuser@thehive.local")) user2 must beSuccessfulTry user1 must beSuccessfulTry diff --git a/thehive/test/org/thp/thehive/services/notification/triggers/TaskAssignedTest.scala b/thehive/test/org/thp/thehive/services/notification/triggers/TaskAssignedTest.scala index 5a4c48d873..e09f38572c 100644 --- a/thehive/test/org/thp/thehive/services/notification/triggers/TaskAssignedTest.scala +++ b/thehive/test/org/thp/thehive/services/notification/triggers/TaskAssignedTest.scala @@ -1,5 +1,6 @@ package org.thp.thehive.services.notification.triggers +import org.thp.scalligraph.EntityName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, DummyUserSrv} import org.thp.scalligraph.traversal.TraversalOps._ @@ -15,13 +16,13 @@ class TaskAssignedTest extends PlaySpecification with TestAppBuilder { "be properly triggered on task assignment" in testApp { app => app[Database].tryTransaction { implicit graph => for { - task1 <- app[TaskSrv].startTraversal.has("title", "case 1 task 1").getOrFail("Task") + task1 <- app[TaskSrv].startTraversal.has(_.title, "case 1 task 1").getOrFail("Task") user1 <- app[UserSrv].startTraversal.getByName("certuser@thehive.local").getOrFail("User") user2 <- app[UserSrv].startTraversal.getByName("certadmin@thehive.local").getOrFail("User") _ <- app[TaskSrv].assign(task1, user1) _ <- app[AuditSrv].flushPendingAudit() - audit <- app[AuditSrv].startTraversal.has("objectId", task1._id).getOrFail("Audit") - orga <- app[OrganisationSrv].get("cert").getOrFail("Organisation") + audit <- app[AuditSrv].startTraversal.has(_.objectId, task1._id.toString).getOrFail("Audit") + orga <- app[OrganisationSrv].get(EntityName("cert")).getOrFail("Organisation") taskAssignedTrigger = new TaskAssigned(app[TaskSrv]) _ = taskAssignedTrigger.filter(audit, Some(task1), orga, Some(user1)) must beTrue _ = taskAssignedTrigger.filter(audit, Some(task1), orga, Some(user2)) must beFalse diff --git a/thehive/test/resources/data/CaseTemplateCustomField.json b/thehive/test/resources/data/CaseTemplateCustomField.json index c3c9344391..da4f07e715 100644 --- a/thehive/test/resources/data/CaseTemplateCustomField.json +++ b/thehive/test/resources/data/CaseTemplateCustomField.json @@ -1,4 +1,4 @@ [ - {"from": "spam", "to": "string1", "stringValue": "string1 custom field"}, - {"from": "spam", "to": "boolean1"} -] \ No newline at end of file + {"from": "spam", "to": "string1", "stringValue": "string1 custom field", "order": 1}, + {"from": "spam", "to": "boolean1", "order": 2} +] From 0bf95c37e088579dd31a90a0749b42717ee9fddf Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 6 Oct 2020 11:36:36 +0200 Subject: [PATCH 062/237] #1501 Mapping macro cleanup --- ScalliGraph | 2 +- .../org/thp/thehive/migration/th3/DBGet.scala | 29 +++++++++---------- .../thp/thehive/controllers/v0/Router.scala | 2 +- .../thehive/controllers/v1/Conversion.scala | 6 ++-- .../thp/thehive/controllers/v1/Router.scala | 18 ++++++------ .../org/thp/thehive/services/CaseSrv.scala | 8 ++--- .../org/thp/thehive/services/package.scala | 29 ------------------- .../thp/thehive/services/CaseSrvTest.scala | 4 +-- 8 files changed, 34 insertions(+), 64 deletions(-) delete mode 100644 thehive/app/org/thp/thehive/services/package.scala diff --git a/ScalliGraph b/ScalliGraph index 8a7900ffdb..b2a923dc4c 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 8a7900ffdbf11ed6e29e0dfa450e3e1ca776af35 +Subproject commit b2a923dc4c6d997ececf8d22c7b04abdbdeac40a diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala index d6f6983175..7994058184 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala @@ -19,19 +19,18 @@ class DBGet @Inject() (db: DBConfiguration, implicit val ec: ExecutionContext) { */ def apply(modelName: String, id: String): Future[JsObject] = db.execute { - // Search by id is not possible on child entity without routing information ⇒ id query - search(db.indexName) - .query(idsQuery(id) /*.types(modelName)*/ ) - .size(1) - .version(true) - } - .map { searchResponse => - searchResponse - .hits - .hits - .headOption - .fold[JsObject](throw NotFoundError(s"$modelName $id not found")) { hit => - DBUtils.hit2json(hit) - } - } + // Search by id is not possible on child entity without routing information => id query + search(db.indexName) + .query(idsQuery(id) /*.types(modelName)*/ ) + .size(1) + .version(true) + }.map { searchResponse => + searchResponse + .hits + .hits + .headOption + .fold[JsObject](throw NotFoundError(s"$modelName $id not found")) { hit => + DBUtils.hit2json(hit) + } + } } diff --git a/thehive/app/org/thp/thehive/controllers/v0/Router.scala b/thehive/app/org/thp/thehive/controllers/v0/Router.scala index 2f219da332..96f8327e37 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Router.scala @@ -75,7 +75,7 @@ class Router @Inject() ( // case GET(p"/case/task/log/$logId") => logCtrl.get(logId) case POST(p"/case/artifact/_search") => observableCtrl.search - // case POST(p"/case/:caseId/artifact/_search") ⇒ observableCtrl.findInCase(caseId) + // case POST(p"/case/:caseId/artifact/_search") => observableCtrl.findInCase(caseId) case POST(p"/case/artifact/_stats") => observableCtrl.stats case POST(p"/case/$caseId/artifact") => observableCtrl.create(caseId) // Audit ok case GET(p"/case/artifact/$observableId") => observableCtrl.get(observableId) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index f7f2db0dbc..53c6e2c158 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -48,13 +48,13 @@ object Conversion { .withFieldComputed(_._createdAt, _._createdAt) .withFieldComputed(_._createdBy, _._createdBy) .withFieldComputed(_.obj, a => a.`object`.map(OutputEntity.apply)) - // .withFieldComputed(_.obj, a ⇒ OutputEntity(a.obj)) + // .withFieldComputed(_.obj, a => OutputEntity(a.obj)) // .withFieldComputed( // _.summary, // _.summary.mapValues( - // opCount ⇒ + // opCount => // opCount.map { - // case (op, count) ⇒ op.toString → count + // case (op, count) => op.toString → count // } // ) .withFieldConst(_.attributeName, None) // FIXME diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 3549028b14..a04cc7769d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -36,16 +36,16 @@ class Router @Inject() ( case PATCH(p"/case/$caseId") => caseCtrl.update(caseId) case POST(p"/case/_merge/$caseIds") => caseCtrl.merge(caseIds) case DELETE(p"/case/$caseId") => caseCtrl.delete(caseId) -// case PATCH(p"api/case/_bulk") ⇒ caseCtrl.bulkUpdate() -// case POST(p"/case/_stats") ⇒ caseCtrl.stats() -// case DELETE(p"/case/$caseId/force") ⇒ caseCtrl.realDelete(caseId) -// case GET(p"/case/$caseId/links") ⇒ caseCtrl.linkedCases(caseId) +// case PATCH(p"api/case/_bulk") => caseCtrl.bulkUpdate() +// case POST(p"/case/_stats") => caseCtrl.stats() +// case DELETE(p"/case/$caseId/force") => caseCtrl.realDelete(caseId) +// case GET(p"/case/$caseId/links") => caseCtrl.linkedCases(caseId) case GET(p"/caseTemplate") => caseTemplateCtrl.list case POST(p"/caseTemplate") => caseTemplateCtrl.create case GET(p"/caseTemplate/$caseTemplateId") => caseTemplateCtrl.get(caseTemplateId) case PATCH(p"/caseTemplate/$caseTemplateId") => caseTemplateCtrl.update(caseTemplateId) - //case DELETE(p"/caseTemplate/$caseTemplateId") ⇒ caseTemplateCtrl.delete(caseTemplateId) + //case DELETE(p"/caseTemplate/$caseTemplateId") => caseTemplateCtrl.delete(caseTemplateId) case POST(p"/user") => userCtrl.create case GET(p"/user/current") => userCtrl.current @@ -64,10 +64,10 @@ class Router @Inject() ( case GET(p"/organisation/$organisationId") => organisationCtrl.get(organisationId) case PATCH(p"/organisation/$organisationId") => organisationCtrl.update(organisationId) -// case GET(p"/share") ⇒ shareCtrl.list -// case POST(p"/share") ⇒ shareCtrl.create -// case GET(p"/share/$shareId") ⇒ shareCtrl.get(shareId) -// case PATCH(p"/share/$shareId") ⇒ shareCtrl.update(shareId) +// case GET(p"/share") => shareCtrl.list +// case POST(p"/share") => shareCtrl.create +// case GET(p"/share/$shareId") => shareCtrl.get(shareId) +// case PATCH(p"/share/$shareId") => shareCtrl.update(shareId) case GET(p"/task") => taskCtrl.list case POST(p"/task") => taskCtrl.create diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 69b4bd66a3..c58479fdee 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -310,8 +310,8 @@ class CaseSrv @Inject() ( // .flatMap(_.customFields().toList // .groupBy(_.name) // .foreach { -// case (name, l) ⇒ -// val values = l.collect { case cfwv: CustomFieldWithValue if cfwv.value.isDefined ⇒ cfwv.value.get } +// case (name, l) => +// val values = l.collect { case cfwv: CustomFieldWithValue if cfwv.value.isDefined => cfwv.value.get } // val cf = customFieldSrv.getOrFail(name) // val caseCustomField = // if (values.size == 1) cf.`type`.setValue(CaseCustomField(), values.head) @@ -324,12 +324,12 @@ class CaseSrv @Inject() ( // cases // .map(get) // .flatMap(_.tasks.toList -// .foreach(task ⇒ caseTaskSrv.create(CaseTask(), task, mergedCase)) +// .foreach(task => caseTaskSrv.create(CaseTask(), task, mergedCase)) // // cases // .map(get) // .flatMap(_.observables.toList -// .foreach(observable ⇒ observableCaseSrv.create(ObservableCase(), observable, mergedCase)) +// .foreach(observable => observableCaseSrv.create(ObservableCase(), observable, mergedCase)) // // get(mergedCase).richCase.head // } diff --git a/thehive/app/org/thp/thehive/services/package.scala b/thehive/app/org/thp/thehive/services/package.scala deleted file mode 100644 index e0d654e7dd..0000000000 --- a/thehive/app/org/thp/thehive/services/package.scala +++ /dev/null @@ -1,29 +0,0 @@ -//package org.thp.thehive -// -//import org.thp.scalligraph.steps.VertexSteps -// -//package object services { -// -// implicit class EntityStepsOps[E <: Product](steps: Traversal.V[Vertex][E]) { -// def asCase: Traversal.V[Case] = steps match { -// case caseSteps: Traversal.V[Case] => caseSteps -// case _ => new CaseSteps(steps.raw)(steps.db, steps.graph) -// } -// def asTask: Traversal.V[Task] = steps match { -// case taskSteps: Traversal.V[Task] => taskSteps -// case _ => new TaskSteps(steps.raw)(steps.db, steps.graph) -// } -// def asLog: Traversal.V[Log] = steps match { -// case logSteps: Traversal.V[Log] => logSteps -// case _ => new LogSteps(steps.raw)(steps.db, steps.graph) -// } -// def asObservable: Traversal.V[Observable] = steps match { -// case observableSteps: Traversal.V[Observable] => observableSteps -// case _ => new ObservableSteps(steps.raw)(steps.db, steps.graph) -// } -// def asAlert: Traversal.V[Alert] = steps match { -// case alertSteps: Traversal.V[Alert] => alertSteps -// case _ => new AlertSteps(steps.raw)(steps.db, steps.graph) -// } -// } -//} diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index 2843c291ac..75a105d0b9 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -143,7 +143,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { "merge two cases" in testApp { app => pending // app[Database].transaction { implicit graph => - // Seq("#2", "#3").toTry(app[CaseSrv].getOrFail) must beSuccessfulTry.which { cases: Seq[Case with Entity] ⇒ + // Seq("#2", "#3").toTry(app[CaseSrv].getOrFail) must beSuccessfulTry.which { cases: Seq[Case with Entity] => // val mergedCase = app[CaseSrv].merge(cases)(graph, dummyUserSrv.getSystemAuthContext) // // mergedCase.title must_=== "case#2 / case#3" @@ -159,7 +159,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { // mergedCase.summary must beNone // mergedCase.impactStatus must beNone // mergedCase.user must beSome("test") - // mergedCase.customFields.map(f ⇒ (f.name, f.typeName, f.value)) must contain( + // mergedCase.customFields.map(f => (f.name, f.typeName, f.value)) must contain( // allOf[(String, String, Option[Any])]( // ("boolean1", "boolean", Some(true)), // ("string1", "string", Some("string1 custom field")) From bbe94d10b60ba1567ba0dd39b888f6c079cc09c3 Mon Sep 17 00:00:00 2001 From: garanews Date: Tue, 6 Oct 2020 15:16:55 +0200 Subject: [PATCH 063/237] Update docker-compose.yml --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5d16e2b6ec..832c4e5a10 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.8" +version: "3.6" services: elasticsearch: image: 'elasticsearch:7.9.0' From 4b2f02e95fbb830312467a7e6ab4ce4f7c3be616 Mon Sep 17 00:00:00 2001 From: garanews Date: Tue, 6 Oct 2020 15:19:55 +0200 Subject: [PATCH 064/237] Update README.md --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index f317715d24..08914d545d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -49,9 +49,9 @@ Structure would look like: ``` ### ElasticSearch -ElasticSearch container likes big mmap count (https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) so from shell you can cgange with +ElasticSearch container likes big mmap count (https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) so from shell you can change with ```sysctl -w vm.max_map_count=262144``` -Due you would run all on same system and maybe you don't have a limited amount of RAM, better to set some size, for ElasticSearch, in docker-compose.yml I added those: +Due you would run all on same system and maybe you have a limited amount of RAM, better to set some size, for ElasticSearch, in docker-compose.yml I added those: ```- bootstrap.memory_lock=true``` ```- "ES_JAVA_OPTS=-Xms256m -Xmx256m"``` From 1c0ec0760b5bd7fc8f646d6d2c2087a92685a3c8 Mon Sep 17 00:00:00 2001 From: garanews Date: Tue, 6 Oct 2020 15:20:59 +0200 Subject: [PATCH 065/237] Update docker-compose.yml --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 832c4e5a10..5d16e2b6ec 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.6" +version: "3.8" services: elasticsearch: image: 'elasticsearch:7.9.0' From ae74ecf42c6179d4dd416717833c7e485252aa5b Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Oct 2020 15:31:50 +0200 Subject: [PATCH 066/237] #1579 Add case filter and observable type count in alert similar case query --- ScalliGraph | 2 +- .../thehive/controllers/v0/AlertCtrl.scala | 2 +- .../thehive/controllers/v1/AlertCtrl.scala | 20 +++++++++++--- .../controllers/v1/AlertRenderer.scala | 5 ++-- thehive/app/org/thp/thehive/models/Case.scala | 2 +- .../org/thp/thehive/services/AlertSrv.scala | 26 ++++++++++++++----- .../thp/thehive/services/ObservableSrv.scala | 2 ++ 7 files changed, 44 insertions(+), 15 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index b2a923dc4c..a6f82a382e 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit b2a923dc4c6d997ececf8d22c7b04abdbdeac40a +Subproject commit a6f82a382eb7191640a621591b20a072f96375c8 diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 8677037f5a..85f54705d6 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -71,7 +71,7 @@ class AlertCtrl @Inject() ( def alertSimilarityRenderer(implicit authContext: AuthContext ): Traversal.V[Alert] => Traversal[JsArray, JList[JMap[String, Any]], Converter[JsArray, JList[JMap[String, Any]]]] = - _.similarCases + _.similarCases(None) .fold .domainMap { similarCases => JsArray { diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 4ad779213c..333bf115d4 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -6,7 +6,7 @@ import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database -import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} +import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.thehive.controllers.v1.Conversion._ @@ -20,6 +20,9 @@ import org.thp.thehive.services._ import play.api.libs.json.{JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} +import scala.reflect.runtime.{universe => ru} + +case class SimilarCaseFilter() @Singleton class AlertCtrl @Inject() ( entrypoint: Entrypoint, @@ -51,12 +54,23 @@ class AlertCtrl @Inject() ( ) ) override val outputQuery: Query = Query.output[RichAlert, Traversal.V[Alert]](_.richAlert) + val caseFilterParser: FieldsParser[Option[InputQuery[Traversal.Unk, Traversal.Unk]]] = + FilterQuery.default(db, properties.`case`).paramParser(ru.typeOf[Traversal.V[Case]]).optional.on("caseFilter") override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Alert], Traversal.V[Observable]]("observables", (alertSteps, _) => alertSteps.observables), Query[Traversal.V[Alert], Traversal.V[Case]]("case", (alertSteps, _) => alertSteps.`case`), - Query[Traversal.V[Alert], Traversal[JsValue, JMap[String, Any], Converter[JsValue, JMap[String, Any]]]]( + Query.withParam[Option[InputQuery[Traversal.Unk, Traversal.Unk]], Traversal.V[Alert], Traversal[ + JsValue, + JMap[String, Any], + Converter[JsValue, JMap[String, Any]] + ]]( "similarCases", - (alertSteps, authContext) => alertSteps.similarCases(authContext).domainMap(Json.toJson(_)) + caseFilterParser, + { (maybeCaseFilterQuery, alertSteps, authContext) => + val maybeCaseFilter: Option[Traversal.V[Case] => Traversal.V[Case]] = + maybeCaseFilterQuery.map(f => cases => f(db, properties.`case`, ru.typeOf[Traversal.V[Case]], cases.cast, authContext).cast) + alertSteps.similarCases(maybeCaseFilter)(authContext).domainMap(Json.toJson(_)) + } ) ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala index 83a3563834..23e0682d1b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertRenderer.scala @@ -18,7 +18,8 @@ trait AlertRenderer { "similarObservableCount" -> similarStats.observable._1, "observableCount" -> similarStats.observable._2, "similarIocCount" -> similarStats.ioc._1, - "iocCount" -> similarStats.ioc._2 + "iocCount" -> similarStats.ioc._2, + "observableTypes" -> similarStats.types ) } def similarCasesStats(implicit @@ -35,7 +36,7 @@ trait AlertRenderer { else if (x._2.ioc._2 > y._2.ioc._2) -1 else if (x._2.ioc._2 < y._2.ioc._2) 1 else 0 - _.similarCases.fold.domainMap(sc => JsArray(sc.sorted.map(Json.toJson(_)))) + _.similarCases(None).fold.domainMap(sc => JsArray(sc.sorted.map(Json.toJson(_)))) } def alertStatsRenderer[D, G, C <: Converter[D, G]](extraData: Set[String])(implicit diff --git a/thehive/app/org/thp/thehive/models/Case.scala b/thehive/app/org/thp/thehive/models/Case.scala index e15367fc82..fa18c888fe 100644 --- a/thehive/app/org/thp/thehive/models/Case.scala +++ b/thehive/app/org/thp/thehive/models/Case.scala @@ -162,4 +162,4 @@ object RichCase { } } -case class SimilarStats(observable: (Int, Int), ioc: (Int, Int)) +case class SimilarStats(observable: (Int, Int), ioc: (Int, Int), types: Map[String, Long]) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index a91be8df3d..dc5edcdaa7 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -369,12 +369,14 @@ object AlertOps { .count .choose(_.is(P.gt(0)), onTrue = true, onFalse = false) - def similarCases(implicit + def similarCases(maybeCaseFilter: Option[Traversal.V[Case] => Traversal.V[Case]])(implicit authContext: AuthContext - ): Traversal[(RichCase, SimilarStats), JMap[String, Any], Converter[(RichCase, SimilarStats), JMap[String, Any]]] = - observables + ): Traversal[(RichCase, SimilarStats), JMap[String, Any], Converter[(RichCase, SimilarStats), JMap[String, Any]]] = { + val similarObservables = observables .similar .visible + maybeCaseFilter + .fold(similarObservables)(caseFilter => similarObservables.filter(o => caseFilter(o.`case`))) .group(_.by(_.`case`)) .unfold .project( @@ -384,17 +386,27 @@ object AlertOps { _.by(_.richCaseWithoutPerms) .by((_: Traversal.V[Case]).observables.groupCount(_.byValue(_.ioc))) ) - ).by(_.selectValues.unfold.groupCount(_.byValue(_.ioc))) + ) + .by( + _.selectValues + .unfold + .project( + _.by(_.groupCount(_.byValue(_.ioc))) + .by(_.groupCount(_.by(_.typeName))) + ) + ) ) .domainMap { - case ((richCase, obsStats), similarStats) => + case ((richCase, obsStats), (iocStats, observableTypeStats)) => val obsStatsMap = obsStats.mapValues(_.toInt) - val similarStatsMap = similarStats.mapValues(_.toInt) + val similarStatsMap = iocStats.mapValues(_.toInt) richCase -> SimilarStats( similarStatsMap.values.sum -> obsStatsMap.values.sum, - similarStatsMap.getOrElse(true, 0) -> obsStatsMap.getOrElse(true, 0) + similarStatsMap.getOrElse(true, 0) -> obsStatsMap.getOrElse(true, 0), + observableTypeStats ) } + } def alertUserOrganisation( permission: Permission diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 6a9ca76fe3..fa61aa98e0 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -356,6 +356,8 @@ object ObservableOps { def observableType: Traversal.V[ObservableType] = traversal.out[ObservableObservableType].v[ObservableType] + def typeName: Traversal[String, String, Converter[String, String]] = observableType.value(_.name) + def shares: Traversal.V[Share] = traversal.in[ShareObservable].v[Share] def share(implicit authContext: AuthContext): Traversal.V[Share] = share(authContext.organisation) From 795d986ca8d6a5f43d90bba3aca51ed11fc023c1 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Oct 2020 15:45:53 +0200 Subject: [PATCH 067/237] #1482 Prevent import on "exportOnly" MISP --- .../org/thp/thehive/connector/misp/services/MispActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispActor.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispActor.scala index 3f63201e39..fd52006516 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispActor.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispActor.scala @@ -51,7 +51,7 @@ class MispActor @Inject() ( context.become(running) logger.info(s"Synchronising MISP events for ${connector.clients.map(_.name).mkString(",")}") Future - .traverse(connector.clients)(mispImportSrv.syncMispEvents(_)(userSrv.getSystemAuthContext)) + .traverse(connector.clients.filter(_.canImport))(mispImportSrv.syncMispEvents(_)(userSrv.getSystemAuthContext)) .map(_ => ()) .onComplete(status => self ! EndOfSynchro(status)) case other => logger.warn(s"Unknown message $other (${other.getClass})") From 7cc8d5cd4bfbf183c4fe5067a7c3880ce1c72dc5 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Oct 2020 15:54:00 +0200 Subject: [PATCH 068/237] #1501 Fix richUser for system user --- thehive/app/org/thp/thehive/services/UserSrv.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/UserSrv.scala b/thehive/app/org/thp/thehive/services/UserSrv.scala index f1d0cb059d..677f11f88d 100644 --- a/thehive/app/org/thp/thehive/services/UserSrv.scala +++ b/thehive/app/org/thp/thehive/services/UserSrv.scala @@ -263,13 +263,13 @@ object UserOps { ) .domainMap { case (user, attachment, profileOrganisations) => + val avatar = attachment.headOption.map(_.attachmentId) organisation .fold(id => profileOrganisations.find(_._2.exists(_._1 == id)), name => profileOrganisations.find(_._2.exists(_._2 == name))) .orElse(profileOrganisations.headOption) - .fold(throw InternalError(s"")) { // FIXME + .fold(RichUser(user, avatar, Profile.admin.name, Set.empty, "no org")) { // fake user (probably "system") case (profile, organisationIdAndName) => - val avatar = attachment.headOption.map(_.attachmentId) - RichUser(user, avatar, profile.name, profile.permissions, organisationIdAndName.headOption.fold("***")(_._2)) + RichUser(user, avatar, profile.name, profile.permissions, organisationIdAndName.headOption.fold("no org")(_._2)) } } From ce584a8466031fad9f41746484d687c438f534cf Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Oct 2020 09:02:23 +0200 Subject: [PATCH 069/237] #1571 Add alert content in (un)follow and (un)read API for backward compatibility --- ScalliGraph | 2 +- .../thehive/controllers/v0/AlertCtrl.scala | 84 ++++++++++++------- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index a6f82a382e..916c53145f 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit a6f82a382eb7191640a621591b20a072f96375c8 +Subproject commit 916c53145f7a830266ae1bc85df2da10ba86598a diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 85f54705d6..68fa42aaf1 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -206,27 +206,37 @@ class AlertCtrl @Inject() ( def markAsRead(alertId: String): Action[AnyContent] = entrypoint("mark alert as read") .authTransaction(db) { implicit request => implicit graph => - alertSrv - .get(EntityIdOrName(alertId)) - .can(Permissions.manageAlert) - .existsOrFail - .map { _ => - alertSrv.markAsRead(EntityIdOrName(alertId)) - Results.NoContent - } + for { + alert <- + alertSrv + .get(EntityIdOrName(alertId)) + .can(Permissions.manageAlert) + .getOrFail("Alert") + _ <- alertSrv.markAsRead(alert._id) + alertWithObservables <- + alertSrv + .get(alert) + .project(_.by(_.richAlert).by(_.observables.richObservable.fold)) + .getOrFail("Alert") + } yield Results.Ok(alertWithObservables.toJson) } def markAsUnread(alertId: String): Action[AnyContent] = entrypoint("mark alert as unread") .authTransaction(db) { implicit request => implicit graph => - alertSrv - .get(EntityIdOrName(alertId)) - .can(Permissions.manageAlert) - .existsOrFail - .map { _ => - alertSrv.markAsUnread(EntityIdOrName(alertId)) - Results.NoContent - } + for { + alert <- + alertSrv + .get(EntityIdOrName(alertId)) + .can(Permissions.manageAlert) + .getOrFail("Alert") + _ <- alertSrv.markAsUnread(alert._id) + alertWithObservables <- + alertSrv + .get(alert) + .project(_.by(_.richAlert).by(_.observables.richObservable.fold)) + .getOrFail("Alert") + } yield Results.Ok(alertWithObservables.toJson) } def createCase(alertId: String): Action[AnyContent] = @@ -249,27 +259,37 @@ class AlertCtrl @Inject() ( def followAlert(alertId: String): Action[AnyContent] = entrypoint("follow alert") .authTransaction(db) { implicit request => implicit graph => - alertSrv - .get(EntityIdOrName(alertId)) - .can(Permissions.manageAlert) - .existsOrFail - .map { _ => - alertSrv.followAlert(EntityIdOrName(alertId)) - Results.NoContent - } + for { + alert <- + alertSrv + .get(EntityIdOrName(alertId)) + .can(Permissions.manageAlert) + .getOrFail("Alert") + _ <- alertSrv.followAlert(alert._id) + alertWithObservables <- + alertSrv + .get(alert) + .project(_.by(_.richAlert).by(_.observables.richObservable.fold)) + .getOrFail("Alert") + } yield Results.Ok(alertWithObservables.toJson) } def unfollowAlert(alertId: String): Action[AnyContent] = entrypoint("unfollow alert") .authTransaction(db) { implicit request => implicit graph => - alertSrv - .get(EntityIdOrName(alertId)) - .can(Permissions.manageAlert) - .existsOrFail - .map { _ => - alertSrv.unfollowAlert(EntityIdOrName(alertId)) - Results.NoContent - } + for { + alert <- + alertSrv + .get(EntityIdOrName(alertId)) + .can(Permissions.manageAlert) + .getOrFail("Alert") + _ <- alertSrv.unfollowAlert(alert._id) + alertWithObservables <- + alertSrv + .get(alert) + .project(_.by(_.richAlert).by(_.observables.richObservable.fold)) + .getOrFail("Alert") + } yield Results.Ok(alertWithObservables.toJson) } private def createObservable(observable: InputObservable)(implicit From 1ebf0abc10bc96444bd3e7a63f2e06e3f4c2e8af Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Oct 2020 17:39:27 +0200 Subject: [PATCH 070/237] #1561 Use multiple transactions on bulk observable creation --- ScalliGraph | 2 +- .../controllers/v0/ObservableCtrl.scala | 72 +++++++++++++------ 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 916c53145f..7491bd43bb 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 916c53145f7a830266ae1bc85df2da10ba86598a +Subproject commit 7491bd43bb5f6b72079f79aafb1c1fa15e83f7b1 diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 329706882a..35e834e5da 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -16,7 +16,7 @@ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TagOps._ import org.thp.thehive.services._ -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.{JsArray, JsObject, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} import scala.util.Success @@ -28,6 +28,7 @@ class ObservableCtrl @Inject() ( observableSrv: ObservableSrv, observableTypeSrv: ObservableTypeSrv, caseSrv: CaseSrv, + errorHandler: ErrorHandler, @Named("v0") override val queryExecutor: QueryExecutor, override val publicData: PublicObservable ) extends ObservableRenderer @@ -35,30 +36,55 @@ class ObservableCtrl @Inject() ( def create(caseId: String): Action[AnyContent] = entrypoint("create artifact") .extract("artifact", FieldsParser[InputObservable]) - .authTransaction(db) { implicit request => implicit graph => + .auth { implicit request => val inputObservable: InputObservable = request.body("artifact") - for { - case0 <- - caseSrv - .get(EntityIdOrName(caseId)) - .can(Permissions.manageObservable) - .orFail(AuthorizationError("Operation not permitted")) - observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) - observablesWithData <- - inputObservable - .data - .toTry(d => observableSrv.create(inputObservable.toObservable, observableType, d, inputObservable.tags, Nil)) - observableWithAttachment <- - inputObservable - .attachment - .map(a => observableSrv.create(inputObservable.toObservable, observableType, a, inputObservable.tags, Nil)) - .flip - createdObservables <- (observablesWithData ++ observableWithAttachment).toTry { richObservables => - caseSrv - .addObservable(case0, richObservables) - .map(_ => richObservables) + db + .roTransaction { implicit graph => + for { + case0 <- + caseSrv + .get(EntityIdOrName(caseId)) + .can(Permissions.manageObservable) + .orFail(AuthorizationError("Operation not permitted")) + observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) + } yield (case0, observableType) + } + .flatMap { + case (case0, observableType) => + db + .tryTransaction { implicit graph => + inputObservable + .attachment + .map { a => + observableSrv + .create(inputObservable.toObservable, observableType, a, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) + } + .flip + } + .map { + case None => + val (successes, failures) = inputObservable + .data + .foldLeft(Seq.empty[JsValue] -> Seq.empty[JsValue]) { + case ((successes, failures), data) => + db + .tryTransaction { implicit graph => + observableSrv + .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) + } + .fold( + failure => + (successes, failures :+ errorHandler.toErrorResult(failure)._2 ++ Json.obj("object" -> Json.obj("data" -> data))), + success => (successes :+ success, failures) + ) + } + if (failures.isEmpty) Results.Created(JsArray(successes)) + else Results.MultiStatus(Json.obj("success" -> successes, "failure" -> failures)) + case Some(output) => Results.Created(output) + } } - } yield Results.Created(createdObservables.toJson) } def get(observableId: String): Action[AnyContent] = From e4cb654eaa0559a56a6da88a784a57137715857c Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Oct 2020 18:16:16 +0200 Subject: [PATCH 071/237] #1571 Fix tests --- .../org/thp/thehive/controllers/v0/AlertCtrlTest.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala index 14a594cb2f..0741f9acac 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala @@ -196,7 +196,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { val request2 = FakeRequest("POST", "/api/v0/alert/testType;testSource;ref3/markAsRead") .withHeaders("user" -> "certuser@thehive.local") val result2 = app[AlertCtrl].markAsRead("testType;testSource;ref3")(request2) - status(result2) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") + status(result2) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result2)}") val request3 = FakeRequest("GET", "/api/v0/alert/testType;testSource;ref3") .withHeaders("user" -> "certuser@thehive.local") @@ -207,7 +207,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { val request4 = FakeRequest("POST", "/api/v0/alert/testType;testSource;ref3/markAsUnread") .withHeaders("user" -> "certuser@thehive.local") val result4 = app[AlertCtrl].markAsUnread("testType;testSource;ref3")(request4) - status(result4) should equalTo(204).updateMessage(s => s"$s\n${contentAsString(result4)}") + status(result4) should equalTo(200).updateMessage(s => s"$s\n${contentAsString(result4)}") val request5 = FakeRequest("GET", "/api/v0/alert/testType;testSource;ref3") .withHeaders("user" -> "certuser@thehive.local") @@ -226,7 +226,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { val request2 = FakeRequest("POST", "/api/v0/alert/testType;testSource;ref3/unfollow") .withHeaders("user" -> "certuser@thehive.local") val result2 = app[AlertCtrl].unfollowAlert("testType;testSource;ref3")(request2) - status(result2) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result2)}") + status(result2) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result2)}") val request3 = FakeRequest("GET", "/api/v0/alert/testType;testSource;ref3") .withHeaders("user" -> "certuser@thehive.local") @@ -237,7 +237,7 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { val request4 = FakeRequest("POST", "/api/v0/alert/testType;testSource;ref3/follow") .withHeaders("user" -> "certuser@thehive.local") val result4 = app[AlertCtrl].followAlert("testType;testSource;ref3")(request4) - status(result4) should equalTo(204).updateMessage(s => s"$s\n${contentAsString(result4)}") + status(result4) should equalTo(200).updateMessage(s => s"$s\n${contentAsString(result4)}") val request5 = FakeRequest("GET", "/api/v0/alert/testType;testSource;ref3") .withHeaders("user" -> "certuser@thehive.local") From 3dcda35e516147ef2867f7e3343b4c18a3bed442 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Oct 2020 18:41:07 +0200 Subject: [PATCH 072/237] #1561 Fix output when observable creation request contains attachment and data --- .../controllers/v0/ObservableCtrl.scala | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 35e834e5da..e52c3458b2 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -49,41 +49,45 @@ class ObservableCtrl @Inject() ( observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) } yield (case0, observableType) } - .flatMap { + .map { case (case0, observableType) => - db - .tryTransaction { implicit graph => - inputObservable - .attachment - .map { a => + val initialSuccessesAndFailures: (Seq[JsValue], Seq[JsValue]) = inputObservable + .attachment + .map { attachmentFile => + db + .tryTransaction { implicit graph => observableSrv - .create(inputObservable.toObservable, observableType, a, inputObservable.tags, Nil) + .create(inputObservable.toObservable, observableType, attachmentFile, inputObservable.tags, Nil) .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) } - .flip + .fold( + e => + Nil -> Seq( + errorHandler.toErrorResult(e)._2 ++ Json + .obj("object" -> Json.obj("attachment" -> Json.obj("name" -> attachmentFile.filename))) + ), + s => Seq(s) -> Nil + ) } - .map { - case None => - val (successes, failures) = inputObservable - .data - .foldLeft(Seq.empty[JsValue] -> Seq.empty[JsValue]) { - case ((successes, failures), data) => - db - .tryTransaction { implicit graph => - observableSrv - .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) - .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) - } - .fold( - failure => - (successes, failures :+ errorHandler.toErrorResult(failure)._2 ++ Json.obj("object" -> Json.obj("data" -> data))), - success => (successes :+ success, failures) - ) + .getOrElse(Nil -> Nil) + + val (successes, failures) = inputObservable + .data + .foldLeft(initialSuccessesAndFailures) { + case ((successes, failures), data) => + db + .tryTransaction { implicit graph => + observableSrv + .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) } - if (failures.isEmpty) Results.Created(JsArray(successes)) - else Results.MultiStatus(Json.obj("success" -> successes, "failure" -> failures)) - case Some(output) => Results.Created(output) + .fold( + failure => (successes, failures :+ errorHandler.toErrorResult(failure)._2 ++ Json.obj("object" -> Json.obj("data" -> data))), + success => (successes :+ success, failures) + ) } + if (failures.isEmpty) Results.Created(JsArray(successes)) + else Results.MultiStatus(Json.obj("success" -> successes, "failure" -> failures)) } } From 4393938208edd88d759d8669742919cfca1e0aef Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 19 Oct 2020 08:05:15 +0200 Subject: [PATCH 073/237] #1566 Export tags to MISP event --- .../org/thp/misp/client/MispClient.scala | 91 +++++++++++++------ .../scala/org/thp/misp/dto/Attribute.scala | 4 +- .../src/main/scala/org/thp/misp/dto/Tag.scala | 5 +- .../misp/services/MispExportSrv.scala | 34 +++++-- .../misp/services/TheHiveMispClient.scala | 7 +- .../services/TestMispClientProvider.scala | 34 +++---- 6 files changed, 119 insertions(+), 56 deletions(-) diff --git a/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala b/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala index 3a664de478..edae588bc5 100644 --- a/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala +++ b/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala @@ -7,7 +7,7 @@ import akka.stream.alpakka.json.scaladsl.JsonReader import akka.stream.scaladsl.{JsonFraming, Source} import akka.util.ByteString import org.thp.client.{ApplicationError, Authentication, ProxyWS} -import org.thp.misp.dto.{Attribute, Event, Organisation, User} +import org.thp.misp.dto.{Attribute, Event, Organisation, Tag, User} import org.thp.scalligraph.InternalError import play.api.Logger import play.api.http.Status @@ -45,10 +45,11 @@ class MispClient( Failure(InternalError(s"MISP server $name is inaccessible", t)) } - private def configuredProxy: Option[String] = ws match { - case c: ProxyWS => c.proxy.map(p => s"http://${p.host}:${p.port}") - case _ => None - } + private def configuredProxy: Option[String] = + ws match { + case c: ProxyWS => c.proxy.map(p => s"http://${p.host}:${p.port}") + case _ => None + } logger.info(s"""Add MISP connection $name | url: $baseUrl | proxy: ${configuredProxy.getOrElse("")} @@ -62,27 +63,52 @@ class MispClient( private def request(url: String): WSRequest = auth(ws.url(s"$strippedUrl/$url").withHttpHeaders("Accept" -> "application/json")) - private def get(url: String)(implicit ec: ExecutionContext): Future[JsValue] = + private def get(url: String)(implicit ec: ExecutionContext): Future[JsValue] = { + logger.trace(s"MISP request: GET $url") request(url).get().transform { - case Success(r) if r.status == Status.OK => Success(r.json) - case Success(r) => Try(r.json) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Success(r.json) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Try(r.json) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } + } - private def post(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[JsValue] = + private def post(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[JsValue] = { + logger.trace(s"MISP request: POST $url\n$body") request(url).post(body).transform { - case Success(r) if r.status == Status.OK => Success(r.json) - case Success(r) => Try(r.json) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Success(r.json) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Try(r.json) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } + } - private def post(url: String, body: Source[ByteString, _])(implicit ec: ExecutionContext): Future[JsValue] = + private def post(url: String, body: Source[ByteString, _])(implicit ec: ExecutionContext): Future[JsValue] = { + logger.trace(s"MISP request: POST $url (stream body)") request(url).post(body).transform { - case Success(r) if r.status == Status.OK => Success(r.json) - case Success(r) => Try(r.json) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Success(r.json) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Try(r.json) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } -// + } + + // // private def getStream(url: String)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] = // request(url).withMethod("GET").stream().transform { // case Success(r) if r.status == Status.OK => Success(r.bodyAsSource) @@ -90,12 +116,20 @@ class MispClient( // case Failure(t) => throw t // } - private def postStream(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] = + private def postStream(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] = { + logger.trace(s"MISP request: POST $url\n$body") request(url).withMethod("POST").withBody(body).stream().transform { - case Success(r) if r.status == Status.OK => Success(r.bodyAsSource) - case Success(r) => Try(r.bodyAsSource) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText} (stream body)") + Success(r.bodyAsSource) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText} (stream body)") + Try(r.bodyAsSource) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } + } def getCurrentUser(implicit ec: ExecutionContext): Future[User] = { logger.debug("Get current user") @@ -177,7 +211,8 @@ class MispClient( maybeAttribute.fold(error => { logger.warn(s"Attribute has invalid format: ${data.decodeString("UTF-8")}", error); Nil }, List(_)) } .mapAsyncUnordered(2) { - case attribute @ Attribute(id, "malware-sample" | "attachment", _, _, _, _, _, _, _, None, _, _, _, _) => // TODO need to unzip malware samples ? + case attribute @ Attribute(id, "malware-sample" | "attachment", _, _, _, _, _, _, _, None, _, _, _, _) => + // TODO need to unzip malware samples ? downloadAttachment(id).map { case (filename, contentType, src) => attribute.copy(data = Some((filename, contentType, src))) } @@ -204,14 +239,16 @@ class MispClient( case Failure(t) => throw t } - def uploadAttachment(eventId: String, comment: String, filename: String, data: Source[ByteString, _])( - implicit ec: ExecutionContext + def uploadAttachment(eventId: String, comment: String, filename: String, data: Source[ByteString, _])(implicit + ec: ExecutionContext ): Future[JsValue] = { val stream = data .via(Base64Flow.encode()) .intersperse( ByteString( - s"""{"request":{"category":"Payload delivery","type":"malware-sample","comment":${JsString(comment).toString},"files":[{"filename":${JsString( + s"""{"request":{"category":"Payload delivery","type":"malware-sample","comment":${JsString( + comment + ).toString},"files":[{"filename":${JsString( filename ).toString},"data":"""" ), @@ -229,6 +266,7 @@ class MispClient( analysis: Int, distribution: Int, attributes: Seq[Attribute], + tags: Seq[Tag], extendsEvent: Option[String] = None )(implicit ec: ExecutionContext): Future[String] = { logger.debug(s"Create MISP event $info, with ${attributes.size} attributes") @@ -243,6 +281,7 @@ class MispClient( "analysis" -> analysis.toString, "distribution" -> distribution, "Attribute" -> stringAttributes, + "Tag" -> tags, "extends_uuid" -> extendsEvent ) ) diff --git a/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala b/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala index 33c65b1927..1b4dc4fcdb 100644 --- a/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala +++ b/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala @@ -55,8 +55,8 @@ object Attribute { "category" -> attribute.category, "type" -> attribute.`type`, "value" -> attribute.value, - "comment" -> attribute.comment -// "Tag" -> attribute.tags + "comment" -> attribute.comment, + "Tag" -> attribute.tags ) } } diff --git a/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala b/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala index de4880844f..683b1ee489 100644 --- a/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala +++ b/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala @@ -20,8 +20,5 @@ object Tag { } and (JsPath \ "exportable").readNullable[Boolean])(Tag.apply _) - implicit val writes: Writes[Tag] = Writes[Tag] { - case Tag(Some(id), name, colour, _) => Json.obj("id" -> id, "name" -> name, "colour" -> colour.map(c => f"#$c%06X")) - case Tag(_, name, _, _) => JsString(name) - } + implicit val writes: Writes[Tag] = Json.writes[Tag] } diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala index fdcfabbb56..58c00318cd 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala @@ -31,7 +31,12 @@ class MispExportSrv @Inject() ( lazy val logger: Logger = Logger(getClass) - def observableToAttribute(observable: RichObservable): Option[Attribute] = + def observableToAttribute(observable: RichObservable, exportTags: Boolean): Option[Attribute] = { + lazy val mispTags = + if (exportTags) + observable.tags.map(t => MispTag(None, t.toString, Some(t.colour), None)) ++ tlpTags.get(observable.tlp) + else + tlpTags.get(observable.tlp).toSeq connector .attributeConverter(observable.`type`) .map { @@ -50,7 +55,7 @@ class MispExportSrv @Inject() ( value = observable.data.fold(observable.attachment.get.name)(_.data), firstSeen = None, lastSeen = None, - tags = observable.tags.map(t => MispTag(None, t.toString, Some(t.colour), None)) + tags = mispTags ) } .orElse { @@ -59,6 +64,7 @@ class MispExportSrv @Inject() ( ) None } + } def getMispClient(mispId: String): Future[TheHiveMispClient] = connector @@ -77,8 +83,8 @@ class MispExportSrv @Inject() ( .filterByType("misp") .headOption - def getAttributes(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Iterator[Attribute] = - caseSrv.get(`case`).observables.isIoc.richObservable.toIterator.flatMap(observableToAttribute) + def getAttributes(`case`: Case with Entity, exportTags: Boolean)(implicit graph: Graph, authContext: AuthContext): Iterator[Attribute] = + caseSrv.get(`case`).observables.isIoc.richObservable.toIterator.flatMap(observableToAttribute(_, exportTags)) def removeDuplicateAttributes(attributes: Iterator[Attribute]): Seq[Attribute] = { var attrSet = Set.empty[(String, String, String)] @@ -93,9 +99,21 @@ class MispExportSrv @Inject() ( builder.result() } - def createEvent(client: TheHiveMispClient, `case`: Case, attributes: Seq[Attribute], extendsEvent: Option[String])(implicit + val tlpTags = Map( + 0 -> MispTag(None, "tlp:white", None, None), + 1 -> MispTag(None, "tlp:green", None, None), + 2 -> MispTag(None, "tlp:amber", None, None), + 3 -> MispTag(None, "tlp:red", None, None) + ) + def createEvent(client: TheHiveMispClient, `case`: Case with Entity, attributes: Seq[Attribute], extendsEvent: Option[String])(implicit ec: ExecutionContext - ): Future[String] = + ): Future[String] = { + val mispTags = + if (client.exportCaseTags) + db.roTransaction { implicit graph => + caseSrv.get(`case`._id).tags.toSeq.map(t => MispTag(None, t.toString, Some(t.colour), None)) ++ tlpTags.get(`case`.tlp) + } + else tlpTags.get(`case`.tlp).toSeq client.createEvent( info = `case`.title, date = `case`.startDate, @@ -104,8 +122,10 @@ class MispExportSrv @Inject() ( analysis = 0, distribution = 0, attributes = attributes, + tags = mispTags, extendsEvent = extendsEvent ) + } def createAlert(client: TheHiveMispClient, `case`: Case with Entity, eventId: String)(implicit graph: Graph, @@ -147,7 +167,7 @@ class MispExportSrv @Inject() ( orgName <- Future.fromTry(client.currentOrganisationName) maybeAlert = db.roTransaction(implicit graph => getAlert(`case`, orgName)) _ = logger.debug(maybeAlert.fold("Related MISP event doesn't exist")(a => s"Related MISP event found : ${a.sourceRef}")) - attributes = db.roTransaction(implicit graph => removeDuplicateAttributes(getAttributes(`case`))) + attributes = db.roTransaction(implicit graph => removeDuplicateAttributes(getAttributes(`case`, client.exportObservableTags))) eventId <- createEvent(client, `case`, attributes, maybeAlert.map(_.sourceRef)) _ <- Future.fromTry(db.tryTransaction(implicit graph => createAlert(client, `case`, eventId))) } yield eventId diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala index 4841cda933..50f0bc2dfb 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala @@ -29,6 +29,7 @@ case class TheHiveMispClientConfig( caseTemplate: Option[String], artifactTags: Seq[String] = Nil, exportCaseTags: Boolean = false, + exportObservableTags: Boolean = false, includedTheHiveOrganisations: Seq[String] = Seq("*"), excludedTheHiveOrganisations: Seq[String] = Nil ) @@ -49,6 +50,7 @@ object TheHiveMispClientConfig { caseTemplate <- (JsPath \ "caseTemplate").readNullable[String] artifactTags <- (JsPath \ "tags").readWithDefault[Seq[String]](Nil) exportCaseTags <- (JsPath \ "exportCaseTags").readWithDefault[Boolean](false) + exportObservableTags <- (JsPath \ "exportObservableTags").readWithDefault[Boolean](false) includedTheHiveOrganisations <- (JsPath \ "includedTheHiveOrganisations").readWithDefault[Seq[String]](Seq("*")) excludedTheHiveOrganisations <- (JsPath \ "excludedTheHiveOrganisations").readWithDefault[Seq[String]](Nil) } yield TheHiveMispClientConfig( @@ -64,6 +66,7 @@ object TheHiveMispClientConfig { caseTemplate, artifactTags, exportCaseTags, + exportObservableTags, includedTheHiveOrganisations, excludedTheHiveOrganisations ) @@ -100,7 +103,8 @@ class TheHiveMispClient( purpose: MispPurpose.Value, val caseTemplate: Option[String], artifactTags: Seq[String], // FIXME use artifactTags - exportCaseTags: Boolean, // FIXME use exportCaseTags + val exportCaseTags: Boolean, + val exportObservableTags: Boolean, includedTheHiveOrganisations: Seq[String], excludedTheHiveOrganisations: Seq[String] ) extends MispClient( @@ -128,6 +132,7 @@ class TheHiveMispClient( config.caseTemplate, config.artifactTags, config.exportCaseTags, + config.exportObservableTags, config.includedTheHiveOrganisations, config.excludedTheHiveOrganisations ) diff --git a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala index 9538840aec..a881cc7298 100644 --- a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala +++ b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala @@ -51,20 +51,22 @@ class TestMispClientProvider @Inject() (Action: DefaultActionBuilder, implicit v Json.parse(data) } - override def get(): TheHiveMispClient = new TheHiveMispClient( - name = "test", - baseUrl = baseUrl, - auth = NoAuthentication, - ws = ws, - maxAge = None, - excludedOrganisations = Nil, - excludedTags = Set.empty, - whitelistTags = Set.empty, - purpose = MispPurpose.ImportAndExport, - caseTemplate = None, - artifactTags = Seq("TEST"), - exportCaseTags = true, - includedTheHiveOrganisations = Seq("*"), - excludedTheHiveOrganisations = Nil - ) + override def get(): TheHiveMispClient = + new TheHiveMispClient( + name = "test", + baseUrl = baseUrl, + auth = NoAuthentication, + ws = ws, + maxAge = None, + excludedOrganisations = Nil, + excludedTags = Set.empty, + whitelistTags = Set.empty, + purpose = MispPurpose.ImportAndExport, + caseTemplate = None, + artifactTags = Seq("TEST"), + exportCaseTags = true, + exportObservableTags = true, + includedTheHiveOrganisations = Seq("*"), + excludedTheHiveOrganisations = Nil + ) } From 49042659ad70971decb79c2a14082728af7be8f7 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 19 Oct 2020 09:51:11 +0200 Subject: [PATCH 074/237] #1474 Fix dashboard sharing --- ScalliGraph | 2 +- .../controllers/v0/DashboardCtrl.scala | 39 ++++++++++--------- .../thp/thehive/services/DashboardSrv.scala | 4 +- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 7491bd43bb..5b48cd7508 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 7491bd43bb5f6b72079f79aafb1c1fa15e83f7b1 +Subproject commit 5b48cd75080a17ba0986ddcc7c572a96de86ddaa diff --git a/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala index 3402cd5798..309afaaf5f 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala @@ -110,28 +110,29 @@ class PublicDashboard @Inject() ( .property("description", UMapping.string)(_.field.updatable) .property("definition", UMapping.string)(_.field.updatable) .property("status", UMapping.string)( - _.select(_.organisation.fold.domainMap(d => if (d.isEmpty) "Private" else "Shared")).custom { // TODO replace by choose step - case (_, "Shared", vertex, _, graph, authContext) => - for { - dashboard <- dashboardSrv.get(vertex)(graph).filter(_.user.current(authContext)).getOrFail("Dashboard") - _ <- dashboardSrv.share(dashboard, authContext.organisation, writable = false)(graph, authContext) - } yield Json.obj("status" -> "Shared") + _.select(_.choose(_.organisation, "Shared", "Private")) + .custom { + case (_, "Shared", vertex, _, graph, authContext) => + for { + dashboard <- dashboardSrv.get(vertex)(graph).filter(_.user.current(authContext)).getOrFail("Dashboard") + _ <- dashboardSrv.share(dashboard, authContext.organisation, writable = false)(graph, authContext) + } yield Json.obj("status" -> "Shared") - case (_, "Private", vertex, _, graph, authContext) => - for { - d <- dashboardSrv.get(vertex)(graph).filter(_.user.current(authContext)).getOrFail("Dashboard") - _ <- dashboardSrv.unshare(d, authContext.organisation)(graph, authContext) - } yield Json.obj("status" -> "Private") + case (_, "Private", vertex, _, graph, authContext) => + for { + d <- dashboardSrv.get(vertex)(graph).filter(_.user.current(authContext)).getOrFail("Dashboard") + _ <- dashboardSrv.unshare(d, authContext.organisation)(graph, authContext) + } yield Json.obj("status" -> "Private") - case (_, "Deleted", vertex, _, graph, authContext) => - for { - d <- dashboardSrv.get(vertex)(graph).filter(_.user.current(authContext)).getOrFail("Dashboard") - _ <- dashboardSrv.remove(d)(graph, authContext) - } yield Json.obj("status" -> "Deleted") + case (_, "Deleted", vertex, _, graph, authContext) => + for { + d <- dashboardSrv.get(vertex)(graph).filter(_.user.current(authContext)).getOrFail("Dashboard") + _ <- dashboardSrv.remove(d)(graph, authContext) + } yield Json.obj("status" -> "Deleted") - case (_, status, _, _, _, _) => - Failure(InvalidFormatAttributeError("status", "String", Set("Shared", "Private", "Deleted"), FString(status))) - } + case (_, status, _, _, _, _) => + Failure(InvalidFormatAttributeError("status", "String", Set("Shared", "Private", "Deleted"), FString(status))) + } ) .build } diff --git a/thehive/app/org/thp/thehive/services/DashboardSrv.scala b/thehive/app/org/thp/thehive/services/DashboardSrv.scala index c799d8c106..ddedda2344 100644 --- a/thehive/app/org/thp/thehive/services/DashboardSrv.scala +++ b/thehive/app/org/thp/thehive/services/DashboardSrv.scala @@ -103,10 +103,10 @@ object DashboardOps { def organisationShares: Traversal[Seq[(String, Boolean)], JList[JMap[String, Any]], Converter[Seq[(String, Boolean)], JList[JMap[String, Any]]]] = traversal - .outE[OrganisationDashboard] + .inE[OrganisationDashboard] .project( _.byValue(_.writable) - .by(_.inV) + .by(_.outV) ) .fold .domainMap(_.map { case (writable, orgs) => (orgs.value[String]("name"), writable) }) From 48d0fb96273068bc754034841a15373c1970255f Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 19 Oct 2020 16:28:01 +0200 Subject: [PATCH 075/237] #1541 Use API v1 in case merge dialog --- .../controllers/case/CaseMergeModalCtrl.js | 27 ++++++++++++------- .../app/views/partials/case/case.merge.html | 2 +- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseMergeModalCtrl.js b/frontend/app/scripts/controllers/case/CaseMergeModalCtrl.js index 71e40d61e9..fc55e1b2e1 100644 --- a/frontend/app/scripts/controllers/case/CaseMergeModalCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseMergeModalCtrl.js @@ -4,7 +4,7 @@ angular.module('theHiveControllers') .controller('CaseMergeModalCtrl', CaseMergeModalCtrl); - function CaseMergeModalCtrl($state, $uibModalInstance, $q, SearchSrv, CaseSrv, UserSrv, NotificationSrv, source, title, prompt) { + function CaseMergeModalCtrl($state, $uibModalInstance, $q, QuerySrv, SearchSrv, CaseSrv, UserSrv, NotificationSrv, source, title, prompt) { var me = this; this.source = source; @@ -13,32 +13,41 @@ this.prompt = prompt; this.search = { type: 'title', - placeholder: 'Search by case title', + placeholder: 'Search by case title. "Ex: Malware*"', minInputLength: 1, input: null, cases: [] }; this.getUserInfo = UserSrv.getCache; - this.getCaseByTitle = function(type, input) { + this.getCaseList = function(type, input) { var defer = $q.defer(); - var query = (type === 'title') ? { - _like: {title: input} + var filter = (type === 'title') ? { + _like: { + title: input + } } : { - caseId: Number.parseInt(input) + _field: 'number', + _value: Number.parseInt(input) }; - SearchSrv(function(data /*, total*/ ) { + QuerySrv.call('v1', + [{_name: 'listCase'}], + { + filter:filter, + name: 'get-case-for-merge' + } + ).then(function(data) { defer.resolve(data); - }, query, 'case', 'all'); + }); return defer.promise; }; this.format = function(caze) { if (caze) { - return '#' + caze.caseId + ' - ' + caze.title; + return '#' + caze.number + ' - ' + caze.title; } return null; }; diff --git a/frontend/app/views/partials/case/case.merge.html b/frontend/app/views/partials/case/case.merge.html index d019a8072b..14b625697b 100644 --- a/frontend/app/views/partials/case/case.merge.html +++ b/frontend/app/views/partials/case/case.merge.html @@ -14,7 +14,7 @@

{{dialog.title | ellipsis:50}}

Date: Mon, 19 Oct 2020 16:30:56 +0200 Subject: [PATCH 076/237] #1489 Display missin case number in waiting tasks and my tasks pages --- frontend/app/views/directives/entity-link.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/views/directives/entity-link.html b/frontend/app/views/directives/entity-link.html index 67e838e7fc..a3e9f6cb19 100644 --- a/frontend/app/views/directives/entity-link.html +++ b/frontend/app/views/directives/entity-link.html @@ -1,13 +1,13 @@ -  #{{value.caseId}} - {{value.title}}  +  #{{value.caseId || value.number}} - {{value.title}}  -  #{{value.caseId}} - {{value.title}}  +  #{{value.caseId || value.number}} - {{value.title}}  From 055158a5d38e70ca11bcb62085ad738e6ac3581a Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 19 Oct 2020 11:12:50 +0200 Subject: [PATCH 077/237] #1474 Add dashboard operation in flow --- frontend/app/views/directives/flow/dashboard.html | 14 ++++++++++++++ frontend/app/views/directives/flow/flow.html | 1 + 2 files changed, 15 insertions(+) create mode 100644 frontend/app/views/directives/flow/dashboard.html diff --git a/frontend/app/views/directives/flow/dashboard.html b/frontend/app/views/directives/flow/dashboard.html new file mode 100644 index 0000000000..bd4111eb82 --- /dev/null +++ b/frontend/app/views/directives/flow/dashboard.html @@ -0,0 +1,14 @@ +
+
+ + {{base.details.title}} +
+ +
+ Dashboard {{base.details.title}} added +
+ +
+ Dashboard {{base.details.title}} removed +
+
diff --git a/frontend/app/views/directives/flow/flow.html b/frontend/app/views/directives/flow/flow.html index 508bc5326a..03a9bab26b 100644 --- a/frontend/app/views/directives/flow/flow.html +++ b/frontend/app/views/directives/flow/flow.html @@ -10,5 +10,6 @@ +

From d5cd5ffc12ced7b4da7412363188e21b8eeb36ba Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 19 Oct 2020 16:48:05 +0200 Subject: [PATCH 078/237] #1541 Add the field "number" in case (v1) --- thehive/app/org/thp/thehive/controllers/v1/Properties.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 8358cc96d7..608c62cd71 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -98,6 +98,7 @@ class Properties @Inject() ( .property("severity", UMapping.int)(_.field.updatable) .property("startDate", UMapping.date)(_.field.updatable) .property("endDate", UMapping.date.optional)(_.field.updatable) + .property("number", UMapping.int)(_.field.readonly) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) .custom { (_, value, vertex, _, graph, authContext) => From 1af72f9a92d52e09bc081650e34945c091831174 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 19 Oct 2020 16:49:48 +0200 Subject: [PATCH 079/237] #1478 Fix date format in migration --- .../scala/org/thp/thehive/migration/Input.scala | 13 ++++++++++--- .../scala/org/thp/thehive/migration/Migrate.scala | 14 ++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/Input.scala index 6e685d5405..e06a97cefb 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Input.scala @@ -29,9 +29,16 @@ object Filter { new SimpleDateFormat("MMdd") ) def parseDate(s: String): Try[Date] = - dateFormats.foldLeft[Try[Date]](Failure(new ParseException(s"Unparseable date: $s", 0))) { (acc, format) => - acc.recoverWith { case _ => Try(format.parse(s)) } - } + dateFormats + .foldLeft[Try[Date]](Failure(new ParseException(s"Unparseable date: $s", 0))) { (acc, format) => + acc.recoverWith { case _ => Try(format.parse(s)) } + } + .recoverWith { + case _ => + Failure( + new ParseException(s"Unparseable date: $s\nExpected format is ${dateFormats.map(_.toPattern).mkString("\"", "\" or \"", "\"")}", 0) + ) + } def readDate(dateConfigName: String, ageConfigName: String) = Try(config.getString(dateConfigName)) .flatMap(parseDate) diff --git a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala index b55d474621..3c244a5085 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala @@ -1,6 +1,8 @@ package org.thp.thehive.migration import java.io.File +import java.text.SimpleDateFormat +import java.util.Date import akka.actor.ActorSystem import akka.stream.Materializer @@ -71,11 +73,11 @@ object Migrate extends App with MigrationOps { .valueName("") .text("migrate only cases whose age is greater than ") .action((v, c) => addConfig(c, "input.filter.minCaseAge" -> v.toString)), - opt[Duration]("case-from-date") + opt[String]("case-from-date") .valueName("") .text("migrate only cases created from ") .action((v, c) => addConfig(c, "input.filter.caseFromDate" -> v.toString)), - opt[Duration]("case-until-date") + opt[String]("case-until-date") .valueName("") .text("migrate only cases created until ") .action((v, c) => addConfig(c, "input.filter.caseUntilDate" -> v.toString)), @@ -97,11 +99,11 @@ object Migrate extends App with MigrationOps { .valueName("") .text("migrate only alerts whose age is greater than ") .action((v, c) => addConfig(c, "input.filter.minAlertAge" -> v.toString)), - opt[Duration]("alert-from-date") + opt[String]("alert-from-date") .valueName("") .text("migrate only alerts created from ") .action((v, c) => addConfig(c, "input.filter.alertFromDate" -> v.toString)), - opt[Duration]("alert-until-date") + opt[String]("alert-until-date") .valueName("") .text("migrate only alerts created until ") .action((v, c) => addConfig(c, "input.filter.alertUntilDate" -> v.toString)), @@ -114,11 +116,11 @@ object Migrate extends App with MigrationOps { .valueName("") .text("migrate only audits whose age is greater than ") .action((v, c) => addConfig(c, "input.filter.maxAuditAge" -> v.toString)), - opt[Duration]("audit-from-date") + opt[String]("audit-from-date") .valueName("") .text("migrate only audits created from ") .action((v, c) => addConfig(c, "input.filter.auditFromDate" -> v.toString)), - opt[Duration]("audit-until-date") + opt[String]("audit-until-date") .valueName("") .text("migrate only audits created until ") .action((v, c) => addConfig(c, "input.filter.auditUntilDate" -> v.toString)), From 2cf110a88e5cc8cb09fafdaa745a83cacd696a94 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 19 Oct 2020 17:00:23 +0200 Subject: [PATCH 080/237] #1541 Fix alert merge into case process to use API v1 attributes --- frontend/app/scripts/controllers/alert/AlertEventCtrl.js | 2 +- frontend/app/scripts/controllers/alert/AlertListCtrl.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js index 0882b3ac47..2a03738846 100644 --- a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js @@ -132,7 +132,7 @@ }); caseModal.result.then(function(selectedCase) { - self.mergeIntoCase(selectedCase.id); + self.mergeIntoCase(selectedCase._id); }).catch(function(err) { if(err && !_.isString(err)) { NotificationSrv.error('AlertEventCtrl', err.data, err.status); diff --git a/frontend/app/scripts/controllers/alert/AlertListCtrl.js b/frontend/app/scripts/controllers/alert/AlertListCtrl.js index b21bdb7a05..8c7d322cf2 100755 --- a/frontend/app/scripts/controllers/alert/AlertListCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertListCtrl.js @@ -381,7 +381,7 @@ }); caseModal.result.then(function(selectedCase) { - return AlertingSrv.bulkMergeInto(_.pluck(self.selection, '_id'), selectedCase.id); + return AlertingSrv.bulkMergeInto(_.pluck(self.selection, '_id'), selectedCase._id); }) .then(function(response) { $rootScope.$broadcast('alert:event-imported'); From d3261cdde3c169ddb77e131660101db021365851 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 19 Oct 2020 17:56:46 +0200 Subject: [PATCH 081/237] #1540 Fix paring of sort field --- thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala index 14e6237461..3577df52f3 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala @@ -44,8 +44,9 @@ trait QueryCtrl { TH3Aggregation.fieldsParser val sortParser: FieldsParser[InputSort] = FieldsParser("sort") { - case (_, FAny(s)) => Good(s) + case (_, FAny(s)) => Good(s.flatMap(_.split(','))) case (_, FSeq(s)) => s.validatedBy(FieldsParser.string.apply) + case (_, FString(s)) => Good(s.split(',').toSeq) case (_, FUndefined) => Good(Nil) }.map("sort") { a => val fields = a.collect { From eedcde4c6cd3f58be2c8902a54dcb08534c2a883 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 20 Oct 2020 09:04:49 +0200 Subject: [PATCH 082/237] #1456 Return session expiration status in stream output --- ScalliGraph | 2 +- .../thehive/controllers/v0/StreamCtrl.scala | 79 ++++++++++--------- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 5b48cd7508..a6637cd432 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 5b48cd75080a17ba0986ddcc7c572a96de86ddaa +Subproject commit a6637cd4321973ec49f8f83df0397e3d7ec3c9af diff --git a/thehive/app/org/thp/thehive/controllers/v0/StreamCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/StreamCtrl.scala index bfb4899386..d3281d63f5 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/StreamCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/StreamCtrl.scala @@ -2,6 +2,7 @@ package org.thp.thehive.controllers.v0 import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.Order +import org.thp.scalligraph.auth.{ExpirationStatus, SessionAuthSrv} import org.thp.scalligraph.controllers.Entrypoint import org.thp.scalligraph.models.{Database, Schema} import org.thp.scalligraph.traversal.TraversalOps._ @@ -11,7 +12,7 @@ import org.thp.thehive.services._ import play.api.libs.json.{JsArray, JsObject, Json} import play.api.mvc.{Action, AnyContent, Results} -import scala.concurrent.ExecutionContext +import scala.concurrent.{ExecutionContext, Future} import scala.util.Success @Singleton @@ -35,43 +36,49 @@ class StreamCtrl @Inject() ( } def get(streamId: String): Action[AnyContent] = - entrypoint("get stream").async { _ => - streamSrv - .get(streamId) - .map { - case auditIds if auditIds.nonEmpty => - db.roTransaction { implicit graph => - val audits = auditSrv - .getMainByIds(Order.desc, auditIds: _*) - .richAuditWithCustomRenderer(auditRenderer) - .toIterator - .map { - case (audit, obj) => - audit - .toJson - .as[JsObject] - .deepMerge( - Json.obj( - "base" -> Json.obj("object" -> obj, "rootId" -> audit.context._id), - "summary" -> jsonSummary(auditSrv, audit.requestId) + entrypoint("get stream").async { request => + if (SessionAuthSrv.isExpired(request)) + Future.successful(Results.Unauthorized) + else + streamSrv + .get(streamId) + .map { + case auditIds if auditIds.nonEmpty => + db.roTransaction { implicit graph => + val audits = auditSrv + .getMainByIds(Order.desc, auditIds: _*) + .richAuditWithCustomRenderer(auditRenderer) + .toIterator + .map { + case (audit, obj) => + audit + .toJson + .as[JsObject] + .deepMerge( + Json.obj( + "base" -> Json.obj("object" -> obj, "rootId" -> audit.context._id), + "summary" -> jsonSummary(auditSrv, audit.requestId) + ) ) - ) - } - Results.Ok(JsArray(audits.toSeq)) - } - case _ => Results.Ok(JsArray.empty) - } + } + if (SessionAuthSrv.isWarning(request)) + new Results.Status(220)(JsArray(audits.toSeq)) + else + Results.Ok(JsArray(audits.toSeq)) + } + case _ if SessionAuthSrv.isWarning(request) => new Results.Status(220)(JsArray.empty) + case _ => Results.Ok(JsArray.empty) + } } - def status: Action[AnyContent] = // TODO - entrypoint("get stream") { _ => - Success( - Results.Ok( - Json.obj( - "remaining" -> 3600, - "warning" -> false - ) - ) - ) + def status: Action[AnyContent] = + entrypoint("get stream") { request => + val status = SessionAuthSrv.expirationStatus(request) match { + case Some(ExpirationStatus.Ok(remaining)) => Json.obj("warning" -> false, "remaining" -> remaining.toMillis) + case Some(ExpirationStatus.Warning(remaining)) => Json.obj("warning" -> true, "remaining" -> remaining.toMillis) + case Some(ExpirationStatus.Error) => Json.obj("warning" -> true, "remaining" -> 0) + case None => Json.obj("warning" -> false, "remaining" -> 1) + } + Success(Results.Ok(status)) } } From f9354b36a9503e98f9da6773ef16767f1be2cd88 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 20 Oct 2020 16:45:26 +0200 Subject: [PATCH 083/237] #1501 Fix admin profile lookup --- thehive/app/org/thp/thehive/services/UserSrv.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/UserSrv.scala b/thehive/app/org/thp/thehive/services/UserSrv.scala index 677f11f88d..fe0ed33fdf 100644 --- a/thehive/app/org/thp/thehive/services/UserSrv.scala +++ b/thehive/app/org/thp/thehive/services/UserSrv.scala @@ -233,7 +233,7 @@ object UserOps { .project( _.byValue(_.login) .byValue(_.name) - .by(_.role.filter(_.organisation.get(organisationName)).profile) + .by(_.role.filter(_.organisation.get(organisationName)).profile.fold) .by(_.organisations.get(organisationName).value(_.name).limit(1).fold) ) .domainMap { @@ -241,7 +241,7 @@ object UserOps { val scope = if (org.contains(Organisation.administration.name)) "admin" else "organisation" - val permissions = Permissions.forScope(scope) & profile.permissions + val permissions = Permissions.forScope(scope) & profile.headOption.fold(Set.empty[Permission])(_.permissions) AuthContextImpl(userId, userName, organisationName, requestId, permissions) } From 1011bb7d5d688bf379ea727053ce3e9aa5558518 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Tue, 20 Oct 2020 16:53:54 +0200 Subject: [PATCH 084/237] #1579 WIP: Improve filter capabilities in alert similarity panel --- .../alert/AlertSimilarCaseListCmp.js | 89 ++++++++++++++++++- .../services/common/data/PaginatedQuerySrv.js | 2 +- frontend/app/styles/case-item.css | 5 +- frontend/app/styles/filters.css | 20 +++++ .../alert/similar-case-list.component.html | 62 ++++++++++++- .../components/alert/similarity/filters.html | 38 ++++++++ .../components/alert/similarity/toolbar.html | 39 ++++++++ 7 files changed, 246 insertions(+), 9 deletions(-) create mode 100644 frontend/app/views/components/alert/similarity/filters.html create mode 100644 frontend/app/views/components/alert/similarity/toolbar.html diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 1762426556..0b5dc6fb81 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -8,13 +8,26 @@ self.CaseResolutionStatus = CaseResolutionStatus; + self.similarityFilters = { + fTitle: undefined, + fMatches: undefined + + }; + + self.rateFilters = { + fObservables: undefined, + fIoc: undefined + }; + + self.matches = []; + self.$onInit = function() { this.filtering = new FilteringSrv('case', 'alert.dialog.similar-cases', { version: 'v1', defaults: { showFilters: true, showStats: false, - pageSize: 15, + pageSize: 2, sort: ['-startDate'] }, defaultFilter: [] @@ -40,12 +53,23 @@ skipStream: true, version: 'v1', loadAll: true, - pageSize: 15, + pageSize: self.filtering.context.pageSize, operations: [ {'_name': 'getAlert', 'idOrName': this.alertId}, - {'_name': 'similarCases'} + {'_name': 'similarCases', 'caseFilter': this.filtering.buildQuery()} ], - onUpdate: function() {} + onUpdate: function(data) { + _.each(data, function(item) { + item.fTitle = item.case.title; + item.fMatches = _.keys(item.observableTypes); + item.fObservables = Math.floor((item.similarObservableCount / item.observableCount) * 100); + item.fIocs = Math.floor((item.similarIocCount / item.iocCount) * 100); + }); + + self.matches = _.uniq(_.flatten(_.map(data, function(item){ + return _.keys(item.observableTypes); + }))).sort(); + } }); }; @@ -55,6 +79,63 @@ }); }; + // Frontend filter methods + this.clearLocalFilters = function() { + self.similarityFilters = { + fTitle: undefined, + fMatches: undefined + }; + + self.rateFilters = { + fObservables: undefined, + fIoc: undefined + }; + }; + + this.greaterThan = function(prop){ + return function(item){ + return !self.rateFilters[prop] || item[prop] >= self.rateFilters[prop]; + }; + }; + + // Filtering methods + this.toggleFilters = function () { + this.filtering.toggleFilters(); + }; + + this.search = function () { + self.load(); + self.filtering.storeContext(); + }; + + this.addFilterValue = function (field, value) { + self.filtering.addFilterValue(field, value); + self.search(); + }; + + /// Clear all filters + this.clearFilters = function () { + self.filtering.clearFilters() + .then(self.search); + }; + + // Remove a filter + this.removeFilter = function (index) { + self.filtering.removeFilter(index) + .then(self.search); + }; + + this.filterBy = function(field, value) { + self.filtering.clearFilters() + .then(function(){ + self.addFilterValue(field, value); + }); + }; + + this.filterSimilarities = function(data) { + return data; + }; + }, controllerAs: '$cmp', diff --git a/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js b/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js index e15ec0e52b..29f077b1e7 100644 --- a/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js +++ b/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js @@ -65,7 +65,7 @@ }); if (angular.isFunction(this.onUpdate)) { - this.onUpdate(); + this.onUpdate(this.allValues); } } else { this.update(); diff --git a/frontend/app/styles/case-item.css b/frontend/app/styles/case-item.css index eba8ea9552..bd8a1cc64b 100644 --- a/frontend/app/styles/case-item.css +++ b/frontend/app/styles/case-item.css @@ -41,7 +41,10 @@ div.case-item>div.case-observables-list { width: 450px; } div.case-item>div.case-similarity { - width: 200px; + width: 140px; +} +div.case-item>div.case-similarity-match { + width: 150px; } div.case-item>div.case-similarity-merge { width: 150px; diff --git a/frontend/app/styles/filters.css b/frontend/app/styles/filters.css index e00b6885e4..070a52dcb2 100644 --- a/frontend/app/styles/filters.css +++ b/frontend/app/styles/filters.css @@ -51,3 +51,23 @@ background: #f5f5f5; } */ + + +.similar-case-list .progress { + height: 2px; + opacity: 1; +} + +.similar-case-list .has-feedback .form-control { + padding-right: 25px; +} + +.similar-case-list .case-item.filter-panel { + padding: 5px 0; + margin-bottom: 5px !important; + border: none !important; +} + +.similar-case-list .case-item.filter-panel .form-group { + margin-bottom: 0 !important; +} diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index 431d99ddcf..9a9ee89b99 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -1,4 +1,18 @@
+ +
+ +
+ +
+
+ + +
+
+
@@ -6,11 +20,11 @@
- +
+
+
+ + +
+
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ +
+
+
+
+ Clear Filters +
+
+
+ -
+ +
@@ -76,6 +127,11 @@ N/A
+
+
+
{{match}} ({{count}})
+
+
diff --git a/frontend/app/views/components/alert/similarity/filters.html b/frontend/app/views/components/alert/similarity/filters.html new file mode 100644 index 0000000000..5b47d4771b --- /dev/null +++ b/frontend/app/views/components/alert/similarity/filters.html @@ -0,0 +1,38 @@ +
+
+

Filters

+
+
+
+
+ + + + +
+
+
+ +
+
+
+
+ + Add a filter + + + Clear + + +
+
+
+ +
+
diff --git a/frontend/app/views/components/alert/similarity/toolbar.html b/frontend/app/views/components/alert/similarity/toolbar.html new file mode 100644 index 0000000000..3e1b0db377 --- /dev/null +++ b/frontend/app/views/components/alert/similarity/toolbar.html @@ -0,0 +1,39 @@ +
+
+ +
+
From 58b2b9b1d95b53d5539a35e2669e31ce11c3313c Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 21 Oct 2020 11:24:56 +0200 Subject: [PATCH 085/237] #1579 WIP: Add client side pagination in case similarity in alert dialog --- .../alert/AlertSimilarCaseListCmp.js | 22 +++++- .../alert/similar-case-list.component.html | 67 +++++++++++++------ .../components/alert/similarity/toolbar.html | 8 ++- 3 files changed, 75 insertions(+), 22 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 0b5dc6fb81..12805b6df4 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -11,7 +11,6 @@ self.similarityFilters = { fTitle: undefined, fMatches: undefined - }; self.rateFilters = { @@ -19,8 +18,14 @@ fIoc: undefined }; + self.sortField = '-sCreatedAt'; self.matches = []; + self.pagination = { + pageSize: 10, + currentPage: 1 + }; + self.$onInit = function() { this.filtering = new FilteringSrv('case', 'alert.dialog.similar-cases', { version: 'v1', @@ -53,7 +58,7 @@ skipStream: true, version: 'v1', loadAll: true, - pageSize: self.filtering.context.pageSize, + //pageSize: self.filtering.context.pageSize, operations: [ {'_name': 'getAlert', 'idOrName': this.alertId}, {'_name': 'similarCases', 'caseFilter': this.filtering.buildQuery()} @@ -64,6 +69,7 @@ item.fMatches = _.keys(item.observableTypes); item.fObservables = Math.floor((item.similarObservableCount / item.observableCount) * 100); item.fIocs = Math.floor((item.similarIocCount / item.iocCount) * 100); + item.sCreatedAt = item.case._createdAt; }); self.matches = _.uniq(_.flatten(_.map(data, function(item){ @@ -136,6 +142,18 @@ return data; }; + this.sortByField = function(field) { + var sort = null; + + if(this.sortField.substr(1) !== field) { + sort = '+' + field; + } else { + sort = (this.sortField === '+' + field) ? '-'+field : '+'+field; + } + + this.sortField = sort; + }; + }, controllerAs: '$cmp', diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index 9a9ee89b99..0bad641daf 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -20,23 +20,53 @@
- - - + +
+
    +
    +
    -
    Title
    -
    Date
    -
    Observables
    -
    IOCs
    +
    + + Title + + + + +
    +
    + + Created At + + + + +
    +
    + + Observables + + + + +
    +
    + + IOCs + + + + +
    Matches
    Action
    +
    @@ -72,12 +102,11 @@
    - - -
    + +
    - -
    + +
    @@ -87,7 +116,7 @@
    None - {{tag}} + {{tag}}
    @@ -103,13 +132,13 @@
    -
    +
    - {{item.case.startDate | shortDate}} + {{item.case._createdAt | shortDate}}
    diff --git a/frontend/app/views/components/alert/similarity/toolbar.html b/frontend/app/views/components/alert/similarity/toolbar.html index 3e1b0db377..85713bdb94 100644 --- a/frontend/app/views/components/alert/similarity/toolbar.html +++ b/frontend/app/views/components/alert/similarity/toolbar.html @@ -26,7 +26,13 @@
    - + +
    + + per page +
    From be15f4a23b21b042d1a33963c5189e20f2643592 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 21 Oct 2020 11:37:19 +0200 Subject: [PATCH 086/237] #1579 WIP: Use the filtered similarities to build the pagination component --- .../app/scripts/components/alert/AlertSimilarCaseListCmp.js | 1 + .../views/components/alert/similar-case-list.component.html | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 12805b6df4..44a501fea4 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -20,6 +20,7 @@ self.sortField = '-sCreatedAt'; self.matches = []; + self.filteredCases = []; self.pagination = { pageSize: 10, diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index 0bad641daf..013d8d73b8 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -21,9 +21,9 @@
    -
    +
      @@ -103,7 +103,7 @@
      -
      +
      From 711bfb4bba17beadf9f4afefbc87395563684de3 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 21 Oct 2020 16:30:50 +0200 Subject: [PATCH 087/237] #1565 Change task status when first log is added --- thehive/app/org/thp/thehive/services/LogSrv.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 9ad586caba..7f8f68b25d 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -1,7 +1,7 @@ package org.thp.thehive.services import java.util - +import scala.util.Success import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.scalligraph.EntityIdOrName @@ -21,8 +21,9 @@ import play.api.libs.json.{JsObject, Json} import scala.util.Try @Singleton -class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv)(implicit @Named("with-thehive-schema") db: Database) - extends VertexSrv[Log] { +class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv, taskSrv: TaskSrv, userSrv: UserSrv)(implicit + @Named("with-thehive-schema") db: Database +) extends VertexSrv[Log] { val taskLogSrv = new EdgeSrv[TaskLog, Task, Log] val logAttachmentSrv = new EdgeSrv[LogAttachment, Log, Attachment] @@ -30,6 +31,8 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv)(implic for { createdLog <- createEntity(log) _ <- taskLogSrv.create(TaskLog(), task, createdLog) + user <- userSrv.current.getOrFail("User") // user is used only if task status is waiting but the code is cleaner + _ <- if (task.status == TaskStatus.Waiting) taskSrv.updateStatus(task, user, TaskStatus.InProgress) else Success(()) _ <- auditSrv.log.create(createdLog, task, RichLog(createdLog, Nil).toJson) } yield createdLog From b3c9cd656df1f604aaa68a219e035a02b78e84f9 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 21 Oct 2020 16:56:26 +0200 Subject: [PATCH 088/237] #1579 WIP: Add a multi selection filter for match criteria --- .../alert/AlertSimilarCaseListCmp.js | 25 ++++++++++++++----- .../alert/similar-case-list.component.html | 8 +++--- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 44a501fea4..4e412fade0 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -9,8 +9,11 @@ self.CaseResolutionStatus = CaseResolutionStatus; self.similarityFilters = { - fTitle: undefined, - fMatches: undefined + fTitle: undefined + }; + + self.matchFilters = { + fMatches: [] }; self.rateFilters = { @@ -89,8 +92,11 @@ // Frontend filter methods this.clearLocalFilters = function() { self.similarityFilters = { - fTitle: undefined, - fMatches: undefined + fTitle: undefined + }; + + self.matchFilters = { + fMatches: [] }; self.rateFilters = { @@ -101,8 +107,15 @@ this.greaterThan = function(prop){ return function(item){ - return !self.rateFilters[prop] || item[prop] >= self.rateFilters[prop]; - }; + return !self.rateFilters[prop] || item[prop] >= self.rateFilters[prop]; + }; + }; + + this.matchFilter = function() { + return function(item){ + return !self.matchFilters.fMatches || self.matchFilters.fMatches.length === 0 || + _.intersection(self.matchFilters.fMatches, item.fMatches).length > 0; + }; }; // Filtering methods diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index 013d8d73b8..e870abc60d 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -89,10 +89,8 @@
      - +
      @@ -103,7 +101,7 @@
      -
      +
      From ce44fefa765d9602394cf03e72c649a45c52475c Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 22 Oct 2020 16:42:10 +0200 Subject: [PATCH 089/237] #1579 WIP: Fix the IOC % filter --- .../components/alert/AlertSimilarCaseListCmp.js | 7 ++++--- .../components/alert/similar-case-list.component.html | 10 +++++----- .../app/views/components/alert/similarity/toolbar.html | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 4e412fade0..850bad9b42 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -18,7 +18,7 @@ self.rateFilters = { fObservables: undefined, - fIoc: undefined + fIocs: undefined }; self.sortField = '-sCreatedAt'; @@ -72,7 +72,8 @@ item.fTitle = item.case.title; item.fMatches = _.keys(item.observableTypes); item.fObservables = Math.floor((item.similarObservableCount / item.observableCount) * 100); - item.fIocs = Math.floor((item.similarIocCount / item.iocCount) * 100); + item.fIocs = Math.floor((item.similarIocCount / item.iocCount) * 100) || 0; + item.sCreatedAt = item.case._createdAt; }); @@ -101,7 +102,7 @@ self.rateFilters = { fObservables: undefined, - fIoc: undefined + fIocs: undefined }; }; diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index e870abc60d..63585991c8 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -70,20 +70,20 @@
      - +
      - +
      - +
      @@ -141,13 +141,13 @@
      - {{(item.similarObservableCount / item.observableCount) | percentage:0}} ({{item.similarObservableCount}} / {{item.observableCount}}) + {{item.fObservables | number:0}} % ({{item.similarObservableCount}} / {{item.observableCount}})
      - {{(item.similarIocCount / item.iocCount) | percentage:0}} ({{item.similarIocCount}} / {{item.iocCount}}) + {{item.fIocs | number:0}} % ({{item.similarIocCount}} / {{item.iocCount}})
      diff --git a/frontend/app/views/components/alert/similarity/toolbar.html b/frontend/app/views/components/alert/similarity/toolbar.html index 85713bdb94..6f10e21bbb 100644 --- a/frontend/app/views/components/alert/similarity/toolbar.html +++ b/frontend/app/views/components/alert/similarity/toolbar.html @@ -30,7 +30,7 @@
      + ng-options="v for v in [10,15,30,50]"> per page
      From 86a929f1955eefd90a9309cf474a074d2bd41e43 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 22 Oct 2020 14:16:57 +0200 Subject: [PATCH 090/237] #1553 Imported customFields when merging alerts --- .../app/org/thp/thehive/services/AlertSrv.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index dc5edcdaa7..77ee744d32 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -277,6 +277,7 @@ class AlertSrv @Inject() ( description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" c <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") _ <- importObservables(alert, `case`) + _ <- importCustomFields(alert, `case`) _ <- alertCaseSrv.create(AlertCase(), alert, `case`) _ <- markAsRead(alert._id) _ <- auditSrv.alertToCase.merge(alert, c) @@ -314,6 +315,22 @@ class AlertSrv @Inject() ( } .map(_ => ()) + def importCustomFields(alert: Alert with Entity, `case`: Case with Entity)(implicit + graph: Graph, + authContext: AuthContext + ): Try[Unit] = + get(alert) + .richCustomFields + .toIterator + .toTry { richCustomField => + caseSrv + .setOrCreateCustomField(`case`, + richCustomField.customField._id, + richCustomField.value, + richCustomField.customFieldValue.order) + } + .map(_ => ()) + def remove(alert: Alert with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { organisation <- organisationSrv.getOrFail(authContext.organisation) From f90c4efbaf28cfd2e43cd3159ba675f563388a27 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 22 Oct 2020 15:15:43 +0200 Subject: [PATCH 091/237] #1501 Fix admin rights --- thehive/app/org/thp/thehive/services/UserSrv.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/UserSrv.scala b/thehive/app/org/thp/thehive/services/UserSrv.scala index fe0ed33fdf..0fb1f1e743 100644 --- a/thehive/app/org/thp/thehive/services/UserSrv.scala +++ b/thehive/app/org/thp/thehive/services/UserSrv.scala @@ -233,15 +233,17 @@ object UserOps { .project( _.byValue(_.login) .byValue(_.name) - .by(_.role.filter(_.organisation.get(organisationName)).profile.fold) + .by(_.profile(organisationName).fold) .by(_.organisations.get(organisationName).value(_.name).limit(1).fold) + .by(_.profile(EntityName(Organisation.administration.name)).fold) ) .domainMap { - case (userId, userName, profile, org) => + case (userId, userName, profile, org, adminProfile) => val scope = if (org.contains(Organisation.administration.name)) "admin" else "organisation" - val permissions = Permissions.forScope(scope) & profile.headOption.fold(Set.empty[Permission])(_.permissions) + val permissions = + Permissions.forScope(scope) & profile.headOption.orElse(adminProfile.headOption).fold(Set.empty[Permission])(_.permissions) AuthContextImpl(userId, userName, organisationName, requestId, permissions) } From 0e049c6fefeb0b8df79504b249b6bdfef9288621 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 22 Oct 2020 16:09:16 +0200 Subject: [PATCH 092/237] #1590 Fix tag property for filters --- .../thehive/controllers/v0/AlertCtrl.scala | 16 +++++- .../thp/thehive/controllers/v0/CaseCtrl.scala | 14 +++++ .../controllers/v0/CaseTemplateCtrl.scala | 18 +++++- .../controllers/v0/ObservableCtrl.scala | 16 +++++- .../thehive/controllers/v1/AlertCtrl.scala | 2 +- .../thehive/controllers/v1/Properties.scala | 56 +++++++++++++++++++ 6 files changed, 117 insertions(+), 5 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 68fa42aaf1..157180e68a 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -62,7 +62,7 @@ class AlertCtrl @Inject() ( .get(request.organisation) .orFail(AuthorizationError("Operation not permitted")) richObservables <- observables.toTry(createObservable).map(_.flatten) - richAlert <- alertSrv.create(request.body("alert").toAlert, organisation, inputAlert.tags, customFields, caseTemplate) + richAlert <- alertSrv.create(inputAlert.toAlert, organisation, inputAlert.tags, customFields, caseTemplate) _ <- auditSrv.mergeAudits(richObservables.toTry(o => alertSrv.addObservable(richAlert.alert, o)))(_ => Success(())) createdObservables = alertSrv.get(richAlert.alert).observables.richObservable.toSeq } yield Results.Created((richAlert -> createdObservables).toJson) @@ -369,6 +369,20 @@ class PublicAlert @Inject() ( .property("lastSyncDate", UMapping.date.optional)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => alertSrv .get(vertex)(graph) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index 53bbd8b1c8..ec72368785 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -240,6 +240,20 @@ class PublicCase @Inject() ( .property("endDate", UMapping.date.optional)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => caseSrv .get(vertex)(graph) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala index b8f36f5df3..6973c978b2 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala @@ -6,11 +6,11 @@ import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.scalligraph.{AttributeCheckingError, BadRequestError, EntityIdOrName, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputCaseTemplate, InputTask} -import org.thp.thehive.models.{CaseTemplate, Permissions, RichCaseTemplate} +import org.thp.thehive.models.{CaseTemplate, Permissions, RichCaseTemplate, Tag} import org.thp.thehive.services.CaseTemplateOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TagOps._ @@ -115,6 +115,20 @@ class PublicCaseTemplate @Inject() ( .property("severity", UMapping.int.optional)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => caseTemplateSrv .get(vertex)(graph) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index e52c3458b2..8278610233 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -6,7 +6,7 @@ import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.InputObservable import org.thp.thehive.models._ @@ -210,6 +210,20 @@ class PublicObservable @Inject() ( .property("sighted", UMapping.boolean)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => observableSrv .get(vertex)(graph) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 333bf115d4..d4ca260509 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -85,7 +85,7 @@ class AlertCtrl @Inject() ( for { organisation <- userSrv.current.organisations(Permissions.manageAlert).getOrFail("Organisation") customFields = inputAlert.customFieldValue.map(cf => cf.name -> cf.value).toMap - richAlert <- alertSrv.create(request.body("alert").toAlert, organisation, inputAlert.tags, customFields, caseTemplate) + richAlert <- alertSrv.create(inputAlert.toAlert, organisation, inputAlert.tags, customFields, caseTemplate) } yield Results.Created(richAlert.toJson) } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 608c62cd71..5151fb2b4e 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -49,6 +49,20 @@ class Properties @Inject() ( .property("lastSyncDate", UMapping.date.optional)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => alertSrv .get(vertex)(graph) @@ -101,6 +115,20 @@ class Properties @Inject() ( .property("number", UMapping.int)(_.field.readonly) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => caseSrv .get(vertex)(graph) @@ -206,6 +234,20 @@ class Properties @Inject() ( .property("severity", UMapping.int.optional)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => caseTemplateSrv .get(vertex)(graph) @@ -295,6 +337,20 @@ class Properties @Inject() ( .property("sighted", UMapping.boolean)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) + .filter((_, cases) => + cases + .tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) .custom { (_, value, vertex, _, graph, authContext) => observableSrv .getOrFail(vertex)(graph) From 57e53796b19d9c7a47daab2d544f4c8b4f737e12 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 22 Oct 2020 17:45:19 +0200 Subject: [PATCH 093/237] #1450 Make initialisation timeout configurable (default 1 hour) --- .../thehive/models/SchemaUpdaterActor.scala | 31 +++++++++++-------- thehive/conf/reference.conf | 7 +++-- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/thehive/app/org/thp/thehive/models/SchemaUpdaterActor.scala b/thehive/app/org/thp/thehive/models/SchemaUpdaterActor.scala index 4d8d180daa..1823d030ae 100644 --- a/thehive/app/org/thp/thehive/models/SchemaUpdaterActor.scala +++ b/thehive/app/org/thp/thehive/models/SchemaUpdaterActor.scala @@ -9,20 +9,23 @@ import org.thp.scalligraph.janus.JanusDatabase import org.thp.scalligraph.models.Database import org.thp.thehive.ClusterSetup import org.thp.thehive.services.LocalUserSrv -import play.api.Logger +import play.api.{Configuration, Logger} -import scala.concurrent.duration.DurationInt +import scala.concurrent.duration.{DurationInt, FiniteDuration} import scala.concurrent.{Await, ExecutionContext} import scala.util.{Failure, Try} @Singleton class DatabaseProvider @Inject() ( + configuration: Configuration, database: Database, theHiveSchema: TheHiveSchemaDefinition, actorSystem: ActorSystem, - clusterSetup: ClusterSetup + clusterSetup: ClusterSetup // this dependency is here to ensure that cluster setup is finished ) extends Provider[Database] { import SchemaUpdaterActor._ + + lazy val dbInitialisationTimeout: FiniteDuration = configuration.get[FiniteDuration]("db.initialisationTimeout") lazy val schemaUpdaterActor: ActorRef = { val singletonManager = actorSystem.actorOf( @@ -43,16 +46,17 @@ class DatabaseProvider @Inject() ( ) } - def databaseInstance: String = database match { - case jdb: JanusDatabase => jdb.instanceId - case _ => "" - } + def databaseInstance: String = + database match { + case jdb: JanusDatabase => jdb.instanceId + case _ => "" + } override def get(): Database = { - implicit val timeout: Timeout = Timeout(5.minutes) + implicit val timeout: Timeout = Timeout(dbInitialisationTimeout) Await.result(schemaUpdaterActor ? RequestDBStatus(databaseInstance), timeout.duration) match { case DBStatus(status) => - status.get + status.get // if the status is a failure, throw an exception. database.asInstanceOf[Database] } } @@ -90,10 +94,11 @@ class SchemaUpdaterActor @Inject() (theHiveSchema: TheHiveSchemaDefinition, data } def hasUnknownConnections(instanceIds: Set[String]): Boolean = (originalConnectionIds -- instanceIds).nonEmpty - def dropUnknownConnections(instanceIds: Set[String]): Unit = database match { - case jdb: JanusDatabase => jdb.dropConnections((originalConnectionIds -- instanceIds).toSeq) - case _ => - } + def dropUnknownConnections(instanceIds: Set[String]): Unit = + database match { + case jdb: JanusDatabase => jdb.dropConnections((originalConnectionIds -- instanceIds).toSeq) + case _ => + } override def receive: Receive = { case RequestDBStatus(instanceId) => diff --git a/thehive/conf/reference.conf b/thehive/conf/reference.conf index 75c2c1a453..02b10595b0 100644 --- a/thehive/conf/reference.conf +++ b/thehive/conf/reference.conf @@ -1,5 +1,8 @@ -db.provider: janusgraph -db.janusgraph.index.search.directory: /tmp/thehive.idx +db { + provider: janusgraph + janusgraph.index.search.directory: /tmp/thehive.idx + initialisationTimeout: 1 hour +} storage { provider: localfs From 6d49618d2f5499a07d533ffccc8822b5acf5f8ea Mon Sep 17 00:00:00 2001 From: To-om Date: Sun, 25 Oct 2020 08:57:49 +0100 Subject: [PATCH 094/237] #1592 Fix user visibility for admin --- .../controllers/admin/organisation/OrgDetailsCtrl.js | 5 ----- frontend/app/scripts/services/api/OrganisationSrv.js | 6 +----- thehive/app/org/thp/thehive/services/OrganisationSrv.scala | 3 ++- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/frontend/app/scripts/controllers/admin/organisation/OrgDetailsCtrl.js b/frontend/app/scripts/controllers/admin/organisation/OrgDetailsCtrl.js index ebf7e33fbd..b122522177 100644 --- a/frontend/app/scripts/controllers/admin/organisation/OrgDetailsCtrl.js +++ b/frontend/app/scripts/controllers/admin/organisation/OrgDetailsCtrl.js @@ -55,11 +55,6 @@ '_name': 'users' } ], - config: { - headers: { - 'X-Organisation': self.org.name - } - }, onFailure: function(err) { if(err && err.status === 400) { self.filtering.resetContext(); diff --git a/frontend/app/scripts/services/api/OrganisationSrv.js b/frontend/app/scripts/services/api/OrganisationSrv.js index b225391021..9d9869b806 100644 --- a/frontend/app/scripts/services/api/OrganisationSrv.js +++ b/frontend/app/scripts/services/api/OrganisationSrv.js @@ -54,11 +54,7 @@ { '_name': 'users' } - ], { - headers: { - 'X-Organisation': orgId - } - }).then(function(response) { + ]).then(function(response) { return $q.resolve(response.data); }); }; diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index 8038415de6..69af5f84de 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -159,7 +159,8 @@ object OrganisationOps { def dashboards: Traversal.V[Dashboard] = traversal.out[OrganisationDashboard].v[Dashboard] def visible(implicit authContext: AuthContext): Traversal.V[Organisation] = - if (authContext.isPermitted(Permissions.manageOrganisation)) traversal + if (authContext.isPermitted(Permissions.manageOrganisation)) + traversal else traversal.filter(_.visibleOrganisationsTo.users.current) From e8033b82d15bb6bf8c096f8f511f46ca5c1a42d0 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 26 Oct 2020 16:08:15 +0100 Subject: [PATCH 095/237] #1579 Update the date filter field to support dynamic dates --- .../controllers/alert/AlertEventCtrl.js | 57 ------------------- .../directives/dashboard/filter-editor.js | 33 ++++++++++- frontend/app/scripts/filters/filter-value.js | 33 +++++++++-- .../services/common/QueryBuilderSrv.js | 26 ++++++--- .../app/scripts/services/common/UtilsSrv.js | 32 +++++++++++ .../alert/similar-case-list.component.html | 4 +- .../directives/dashboard/filter-editor.html | 21 ++++++- 7 files changed, 131 insertions(+), 75 deletions(-) diff --git a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js index 2a03738846..0db05c183c 100644 --- a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js @@ -14,18 +14,6 @@ self.loading = true; - self.pagination = { - pageSize: 10, - currentPage: 1, - filter: '', - data: [] - }; - - self.similarityFilters = {}; - self.similaritySorts = ['-startDate', '-similarArtifactCount', '-similarIocCount', '-iocCount']; - self.currentSimilarFilter = ''; - self.similarCasesStats = []; - self.customFieldsCache = CustomFieldsSrv; self.counts = { @@ -51,7 +39,6 @@ AlertingSrv.get(eventId).then(function(data) { self.event = data; self.loading = false; - self.initSimilarCasesFilter(self.event.similarCases); self.dataTypes = _.countBy(self.event.artifacts, function(attr) { return attr.dataType; @@ -196,50 +183,6 @@ $uibModalInstance.dismiss(); }; - self.initSimilarCasesFilter = function(data) { - var stats = { - 'Open': 0 - }; - - // Init the stats object - _.each(_.without(_.keys(CaseResolutionStatus), 'Duplicated'), function(key) { - stats[key] = 0; - }); - - _.each(data, function(item) { - if(item.status === 'Open') { - stats[item.status] = stats[item.status] + 1; - } else { - stats[item.resolutionStatus] = stats[item.resolutionStatus] + 1; - } - }); - - var result = []; - _.each(_.keys(stats), function(key) { - result.push({ - key: key, - value: stats[key] - }); - }); - - self.similarCasesStats = result; - }; - - self.filterSimilarCases = function(filter) { - self.currentSimilarFilter = filter; - if(filter === '') { - self.similarityFilters = {}; - } else if(filter === 'Open') { - self.similarityFilters = { - status: filter - }; - } else { - self.similarityFilters = { - resolutionStatus: filter - }; - } - }; - self.copyId = function(id) { clipboard.copyText(id); }; diff --git a/frontend/app/scripts/directives/dashboard/filter-editor.js b/frontend/app/scripts/directives/dashboard/filter-editor.js index 033923ba56..def05a2327 100644 --- a/frontend/app/scripts/directives/dashboard/filter-editor.js +++ b/frontend/app/scripts/directives/dashboard/filter-editor.js @@ -1,6 +1,6 @@ (function() { 'use strict'; - angular.module('theHiveDirectives').directive('filterEditor', function($q, AuthenticationSrv, UserSrv) { + angular.module('theHiveDirectives').directive('filterEditor', function($q, AuthenticationSrv, UserSrv, UtilsSrv) { return { restrict: 'E', scope: { @@ -10,6 +10,37 @@ }, templateUrl: 'views/directives/dashboard/filter-editor.html', link: function(scope) { + scope.dateOperator = { + custom: 'Custom', + today: 'Today', + last7days: 'Last 7 days', + last30days: 'Last 30 days', + last3months: 'Last 3 months', + last6months: 'Last 6 months', + lastyear: 'Last year' + }; + + scope.setDateFilterOperator = function(filter, operator) { + operator = operator || 'custom'; + + var dateRange = UtilsSrv.getDateRange(operator); + + if(operator === 'custom') { + filter.value = { + operator: operator, + from: dateRange.from, + to: dateRange.to + }; + } else { + filter.value = { + operator: operator, + from: null, + to: null + }; + } + + }; + scope.editorFor = function(filter) { if (filter.type === null) { return; diff --git a/frontend/app/scripts/filters/filter-value.js b/frontend/app/scripts/filters/filter-value.js index 44730e3da8..fa7aeee565 100644 --- a/frontend/app/scripts/filters/filter-value.js +++ b/frontend/app/scripts/filters/filter-value.js @@ -1,7 +1,7 @@ (function() { 'use strict'; - angular.module('theHiveFilters').filter('filterValue', function() { + angular.module('theHiveFilters').filter('filterValue', function(UtilsSrv) { return function(value) { if (angular.isArray(value)) { return _.map(value, function(item) { @@ -9,12 +9,35 @@ }).join(', '); } else if(angular.isObject(value) && value.from !== undefined && value.to !== undefined) { var result = []; - if(value.from !== null) { - result.push('From: ' + moment(value.from).hour(0).minutes(0).seconds(0).format('MM/DD/YY HH:mm')); + + result.push({ + custom: 'Custom', + today: 'Today', + last7days: 'Last 7 days', + last30days: 'Last 30 days', + last3months: 'Last 3 months', + last6months: 'Last 6 months', + lastyear: 'Last year' + }[value.operator] || 'Custom'); + + var start, end; + + if(value.operator && value.operator !== 'custom') { + var dateRange = UtilsSrv.getDateRange(value.operator); + + start = dateRange.from; + end = dateRange.to; + } else { + start = value.from; + end = value.to; + } + + if(start !== null) { + result.push('From: ' + moment(start).hour(0).minutes(0).seconds(0).format('MM/DD/YY HH:mm')); } - if(value.to !== null) { - result.push('To: ' + moment(value.to).hour(23).minutes(59).seconds(59).format('MM/DD/YY HH:mm')); + if(end !== null) { + result.push('To: ' + moment(end).hour(23).minutes(59).seconds(59).format('MM/DD/YY HH:mm')); } return result.join(', '); diff --git a/frontend/app/scripts/services/common/QueryBuilderSrv.js b/frontend/app/scripts/services/common/QueryBuilderSrv.js index af66d42a7a..9574a44460 100644 --- a/frontend/app/scripts/services/common/QueryBuilderSrv.js +++ b/frontend/app/scripts/services/common/QueryBuilderSrv.js @@ -1,6 +1,6 @@ (function() { 'use strict'; - angular.module('theHiveServices').service('QueryBuilderSrv', function() { + angular.module('theHiveServices').service('QueryBuilderSrv', function(UtilsSrv) { var self = this; this._buildQueryFromDefaultFilter = function(fieldDef, filter) { @@ -116,19 +116,27 @@ this._buildQueryFromDateFilter = function(fieldDef, filter) { var value = filter.value, + operator = filter.value.operator || 'custom', start, end; - if(value.from && value.from !== null) { - start = _.isString(value.from) ? (new Date(value.from)).getTime() : value.from.getTime(); - } else { - start = null; - } + if(operator === 'custom') { + if(value.from && value.from !== null) { + start = _.isString(value.from) ? (new Date(value.from)).getTime() : value.from.getTime(); + } else { + start = null; + } - if(value.to && value.to !== null) { - end = _.isString(value.to) ? (new Date(value.to)).setHours(23, 59, 59, 999) : value.to.getTime(); + if(value.to && value.to !== null) { + end = _.isString(value.to) ? (new Date(value.to)).setHours(23, 59, 59, 999) : value.to.getTime(); + } else { + end = null; + } } else { - end = null; + var dateRange = UtilsSrv.getDateRange(operator); + + start = dateRange.from.getTime(); + end = dateRange.to.getTime(); } if (start !== null && end !== null) { diff --git a/frontend/app/scripts/services/common/UtilsSrv.js b/frontend/app/scripts/services/common/UtilsSrv.js index 5a4fdb4288..35076d9807 100644 --- a/frontend/app/scripts/services/common/UtilsSrv.js +++ b/frontend/app/scripts/services/common/UtilsSrv.js @@ -118,6 +118,38 @@ }); return parsedQuery ? parsedQuery.substr(paramName.length + 1) : undefined; } + }, + + getDateRange: function(operator) { + var from, + to = moment(); + + switch(operator) { + case 'last7days': + from = moment().subtract(7, 'days'); + break; + case 'last30days': + from = moment().subtract(30, 'days'); + break; + case 'last3months': + from = moment().subtract(3, 'months'); + break; + case 'last6months': + from = moment().subtract(6, 'months'); + break; + case 'lastyear': + from = moment().subtract(1, 'years'); + break; + case 'today': + default: + from = moment(); + break; + } + + return { + from: from.hour(0).minutes(0).seconds(0).toDate(), + to: to.hour(23).minutes(59).seconds(59).toDate() + }; } }; diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index 63585991c8..86a17ffd0e 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -136,7 +136,9 @@
      - {{item.case._createdAt | shortDate}} + + {{item.case._createdAt | shortDate}} +
      diff --git a/frontend/app/views/directives/dashboard/filter-editor.html b/frontend/app/views/directives/dashboard/filter-editor.html index 7ffe88890f..cb2b6dcb61 100644 --- a/frontend/app/views/directives/dashboard/filter-editor.html +++ b/frontend/app/views/directives/dashboard/filter-editor.html @@ -96,10 +96,27 @@
      -
      +
      + + +
      +
      -
      +
      From 98b044f1f71fb97e6fc5e17aeea006af47df4946 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 22 Oct 2020 16:20:11 +0200 Subject: [PATCH 096/237] #1557 Description now list all alerts merged --- .../thp/thehive/controllers/v0/AlertCtrl.scala | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 157180e68a..daed87794e 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -191,13 +191,17 @@ class AlertCtrl @Inject() ( val alertIds: Seq[String] = request.body("alertIds") val caseId: String = request.body("caseId") for { - case0 <- caseSrv.get(EntityIdOrName(caseId)).can(Permissions.manageCase).getOrFail("Case") _ <- alertIds.toTry { alertId => - alertSrv - .get(EntityIdOrName(alertId)) - .can(Permissions.manageAlert) - .getOrFail("Alert") - .flatMap(alertSrv.mergeInCase(_, case0)) + caseSrv.get(EntityIdOrName(caseId)) + .can(Permissions.manageCase) + .getOrFail("Case") + .flatMap(`case` => + alertSrv + .get(EntityIdOrName(alertId)) + .can(Permissions.manageAlert) + .getOrFail("Alert") + .flatMap(alertSrv.mergeInCase(_, `case`)) + ) } richCase <- caseSrv.get(EntityIdOrName(caseId)).richCase.getOrFail("Case") } yield Results.Ok(richCase.toJson) From d25867c052c9616fa098476a150f6ddc8c37c8a3 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Mon, 26 Oct 2020 14:31:30 +0100 Subject: [PATCH 097/237] #1557 One CaseUpdate per alert merge --- .../thehive/controllers/v0/Conversion.scala | 1 - .../org/thp/thehive/services/AlertSrv.scala | 30 ++++++++++++------- .../org/thp/thehive/services/AuditSrv.scala | 4 +-- .../notification/notifiers/Webhook.scala | 9 +++++- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index 145981b6cc..f972afd972 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -28,7 +28,6 @@ object Conversion { def fromObjectType(objectType: String): String = objectType match { - // case "Case" =>"case" case "Task" => "case_task" case "Log" => "case_task_log" case "Observable" => "case_artifact" diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 77ee744d32..ddcf2d02e7 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -271,17 +271,25 @@ class AlertSrv @Inject() ( updatedCase <- mergeInCase(alert, case0) } yield updatedCase - def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = - for { - _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) - description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" - c <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") - _ <- importObservables(alert, `case`) - _ <- importCustomFields(alert, `case`) - _ <- alertCaseSrv.create(AlertCase(), alert, `case`) - _ <- markAsRead(alert._id) - _ <- auditSrv.alertToCase.merge(alert, c) - } yield c + def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = { + auditSrv.mergeAudits { + // No audit for markAsRead and observables + markAsRead(alert._id) + importObservables(alert, `case`) + importCustomFields(alert, `case`) + + // Audits for customFields, description and tags + val customFields = get(alert).richCustomFields.toSeq.map(_.toOutput.toJson) + val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" + val tags = get(alert).tags.toSeq.map(_.toString) + + caseSrv.get(`case`).update(_.description, description).getOrFail("Case") + caseSrv.addTags(`case`, tags.toSet) + Success(Json.obj("customFields" -> customFields, "description" -> description, "tags" -> tags)) + } (audits => auditSrv.alertToCase.merge(alert, `case`, Some(audits))) + + caseSrv.get(`case`).getOrFail("Case") + } def importObservables(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index f5fe61c8ff..077b177834 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -173,8 +173,8 @@ class AuditSrv @Inject() ( def delete(entity: E with Entity, context: Option[C with Entity])(implicit graph: Graph, authContext: AuthContext): Try[Unit] = auditSrv.create(Audit(Audit.delete, entity, None), context, None) - def merge(entity: E with Entity, destination: C with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - auditSrv.create(Audit(Audit.merge, entity), Some(destination), None) + def merge(entity: E with Entity, destination: C with Entity, details: Option[JsObject] = None)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + auditSrv.create(Audit(Audit.merge, destination, details.map(_.toString())), Some(destination), None) } class SelfContextObjectAudit[E <: Product] { diff --git a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala index d7b65a3448..757a9e91a3 100644 --- a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala +++ b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala @@ -13,6 +13,7 @@ import org.thp.scalligraph.traversal.{Converter, IdentityConverter, Traversal} import org.thp.scalligraph.{BadConfigurationError, EntityIdOrName} import org.thp.thehive.controllers.v0.AuditRenderer import org.thp.thehive.controllers.v0.Conversion.fromObjectType +import org.thp.thehive.models.Audit._ import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.AuditOps._ @@ -197,7 +198,7 @@ class Webhook( case (audit, obj) => val objectType = audit.objectType.getOrElse(audit.context._label) Json.obj( - "operation" -> audit.action, + "operation" -> v0Action(audit.action), "details" -> audit.details.fold[JsValue](JsObject.empty)(fixCustomFieldDetails(objectType, _)), "objectType" -> fromObjectType(objectType), "objectId" -> audit.objectId, @@ -227,6 +228,12 @@ class Webhook( case _ => Failure(BadConfigurationError(s"Message version $version in webhook is not supported")) } + def v0Action(action: String): String = + action match { + case Audit.merge => Audit.update + case action => action + } + override def execute( audit: Audit with Entity, context: Option[Entity], From 958b9d03bf2f0156603eac1968f87d5f9a12c207 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 27 Oct 2020 09:18:49 +0100 Subject: [PATCH 098/237] #1557 Correct format for detail.customFields in webhook --- .../org/thp/thehive/services/AlertSrv.scala | 21 ++++++++++--------- .../org/thp/thehive/services/AuditSrv.scala | 2 +- .../notification/notifiers/Webhook.scala | 11 ++++++++++ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index ddcf2d02e7..375ffc7960 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -273,20 +273,21 @@ class AlertSrv @Inject() ( def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = { auditSrv.mergeAudits { - // No audit for markAsRead and observables - markAsRead(alert._id) - importObservables(alert, `case`) - importCustomFields(alert, `case`) - - // Audits for customFields, description and tags val customFields = get(alert).richCustomFields.toSeq.map(_.toOutput.toJson) val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" val tags = get(alert).tags.toSeq.map(_.toString) - caseSrv.get(`case`).update(_.description, description).getOrFail("Case") - caseSrv.addTags(`case`, tags.toSet) - Success(Json.obj("customFields" -> customFields, "description" -> description, "tags" -> tags)) - } (audits => auditSrv.alertToCase.merge(alert, `case`, Some(audits))) + for { + _ <- markAsRead(alert._id) + _ <- importObservables(alert, `case`) + _ <- importCustomFields(alert, `case`) + _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") + _ <- caseSrv.addTags(`case`, tags.toSet) + // No audit for markAsRead and observables + // Audits for customFields, description and tags + details <- Success(Json.obj("customFields" -> customFields, "description" -> description, "tags" -> tags)) + } yield details + } (details => auditSrv.alertToCase.merge(alert, `case`, Some(details))) caseSrv.get(`case`).getOrFail("Case") } diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index 077b177834..449335dc7a 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -174,7 +174,7 @@ class AuditSrv @Inject() ( auditSrv.create(Audit(Audit.delete, entity, None), context, None) def merge(entity: E with Entity, destination: C with Entity, details: Option[JsObject] = None)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - auditSrv.create(Audit(Audit.merge, destination, details.map(_.toString())), Some(destination), None) + auditSrv.create(Audit(Audit.merge, destination, details.map(_.toString())), Some(destination), Some(destination)) } class SelfContextObjectAudit[E <: Product] { diff --git a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala index 757a9e91a3..de4e6d4c6f 100644 --- a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala +++ b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala @@ -184,6 +184,17 @@ class Webhook( customFieldSrv .getOrFail(EntityIdOrName(fieldName)) .fold(_ => keyValue, cf => "customFields" -> Json.obj(fieldName -> Json.obj(cf.`type`.toString -> value))) + case ("customFields", JsArray(cfs)) => + "customFields" -> cfs + .flatMap { cf => + for { + name <- (cf \ "name").asOpt[String] + tpe <- (cf \ "type").asOpt[String] + value = (cf \ "value").asOpt[JsValue] + order = (cf \ "order").asOpt[Int] + } yield Json.obj(name -> Json.obj(tpe -> value, "order" -> order)) + } + .foldLeft(JsObject.empty)(_ ++ _) case keyValue => keyValue }) } From 8fc7b19f0ec01c6a776a4f94008785c3aa128ad5 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 27 Oct 2020 11:09:23 +0100 Subject: [PATCH 099/237] #1557 PR review changes --- .../app/org/thp/thehive/services/AlertSrv.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 375ffc7960..ea3e1b41c5 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -273,23 +273,25 @@ class AlertSrv @Inject() ( def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = { auditSrv.mergeAudits { - val customFields = get(alert).richCustomFields.toSeq.map(_.toOutput.toJson) val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" - val tags = get(alert).tags.toSeq.map(_.toString) for { _ <- markAsRead(alert._id) _ <- importObservables(alert, `case`) _ <- importCustomFields(alert, `case`) _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") - _ <- caseSrv.addTags(`case`, tags.toSet) + _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) // No audit for markAsRead and observables // Audits for customFields, description and tags - details <- Success(Json.obj("customFields" -> customFields, "description" -> description, "tags" -> tags)) + newDescription <- Try(caseSrv.get(`case`).richCase.head.description) + details <- Success(Json.obj( + "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), + "description" -> newDescription, + "tags" -> caseSrv.get(`case`).tags.toSeq.map(_.toString)) + ) } yield details } (details => auditSrv.alertToCase.merge(alert, `case`, Some(details))) - - caseSrv.get(`case`).getOrFail("Case") + .flatMap(_ => caseSrv.get(`case`).getOrFail("Case")) } def importObservables(alert: Alert with Entity, `case`: Case with Entity)(implicit From 4a1999c34fa64918eff11f3e79ac285c052b8e3f Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 27 Oct 2020 13:13:05 +0100 Subject: [PATCH 100/237] #1557 PR review changes 2 --- thehive/app/org/thp/thehive/services/AlertSrv.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index ea3e1b41c5..0ac7644654 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -283,15 +283,15 @@ class AlertSrv @Inject() ( _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) // No audit for markAsRead and observables // Audits for customFields, description and tags - newDescription <- Try(caseSrv.get(`case`).richCase.head.description) + c <- caseSrv.getOrFail(`case`._id) details <- Success(Json.obj( "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), - "description" -> newDescription, + "description" -> c.description, "tags" -> caseSrv.get(`case`).tags.toSeq.map(_.toString)) ) } yield details } (details => auditSrv.alertToCase.merge(alert, `case`, Some(details))) - .flatMap(_ => caseSrv.get(`case`).getOrFail("Case")) + .flatMap(_ => caseSrv.getOrFail(`case`._id)) } def importObservables(alert: Alert with Entity, `case`: Case with Entity)(implicit From 5a6ee7af8b7a7e56143eb406f5e25630bf7125bc Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 27 Oct 2020 13:43:17 +0100 Subject: [PATCH 101/237] #1557 Use the case returned by mergeInCase to prevent case retrieving for each alert --- .../thehive/controllers/v0/AlertCtrl.scala | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index daed87794e..37662eaaa4 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -190,21 +190,26 @@ class AlertCtrl @Inject() ( .authTransaction(db) { implicit request => implicit graph => val alertIds: Seq[String] = request.body("alertIds") val caseId: String = request.body("caseId") - for { - _ <- alertIds.toTry { alertId => - caseSrv.get(EntityIdOrName(caseId)) - .can(Permissions.manageCase) - .getOrFail("Case") - .flatMap(`case` => - alertSrv - .get(EntityIdOrName(alertId)) - .can(Permissions.manageAlert) - .getOrFail("Alert") - .flatMap(alertSrv.mergeInCase(_, `case`)) - ) + + val destinationCase = caseSrv + .get(EntityIdOrName(caseId)) + .can(Permissions.manageCase) + .getOrFail("Case") + + alertIds + .foldLeft(destinationCase) { (caseTry, alertId) => + for { + case0 <- caseTry + alert <- + alertSrv + .get(EntityIdOrName(alertId)) + .can(Permissions.manageAlert) + .getOrFail("Alert") + updatedCase <- alertSrv.mergeInCase(alert, case0) + } yield updatedCase } - richCase <- caseSrv.get(EntityIdOrName(caseId)).richCase.getOrFail("Case") - } yield Results.Ok(richCase.toJson) + .flatMap(c => caseSrv.get(c._id).richCase.getOrFail("Case")) + .map(rc => Results.Ok(rc.toJson)) } def markAsRead(alertId: String): Action[AnyContent] = From a013145d2b59a35a54280d365c1c2ed5cfb1d44b Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 28 Oct 2020 09:43:57 +0100 Subject: [PATCH 102/237] #1599 Add operations on customFields --- .../controllers/v1/CustomFieldCtrl.scala | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala index a3295d021c..3acbb91a62 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala @@ -1,9 +1,12 @@ package org.thp.thehive.controllers.v1 import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} -import org.thp.scalligraph.models.Database +import org.thp.scalligraph.models.{Database, Entity, UMapping} +import org.thp.scalligraph.query.{ParamQuery, PublicProperties, PublicPropertyListBuilder, Query} import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ import org.thp.thehive.services.CustomFieldSrv @@ -12,7 +15,33 @@ import play.api.mvc.{Action, AnyContent, Results} import scala.util.Success @Singleton -class CustomFieldCtrl @Inject() (entrypoint: Entrypoint, @Named("with-thehive-schema") db: Database, customFieldSrv: CustomFieldSrv) { +class CustomFieldCtrl @Inject() (entrypoint: Entrypoint, @Named("with-thehive-schema") db: Database, customFieldSrv: CustomFieldSrv) + extends QueryableCtrl { + + override val entityName: String = "CustomField" + override val initialQuery: Query = Query.init[Traversal.V[CustomField]]("listCustomField", (graph, _) => customFieldSrv.startTraversal(graph)) + override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[CustomField], IteratorOutput]( + "page", + FieldsParser[OutputParam], + { + case (OutputParam(from, to, _), customFieldSteps, _) => + customFieldSteps.page(from, to, withTotal = true) + } + ) + override val outputQuery: Query = Query.output[CustomField with Entity] + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[CustomField]]( + "getCustomField", + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => customFieldSrv.get(idOrName)(graph) + ) + override val publicProperties: PublicProperties = PublicPropertyListBuilder[CustomField] + .property("name", UMapping.string)(_.rename("displayName").updatable) + .property("description", UMapping.string)(_.field.updatable) + .property("reference", UMapping.string)(_.rename("name").readonly) + .property("mandatory", UMapping.boolean)(_.field.updatable) + .property("type", UMapping.string)(_.field.readonly) + .property("options", UMapping.json.sequence)(_.field.updatable) + .build def create: Action[AnyContent] = entrypoint("create custom field") From 5e9907789b1b6836af1e2519a11fe55ed0c1f24f Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 28 Oct 2020 09:44:05 +0100 Subject: [PATCH 103/237] #1599 Add operations on profile --- .../controllers/v0/TheHiveQueryExecutor.scala | 30 ++++++------- .../controllers/v1/TheHiveQueryExecutor.scala | 43 +++++++++++++------ 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala index 3db5418fe7..c783dc61ea 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala @@ -35,25 +35,25 @@ object OutputParam { @Singleton class TheHiveQueryExecutor @Inject() ( @Named("with-thehive-schema") override val db: Database, - `case`: PublicCase, - task: PublicTask, - log: PublicLog, - observable: PublicObservable, alert: PublicAlert, - user: PublicUser, + audit: PublicAudit, + `case`: PublicCase, caseTemplate: PublicCaseTemplate, + customField: PublicCustomField, + observableType: PublicObservableType, dashboard: PublicDashboard, + log: PublicLog, + observable: PublicObservable, organisation: PublicOrganisation, - audit: PublicAudit, + page: PublicPage, profile: PublicProfile, tag: PublicTag, - page: PublicPage, - observableType: PublicObservableType, - customField: PublicCustomField + task: PublicTask, + user: PublicUser ) extends QueryExecutor { - lazy val publicDatas: List[PublicData] = - `case` :: task :: log :: observable :: alert :: user :: caseTemplate :: dashboard :: organisation :: audit :: profile :: tag :: page :: observableType :: customField :: Nil + lazy val publicDatas: Seq[PublicData] = + Seq(alert, audit, `case`, caseTemplate, customField, dashboard, log, observable, observableType, organisation, page, profile, tag, task, user) def metaProperties: PublicProperties = PublicPropertyListBuilder @@ -88,10 +88,10 @@ class TheHiveQueryExecutor @Inject() ( } override lazy val queries: Seq[ParamQuery[_]] = - publicDatas.map(_.initialQuery) ::: - publicDatas.map(_.getQuery) ::: - publicDatas.map(_.pageQuery) ::: - publicDatas.map(_.outputQuery) ::: + publicDatas.map(_.initialQuery) ++ + publicDatas.map(_.getQuery) ++ + publicDatas.map(_.pageQuery) ++ + publicDatas.map(_.outputQuery) ++ publicDatas.flatMap(_.extraQueries) override val version: (Int, Int) = 0 -> 0 } diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index 1f4fd9ae4e..a1eef7467c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -20,21 +20,40 @@ object OutputParam { @Singleton class TheHiveQueryExecutor @Inject() ( + alertCtrl: AlertCtrl, + auditCtrl: AuditCtrl, caseCtrl: CaseCtrl, - taskCtrl: TaskCtrl, + caseTemplateCtrl: CaseTemplateCtrl, + customFieldCtrl: CustomFieldCtrl, logCtrl: LogCtrl, observableCtrl: ObservableCtrl, - alertCtrl: AlertCtrl, - userCtrl: UserCtrl, - caseTemplateCtrl: CaseTemplateCtrl, -// dashboardCtrl: DashboardCtrl, organisationCtrl: OrganisationCtrl, - auditCtrl: AuditCtrl, + profileCtrl: ProfileCtrl, + taskCtrl: TaskCtrl, + userCtrl: UserCtrl, + // dashboardCtrl: DashboardCtrl, @Named("with-thehive-schema") implicit val db: Database ) extends QueryExecutor { - lazy val controllers: List[QueryableCtrl] = - caseCtrl :: taskCtrl :: alertCtrl :: userCtrl :: caseTemplateCtrl :: organisationCtrl :: auditCtrl :: observableCtrl :: logCtrl :: Nil + lazy val controllers: Seq[QueryableCtrl] = + Seq( + alertCtrl, + auditCtrl, + caseCtrl, + caseTemplateCtrl, + customFieldCtrl, + // dashboardCtrl, + logCtrl, + observableCtrl, +// observableTypeCtrl, + organisationCtrl, +// pageCtrl, + profileCtrl, +// tagCtrl, + taskCtrl, + userCtrl + ) + override val version: (Int, Int) = 1 -> 1 def metaProperties: PublicProperties = @@ -48,9 +67,9 @@ class TheHiveQueryExecutor @Inject() ( override lazy val publicProperties: PublicProperties = controllers.foldLeft(metaProperties)(_ ++ _.publicProperties) override lazy val queries: Seq[ParamQuery[_]] = - controllers.map(_.initialQuery) ::: - controllers.map(_.getQuery) ::: - controllers.map(_.pageQuery) ::: - controllers.map(_.outputQuery) ::: + controllers.map(_.initialQuery) ++ + controllers.map(_.getQuery) ++ + controllers.map(_.pageQuery) ++ + controllers.map(_.outputQuery) ++ controllers.flatMap(_.extraQueries) } From 689243703428e34e42077dfdb714b4c0e74ab2a0 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 28 Oct 2020 09:45:28 +0100 Subject: [PATCH 104/237] #1599 Fix organisation visibility on listOrganisation query --- .../org/thp/thehive/controllers/v1/OrganisationCtrl.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala index 336ec1458c..115327a732 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/OrganisationCtrl.scala @@ -27,7 +27,13 @@ class OrganisationCtrl @Inject() ( override val entityName: String = "organisation" override val publicProperties: PublicProperties = properties.organisation override val initialQuery: Query = - Query.init[Traversal.V[Organisation]]("listOrganisation", (graph, authContext) => organisationSrv.visibleOrganisation(graph, authContext)) + Query.init[Traversal.V[Organisation]]( + "listOrganisation", + (graph, authContext) => + organisationSrv + .startTraversal(graph) + .visible(authContext) + ) override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Organisation], IteratorOutput]( "page", FieldsParser[OutputParam], From c0283df2a4329ee402549c6eabe436e8c1cd9c9c Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 28 Oct 2020 09:49:22 +0100 Subject: [PATCH 105/237] #1600 Fix wsConfig key in configuration sample --- conf/application.sample.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/application.sample.conf b/conf/application.sample.conf index 617d3f7d78..0f36eb225b 100644 --- a/conf/application.sample.conf +++ b/conf/application.sample.conf @@ -71,7 +71,7 @@ storage { // type: "bearer" // key: "***" # Cortex API key // } -// ws {} # HTTP client configuration (SSL and proxy) +// wsConfig {} # HTTP client configuration (SSL and proxy) // } // ] // } @@ -90,7 +90,7 @@ storage { // type = key // key = "***" # MISP API key // } -// ws {} # HTTP client configuration (SSL and proxy) +// wsConfig {} # HTTP client configuration (SSL and proxy) // } // ] //} From c374028c9fbefb71d5e9731ac50a06cb42983065 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Tue, 27 Oct 2020 17:36:26 +0100 Subject: [PATCH 106/237] #1556 No MatchError thanks to a Gremlin filter step --- thehive/app/org/thp/thehive/services/AuditSrv.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index 449335dc7a..9892c34dce 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -296,6 +296,7 @@ object AuditOps { def richAudit: Traversal[RichAudit, JMap[String, Any], Converter[RichAudit, JMap[String, Any]]] = traversal + .filter(_.context) .project( _.by .by(_.`case`.entity.fold) @@ -312,6 +313,7 @@ object AuditOps { entityRenderer: Traversal.V[Audit] => Traversal[D, G, C] ): Traversal[(RichAudit, D), JMap[String, Any], Converter[(RichAudit, D), JMap[String, Any]]] = traversal + .filter(_.context) .project( _.by .by(_.`case`.entity.fold) @@ -320,10 +322,9 @@ object AuditOps { .by(entityRenderer) ) .domainMap { - case (audit, context, visibilityContext, obj, renderedObject) if context.nonEmpty || visibilityContext.nonEmpty => + case (audit, context, visibilityContext, obj, renderedObject) => val ctx = if (context.isEmpty) visibilityContext.head else context.head RichAudit(audit, ctx, visibilityContext.head, obj.headOption) -> renderedObject - // case otherwise // FIXME } // def forCase(caseId: String): Traversal.V[Audit] = traversal.filter(_.`case`.hasId(caseId)) From 934548a0c1110a9d27e5fcaf9c9fee6710c03d5b Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 28 Oct 2020 11:17:01 +0100 Subject: [PATCH 107/237] #1556 Added details to Audit when deleting a Case --- thehive/app/org/thp/thehive/services/AuditSrv.scala | 4 ++-- thehive/app/org/thp/thehive/services/CaseSrv.scala | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index 9892c34dce..da4cd7981e 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -186,8 +186,8 @@ class AuditSrv @Inject() ( if (details == JsObject.empty) Success(()) else auditSrv.create(Audit(Audit.update, entity, Some(details.toString)), Some(entity), Some(entity)) - def delete(entity: E with Entity, context: Product with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - auditSrv.create(Audit(Audit.delete, entity, None), Some(context), None) + def delete(entity: E with Entity, context: Product with Entity, details: Option[JsObject] = None)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + auditSrv.create(Audit(Audit.delete, entity, details.map(_.toString())), Some(context), None) } class UserAudit extends SelfContextObjectAudit[User] { diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index c58479fdee..0c0caf292f 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -176,14 +176,16 @@ class CaseSrv @Inject() ( } yield () } - def remove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + def remove(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + val details = Json.obj("number" -> `case`.number, "title" -> `case`.title) for { organisation <- organisationSrv.getOrFail(authContext.organisation) - _ <- auditSrv.`case`.delete(`case`, organisation) + _ <- auditSrv.`case`.delete(`case`, organisation, Some(details)) } yield { get(`case`).share.remove() get(`case`).remove() } + } override def getByName(name: String)(implicit graph: Graph): Traversal.V[Case] = Try(startTraversal.getByNumber(name.toInt)).getOrElse(startTraversal.limit(0)) From 69b94253b65c5c138b85b8559f8aabd941310e55 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 28 Oct 2020 15:49:48 +0100 Subject: [PATCH 108/237] #1543 Replaced soft delete by hard delete --- .../org/thp/thehive/controllers/v0/CaseCtrl.scala | 11 ----------- .../org/thp/thehive/controllers/v0/Router.scala | 4 ++-- .../org/thp/thehive/controllers/v1/CaseCtrl.scala | 14 ++++++++------ .../org/thp/thehive/controllers/v1/Router.scala | 1 - thehive/app/org/thp/thehive/models/Case.scala | 2 +- .../services/notification/notifiers/Webhook.scala | 1 - .../thp/thehive/controllers/v0/CaseCtrlTest.scala | 2 +- 7 files changed, 12 insertions(+), 23 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index ec72368785..9cd28662a3 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -135,17 +135,6 @@ class CaseCtrl @Inject() ( } def delete(caseIdOrNumber: String): Action[AnyContent] = - entrypoint("delete case") - .authTransaction(db) { implicit request => implicit graph => - caseSrv - .get(EntityIdOrName(caseIdOrNumber)) - .can(Permissions.manageCase) - .update(_.status, CaseStatus.Deleted) - .getOrFail("Case") - .map(_ => Results.NoContent) - } - - def realDelete(caseIdOrNumber: String): Action[AnyContent] = entrypoint("delete case") .authTransaction(db) { implicit request => implicit graph => for { diff --git a/thehive/app/org/thp/thehive/controllers/v0/Router.scala b/thehive/app/org/thp/thehive/controllers/v0/Router.scala index 96f8327e37..547c48faf7 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Router.scala @@ -91,10 +91,10 @@ class Router @Inject() ( case PATCH(p"/case/_bulk") => caseCtrl.bulkUpdate // Not used by the frontend case PATCH(p"/case/$caseId") => caseCtrl.update(caseId) // Audit ok case POST(p"/case/_merge/$caseIds") => caseCtrl.merge(caseIds) // Not implemented in backend and not used by frontend - case DELETE(p"/case/$caseId") => caseCtrl.delete(caseId) // Not used by frontend case POST(p"/case/_search") => caseCtrl.search case POST(p"/case/_stats") => caseCtrl.stats - case DELETE(p"/case/$caseId/force") => caseCtrl.realDelete(caseId) // Audit ok + case DELETE(p"/case/$caseId") => caseCtrl.delete(caseId) // Not used by the frontend + case DELETE(p"/case/$caseId/force") => caseCtrl.delete(caseId) // Audit ok case GET(p"/case/$caseId/links") => caseCtrl.linkedCases(caseId) case GET(p"/case/template") => caseTemplateCtrl.search diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index aeb3be90a7..3139f7d6e3 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -112,12 +112,14 @@ class CaseCtrl @Inject() ( def delete(caseIdOrNumber: String): Action[AnyContent] = entrypoint("delete case") .authTransaction(db) { implicit request => implicit graph => - caseSrv - .get(EntityIdOrName(caseIdOrNumber)) - .can(Permissions.manageCase) - .update(_.status, CaseStatus.Deleted) - .getOrFail("Case") - .map(_ => Results.NoContent) + for { + c <- + caseSrv + .get(EntityIdOrName(caseIdOrNumber)) + .can(Permissions.manageCase) + .getOrFail("Case") + _ <- caseSrv.remove(c) + } yield Results.NoContent } def merge(caseIdsOrNumbers: String): Action[AnyContent] = diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index a04cc7769d..feffe865bb 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -38,7 +38,6 @@ class Router @Inject() ( case DELETE(p"/case/$caseId") => caseCtrl.delete(caseId) // case PATCH(p"api/case/_bulk") => caseCtrl.bulkUpdate() // case POST(p"/case/_stats") => caseCtrl.stats() -// case DELETE(p"/case/$caseId/force") => caseCtrl.realDelete(caseId) // case GET(p"/case/$caseId/links") => caseCtrl.linkedCases(caseId) case GET(p"/caseTemplate") => caseTemplateCtrl.list diff --git a/thehive/app/org/thp/thehive/models/Case.scala b/thehive/app/org/thp/thehive/models/Case.scala index fa18c888fe..1990523baf 100644 --- a/thehive/app/org/thp/thehive/models/Case.scala +++ b/thehive/app/org/thp/thehive/models/Case.scala @@ -8,7 +8,7 @@ import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType} import play.api.libs.json.{Format, Json} object CaseStatus extends Enumeration { - val Open, Resolved, Deleted, Duplicated = Value + val Open, Resolved, Duplicated = Value implicit val format: Format[CaseStatus.Value] = Json.formatEnum(CaseStatus) } diff --git a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala index de4e6d4c6f..20bec8981e 100644 --- a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala +++ b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala @@ -13,7 +13,6 @@ import org.thp.scalligraph.traversal.{Converter, IdentityConverter, Traversal} import org.thp.scalligraph.{BadConfigurationError, EntityIdOrName} import org.thp.thehive.controllers.v0.AuditRenderer import org.thp.thehive.controllers.v0.Conversion.fromObjectType -import org.thp.thehive.models.Audit._ import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.AuditOps._ diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala index e36be809b7..a3e623fe55 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala @@ -354,7 +354,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { val requestDel = FakeRequest("DELETE", s"/api/v0/case/#1/force") .withHeaders("user" -> "certuser@thehive.local") - val resultDel = app[CaseCtrl].realDelete("1")(requestDel) + val resultDel = app[CaseCtrl].delete("1")(requestDel) status(resultDel) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(resultDel)}") app[Database].roTransaction { implicit graph => From 9d9b17c1d1ff0189af6938647ac4aa78a804c058 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 28 Oct 2020 17:49:07 +0100 Subject: [PATCH 109/237] #1543 Updated the database's schema --- .../app/org/thp/thehive/models/TheHiveSchemaDefinition.scala | 4 ++++ .../test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 41b8efcaf6..f994c29280 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -68,6 +68,10 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .noop // .addIndex("Tag", IndexType.unique, "namespace", "predicate", "value") .noop // .addIndex("Audit", IndexType.basic, "requestId", "mainAction") .rebuildIndexes + .updateGraph("Remove cases with a Deleted status", "Case") { traversal => + traversal.unsafeHas("status", "Deleted").remove() + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala index a3e623fe55..3d90d757f2 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala @@ -293,7 +293,7 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { val requestSearch = FakeRequest("POST", s"/api/v0/case/_search?range=0-15&sort=-flag&sort=-startDate&nstats=true") .withHeaders("user" -> "socuser@thehive.local") .withJsonBody( - Json.parse("""{"query":{"_and":[{"_field":"customFields.boolean1","_value":true},{"_not":{"status":"Deleted"}}]}}""") + Json.parse("""{"query":{"_and":[{"_field":"customFields.boolean1","_value":true}]}}""") ) val resultSearch = app[CaseCtrl].search()(requestSearch) status(resultSearch) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(resultSearch)}") From 3f3abe90f901fe5a6fb79ae56c8115cf2630d3e8 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 28 Oct 2020 15:29:22 +0100 Subject: [PATCH 110/237] #1478 Remove warnings --- .../scala/org/thp/thehive/migration/Migrate.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala index 3c244a5085..c946b48ad1 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala @@ -1,8 +1,6 @@ package org.thp.thehive.migration import java.io.File -import java.text.SimpleDateFormat -import java.util.Date import akka.actor.ActorSystem import akka.stream.Materializer @@ -76,11 +74,11 @@ object Migrate extends App with MigrationOps { opt[String]("case-from-date") .valueName("") .text("migrate only cases created from ") - .action((v, c) => addConfig(c, "input.filter.caseFromDate" -> v.toString)), + .action((v, c) => addConfig(c, "input.filter.caseFromDate" -> v)), opt[String]("case-until-date") .valueName("") .text("migrate only cases created until ") - .action((v, c) => addConfig(c, "input.filter.caseUntilDate" -> v.toString)), + .action((v, c) => addConfig(c, "input.filter.caseUntilDate" -> v)), /* case number */ opt[Duration]("case-from-number") .valueName("") @@ -102,11 +100,11 @@ object Migrate extends App with MigrationOps { opt[String]("alert-from-date") .valueName("") .text("migrate only alerts created from ") - .action((v, c) => addConfig(c, "input.filter.alertFromDate" -> v.toString)), + .action((v, c) => addConfig(c, "input.filter.alertFromDate" -> v)), opt[String]("alert-until-date") .valueName("") .text("migrate only alerts created until ") - .action((v, c) => addConfig(c, "input.filter.alertUntilDate" -> v.toString)), + .action((v, c) => addConfig(c, "input.filter.alertUntilDate" -> v)), /* audit age */ opt[Duration]("max-audit-age") .valueName("") @@ -119,11 +117,11 @@ object Migrate extends App with MigrationOps { opt[String]("audit-from-date") .valueName("") .text("migrate only audits created from ") - .action((v, c) => addConfig(c, "input.filter.auditFromDate" -> v.toString)), + .action((v, c) => addConfig(c, "input.filter.auditFromDate" -> v)), opt[String]("audit-until-date") .valueName("") .text("migrate only audits created until ") - .action((v, c) => addConfig(c, "input.filter.auditUntilDate" -> v.toString)), + .action((v, c) => addConfig(c, "input.filter.auditUntilDate" -> v)), note("Accepted date formats are \"yyyyMMdd[HH[mm[ss]]]\" and \"MMdd\""), note( "The Format for duration is: .\n" + From 30267c69b88b267856cdd474e4d3301261dc33c3 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 29 Oct 2020 07:55:22 +0100 Subject: [PATCH 111/237] Update sbt and play --- .gitignore | 3 +-- ScalliGraph | 2 +- build.sbt | 5 ++--- docker.sbt | 3 --- project/build.properties | 2 +- project/plugins.sbt | 2 +- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 9e91904c55..a8a3588a45 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ RUNNING_PID .cache-main .cache-tests sbt-launch.jar +.bsp/ # Eclipse .project @@ -36,5 +37,3 @@ tmp /.idea/** !/.idea/runConfigurations/ !/.idea/runConfigurations/* - -dev diff --git a/ScalliGraph b/ScalliGraph index a6637cd432..d3d2b24006 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit a6637cd4321973ec49f8f83df0397e3d7ec3c9af +Subproject commit d3d2b240068fd9feac8bef6846d0df985a72ee07 diff --git a/build.sbt b/build.sbt index 22aed1a0a3..cd699f5508 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone val thehiveVersion = "4.0.1-1-SNAPSHOT" -val scala212 = "2.12.11" +val scala212 = "2.12.12" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) @@ -344,8 +344,7 @@ lazy val thehiveMigration = (project in file("migration")) specs % Test ), fork := true, - normalizedName := "migrate", - mainClass := None + normalizedName := "migrate" ) lazy val rpmPackageRelease = (project in file("package/rpm-release")) diff --git a/docker.sbt b/docker.sbt index 2e67f68a97..60d0f5fe54 100644 --- a/docker.sbt +++ b/docker.sbt @@ -13,10 +13,7 @@ version in Docker := { defaultLinuxInstallLocation in Docker := "/opt/thehive" dockerRepository := Some("thehiveproject") dockerUpdateLatest := !version.value.toUpperCase.contains("RC") && !version.value.contains("SNAPSHOT") -dockerEntrypoint := Seq("/opt/thehive/entrypoint") dockerExposedPorts := Seq(9000) -daemonUser in Docker := "thehive" -daemonGroup in Docker := "thehive" mappings in Docker ++= Seq( file("package/docker/entrypoint") -> "/opt/thehive/entrypoint", file("package/logback.xml") -> "/etc/thehive/logback.xml", diff --git a/project/build.properties b/project/build.properties index a919a9b5f4..08e4d79332 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.3.8 +sbt.version=1.4.1 diff --git a/project/plugins.sbt b/project/plugins.sbt index eeda243aba..0eacab22d7 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.2") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.3") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.3.0") addSbtPlugin("org.thehive-project" % "sbt-github-changelog" % "0.3.0") From 5247530686f1eeb2f17e0ac80ad6f8851b783586 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 29 Oct 2020 12:33:38 +0100 Subject: [PATCH 112/237] #1599 Add the field "options" to custom fields --- .../main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index 095142e98d..d1651a4262 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -14,7 +14,7 @@ object InputCustomField { implicit val writes: Writes[InputCustomField] = Json.writes[InputCustomField] } -case class OutputCustomField(name: String, description: String, `type`: String, mandatory: Boolean) +case class OutputCustomField(name: String, description: String, `type`: String, options: Seq[JsValue], mandatory: Boolean) object OutputCustomField { implicit val format: OFormat[OutputCustomField] = Json.format[OutputCustomField] From 18caa17cf917710e2cc42d361779dc305f7d18fa Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 29 Oct 2020 12:34:18 +0100 Subject: [PATCH 113/237] #1599 Make custom fields type updatable --- .../app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala | 2 +- .../app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala index fd9ae73206..d4293f404d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala @@ -110,7 +110,7 @@ class PublicCustomField @Inject() (customFieldSrv: CustomFieldSrv) extends Publi .property("description", UMapping.string)(_.field.updatable) .property("reference", UMapping.string)(_.rename("name").readonly) .property("mandatory", UMapping.boolean)(_.field.updatable) - .property("type", UMapping.string)(_.field.readonly) + .property("type", UMapping.string)(_.field.updatable) .property("options", UMapping.json.sequence)(_.field.updatable) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala index 3acbb91a62..1e5c47e94a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CustomFieldCtrl.scala @@ -39,7 +39,7 @@ class CustomFieldCtrl @Inject() (entrypoint: Entrypoint, @Named("with-thehive-sc .property("description", UMapping.string)(_.field.updatable) .property("reference", UMapping.string)(_.rename("name").readonly) .property("mandatory", UMapping.boolean)(_.field.updatable) - .property("type", UMapping.string)(_.field.readonly) + .property("type", UMapping.string)(_.field.updatable) .property("options", UMapping.json.sequence)(_.field.updatable) .build From fe9b36f1fef720d98b2c4492fca7855ec07bdbb8 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 28 Oct 2020 08:17:30 +0100 Subject: [PATCH 114/237] #1579 Add APIs to list user and org configs --- .../thehive/controllers/v0/ConfigCtrl.scala | 73 +++++++++++++++++-- .../thp/thehive/controllers/v0/Router.scala | 8 +- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala index 8f63df3e47..4b58eab688 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ConfigCtrl.scala @@ -1,26 +1,30 @@ package org.thp.thehive.controllers.v0 -import com.typesafe.config.{Config, ConfigRenderOptions} +import com.typesafe.config.{ConfigRenderOptions, Config => TypeSafeConfig} import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} +import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{AuthorizationError, NotFoundError} import org.thp.thehive.models.Permissions import org.thp.thehive.services.OrganisationOps._ -import org.thp.thehive.services.{OrganisationConfigContext, OrganisationSrv, UserConfigContext} -import play.api.libs.json.{JsNull, JsValue, Json, Writes} -import play.api.mvc.{Action, AnyContent, Results} -import play.api.{ConfigLoader, Logger} +import org.thp.thehive.services.UserOps._ +import org.thp.thehive.services._ +import play.api.libs.json._ +import play.api.mvc.{Action, AnyContent, Result, Results} +import play.api.{ConfigLoader, Configuration, Logger} import scala.util.{Failure, Success, Try} @Singleton class ConfigCtrl @Inject() ( + configuration: Configuration, appConfig: ApplicationConfig, userConfigContext: UserConfigContext, organisationConfigContext: OrganisationConfigContext, organisationSrv: OrganisationSrv, + userSrv: UserSrv, entrypoint: Entrypoint, @Named("with-thehive-schema") db: Database ) { @@ -36,7 +40,7 @@ class ConfigCtrl @Inject() ( ) ) - implicit val jsonConfigLoader: ConfigLoader[JsValue] = (config: Config, path: String) => + implicit val jsonConfigLoader: ConfigLoader[JsValue] = (config: TypeSafeConfig, path: String) => Json.parse(config.getValue(path).render(ConfigRenderOptions.concise())) def list: Action[AnyContent] = @@ -72,6 +76,40 @@ class ConfigCtrl @Inject() ( } } + def mergeConfig(defaultValue: JsValue, names: Seq[String], value: JsValue): JsValue = + names + .headOption + .fold[JsValue](value) { key => + defaultValue + .asOpt[JsObject] + .fold(names.foldRight(value)((k, v) => Json.obj(k -> v))) { default => + default + (key -> mergeConfig((defaultValue \ key).getOrElse(JsNull), names.tail, value)) + } + } + + def userList: Action[AnyContent] = + entrypoint("list user configuration item") + .extract("path", FieldsParser[String].optional.on("path")) + .authRoTransaction(db) { implicit request => implicit graph => + val defaultValue = configuration.get[JsValue]("user.defaults") + val userConfiguration = userSrv + .current + .config + .toIterator + .foldLeft(defaultValue)((default, config) => mergeConfig(default, config.name.split('.').toSeq, config.value)) + + request.body("path") match { + case Some(path: String) => + path + .split('.') + .foldLeft[JsLookupResult](JsDefined(userConfiguration))((cfg, key) => cfg \ key) + .toOption + .fold[Try[Result]](Failure(NotFoundError(s"The configuration $path doesn't exist")))(v => Success(Results.Ok(v))) + case None => + Success(Results.Ok(userConfiguration)) + } + } + def userSet(path: String): Action[AnyContent] = entrypoint("set user configuration item") .extract("value", FieldsParser.json.on("value")) @@ -113,6 +151,29 @@ class ConfigCtrl @Inject() ( } } + def organisationList: Action[AnyContent] = + entrypoint("list organisation configuration item") + .extract("path", FieldsParser[String].optional.on("path")) + .authRoTransaction(db) { implicit request => implicit graph => + val defaultValue = configuration.get[JsValue]("organisation.defaults") + val orgConfiguration = organisationSrv + .current + .config + .toIterator + .foldLeft(defaultValue)((default, config) => mergeConfig(default, config.name.split('.').toSeq, config.value)) + + request.body("path") match { + case Some(path: String) => + path + .split('.') + .foldLeft[JsLookupResult](JsDefined(orgConfiguration))((cfg, key) => cfg \ key) + .toOption + .fold[Try[Result]](Failure(NotFoundError(s"The configuration $path doesn't exist")))(v => Success(Results.Ok(v))) + case None => + Success(Results.Ok(orgConfiguration)) + } + } + def organisationGet(path: String): Action[AnyContent] = entrypoint("get organisation configuration item") .auth { implicit request => diff --git a/thehive/app/org/thp/thehive/controllers/v0/Router.scala b/thehive/app/org/thp/thehive/controllers/v0/Router.scala index 547c48faf7..050122e10d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Router.scala @@ -181,13 +181,15 @@ class Router @Inject() ( case GET(p"/describe/_all") => describeCtrl.describeAll case GET(p"/describe/$modelName") => describeCtrl.describe(modelName) - case GET(p"/config") => configCtrl.list - case GET(p"/config/$path") => configCtrl.get(path) - case PUT(p"/config/$path") => configCtrl.set(path) + case GET(p"/config/user") => configCtrl.userList case GET(p"/config/user/$path") => configCtrl.userGet(path) case PUT(p"/config/user/$path") => configCtrl.userSet(path) + case GET(p"/config/organisation") => configCtrl.organisationList case GET(p"/config/organisation/$path") => configCtrl.organisationGet(path) case PUT(p"/config/organisation/$path") => configCtrl.organisationSet(path) + case GET(p"/config") => configCtrl.list + case GET(p"/config/$path") => configCtrl.get(path) + case PUT(p"/config/$path") => configCtrl.set(path) case GET(p"/profile") => profileCtrl.search case POST(p"/profile/_search") => profileCtrl.search From 570d7f32a55b49c756a90f81f8cd403524f546d6 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 29 Oct 2020 14:42:49 +0100 Subject: [PATCH 115/237] #1579 Refactor the UiSettings service and add new ui options related to alert similar cases section --- frontend/app/scripts/app.js | 12 --- .../alert/AlertSimilarCaseListCmp.js | 24 +++++- .../organisation/OrgConfigListCmp.js | 7 +- .../controllers/alert/AlertEventCtrl.js | 2 +- .../case/CaseTemplatesDialogCtrl.js | 2 +- .../app/scripts/services/api/UiSettingsSrv.js | 78 +++++++++---------- .../alert/similar-case-list.component.html | 2 +- .../app/views/components/org/config.list.html | 22 +++++- thehive/conf/reference.conf | 2 + 9 files changed, 87 insertions(+), 64 deletions(-) diff --git a/frontend/app/scripts/app.js b/frontend/app/scripts/app.js index 39e83c6a71..019121422f 100644 --- a/frontend/app/scripts/app.js +++ b/frontend/app/scripts/app.js @@ -313,18 +313,6 @@ angular.module('thehive', [ permissions: ['manageObservableTemplate'] } }) - // .state('app.administration.ui-settings', { - // url: '/ui-settings', - // templateUrl: 'views/partials/admin/ui-settings.html', - // controller: 'AdminUiSettingsCtrl', - // controllerAs: '$vm', - // title: 'UI settings', - // resolve: { - // uiConfig: function(UiSettingsSrv) { - // return UiSettingsSrv.all(); - // } - // } - // }) .state('app.case', { abstract: true, url: 'case/{caseId}', diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 850bad9b42..8f076e9659 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -3,7 +3,7 @@ angular.module('theHiveComponents') .component('alertSimilarCaseList', { - controller: function($scope, FilteringSrv, PaginatedQuerySrv, CaseResolutionStatus) { + controller: function($scope, FilteringSrv, PaginatedQuerySrv, CaseResolutionStatus, UiSettingsSrv) { var self = this; self.CaseResolutionStatus = CaseResolutionStatus; @@ -30,6 +30,11 @@ currentPage: 1 }; + self.state = { + disallowMerge: UiSettingsSrv.disallowMergeAlertInResolvedSimilarCases() === true, + defaultAlertSimilarCaseFilter: UiSettingsSrv.defaultAlertSimilarCaseFilter() + }; + self.$onInit = function() { this.filtering = new FilteringSrv('case', 'alert.dialog.similar-cases', { version: 'v1', @@ -42,8 +47,23 @@ defaultFilter: [] }); - self.filtering.initContext(this.alertId) + self.filtering.initContext('alert.dialog.similar-cases') .then(function() { + var defaultFilter = { + field: 'status', + type: 'enumeration', + value: { + list: [{ + text: 'Open', + label: 'Open' + }] + } + }; + + if(_.isEmpty(self.filtering.context.filters)) { + self.filtering.addFilter(defaultFilter); + } + self.load(); $scope.$watch('$cmp.list.pageSize', function (newValue) { diff --git a/frontend/app/scripts/components/organisation/OrgConfigListCmp.js b/frontend/app/scripts/components/organisation/OrgConfigListCmp.js index f3592d07e5..143bc820ac 100644 --- a/frontend/app/scripts/components/organisation/OrgConfigListCmp.js +++ b/frontend/app/scripts/components/organisation/OrgConfigListCmp.js @@ -7,9 +7,7 @@ var self = this; self.isDirtySetting = function(key, newValue) { - var currentValue = (self.currentSettings[key] || {}).value; - - return newValue !== currentValue; + return newValue !== self.currentSettings[key]; }; self.save = function(/*form*/) { @@ -36,6 +34,7 @@ }; self.loadSettings = function(configurations) { + var notifyRoot = false; var promise; @@ -52,7 +51,7 @@ self.configs = {}; self.settingsKeys.forEach(function(key) { - self.configs[key] = (configs[key] || {}).value; + self.configs[key] = configs[key]; }); if(notifyRoot) { diff --git a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js index 0db05c183c..c36e492711 100644 --- a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js @@ -21,7 +21,7 @@ similarCases: 0 }; - self.hideEmptyCaseButton = UiSettingsSrv.uiHideEmptyCaseButton(); + self.hideEmptyCaseButton = UiSettingsSrv.hideEmptyCaseButton(); self.updateObservableCount = function(count) { self.counts.observables = count; diff --git a/frontend/app/scripts/controllers/case/CaseTemplatesDialogCtrl.js b/frontend/app/scripts/controllers/case/CaseTemplatesDialogCtrl.js index 1412dd640b..67fa8689d8 100644 --- a/frontend/app/scripts/controllers/case/CaseTemplatesDialogCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseTemplatesDialogCtrl.js @@ -7,7 +7,7 @@ this.state = { filter: '', selected: null, - hideEmptyCaseButton: UiSettingsSrv.uiHideEmptyCaseButton() + hideEmptyCaseButton: UiSettingsSrv.hideEmptyCaseButton() }; this.selectTemplate = function(template) { diff --git a/frontend/app/scripts/services/api/UiSettingsSrv.js b/frontend/app/scripts/services/api/UiSettingsSrv.js index 77821aa051..4ffaabd92b 100644 --- a/frontend/app/scripts/services/api/UiSettingsSrv.js +++ b/frontend/app/scripts/services/api/UiSettingsSrv.js @@ -1,64 +1,58 @@ (function() { 'use strict'; - angular.module('theHiveServices').factory('UiSettingsSrv', function($http, $q) { - - var settings = null; + angular.module('theHiveServices').service('UiSettingsSrv', function($http, $q) { var baseUrl = './api/config/organisation/'; + var self = this; - var keys = [ - 'ui.hideEmptyCaseButton' - ]; + this.settings = null; - var factory = { - keys: keys, - clearCache: function() { - settings = null; - }, + this.keys = [ + 'hideEmptyCaseButton', + 'disallowMergeAlertInResolvedSimilarCases', + 'defaultAlertSimilarCaseFilter' + ]; - get: function(name) { - return settings[name]; - }, + this.clearCache = function() { + self.settings = null; + }; - save: function(name, value) { - return $http.put(baseUrl + name, {value: value}); - }, + this.get = function(name) { + return self.settings[name]; + }; - all: function(force) { - var deferred = $q.defer(); + this.save = function(name, value) { + return $http.put(baseUrl + 'ui.' + name, {value: value}); + }; - if(settings === null || force) { + this.all = function(force) { + var deferred = $q.defer(); - settings = {}; + if(self.settings === null || force) { - $q.all(_.map(keys, function(key) { - return $http.get(baseUrl + key); - })).then(function(responses) { - _.each(responses, function(response) { - var data = response.data; + self.settings = {}; - settings[data.path] = data; - settings[data.path].id = data.path; - }); + $http.get('./api/config/organisation?path=ui') + .then(function(response) { + var data = response.data; - deferred.resolve(settings); - }).catch(function(responses) { - deferred.reject(responses); + self.settings = data; + deferred.resolve(data); + }) + .catch(function(response) { + deferred.reject(response); }); - } else { - deferred.resolve(settings); - } - - return deferred.promise; + } else { + deferred.resolve(self.settings); } + + return deferred.promise; }; - keys.forEach(function(key) { + this.keys.forEach(function(key) { var camelcased = s.camelize(key.replace(/\./gi, '_')); - factory[camelcased] = function() { - return (settings[key] || {}).value; + self[camelcased] = function() { + return (self.settings || {})[key]; }; }); - - return factory; }); })(); diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index 86a17ffd0e..be1bb10afb 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -163,7 +163,7 @@
      - +
      diff --git a/frontend/app/views/components/org/config.list.html b/frontend/app/views/components/org/config.list.html index 585bb686ab..b083028c9d 100644 --- a/frontend/app/views/components/org/config.list.html +++ b/frontend/app/views/components/org/config.list.html @@ -6,12 +6,32 @@
      +
      + +
      +
      + +
      +
      +
      + +
      + +
      + +
      +
      +
      diff --git a/thehive/conf/reference.conf b/thehive/conf/reference.conf index 02b10595b0..1bce841547 100644 --- a/thehive/conf/reference.conf +++ b/thehive/conf/reference.conf @@ -165,4 +165,6 @@ integrityCheck { organisation.defaults { ui.hideEmptyCaseButton: false + ui.disallowMergeAlertInResolvedSimilarCases: false + ui.defaultAlertSimilarCaseFilter: "open-cases" } From bb9af685f8e26c04654eced60629e0287c7363ed Mon Sep 17 00:00:00 2001 From: garanews Date: Mon, 2 Nov 2020 09:05:03 +0100 Subject: [PATCH 116/237] bump versions in docker-compose bump versions in docker-compose --- docker/README.md | 12 ++++++------ docker/docker-compose.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/README.md b/docker/README.md index 08914d545d..3aadbdb298 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,12 +2,12 @@ With this docker-compose.yml you will be able to run the following images: - The Hive 4 - Cassandra 3.11 -- Cortex 3.1.0-0.2RC1 -- Elasticsearch 7.9.0 -- Kibana 7.9.0 -- MISP 2.4.131 -- Mysql 8.0.21 -- Redis 6.0.8 +- Cortex 3.1.0-1 +- Elasticsearch 7.9.3 +- Kibana 7.9.3 +- MISP 2.4.133 +- Mysql 8.0.22 +- Redis 6.0.9 - Shuffle 0.7.1 ## Some Hint diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5d16e2b6ec..ebd066e945 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.8" services: elasticsearch: - image: 'elasticsearch:7.9.0' + image: 'elasticsearch:7.9.3' container_name: elasticsearch restart: unless-stopped ports: @@ -25,7 +25,7 @@ services: - ./elasticsearch/data:/usr/share/elasticsearch/data - ./elasticsearch/logs:/usr/share/elasticsearch/logs kibana: - image: 'docker.elastic.co/kibana/kibana:7.9.0' + image: 'docker.elastic.co/kibana/kibana:7.9.3' container_name: kibana restart: unless-stopped depends_on: @@ -33,7 +33,7 @@ services: ports: - '5601:5601' cortex: - image: 'thehiveproject/cortex:3.1.0-0.2RC1' + image: 'thehiveproject/cortex:3.1.0-1' container_name: cortex restart: unless-stopped volumes: From 2bf529a7c59f73bc84a42b68a56a9f6bcb12ce14 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 2 Nov 2020 16:56:46 +0100 Subject: [PATCH 117/237] #1599 Add observableType queries --- .../thp/thehive/dto/v1/ObservableType.scala | 26 ++++++++ .../thehive/controllers/v1/Conversion.scala | 22 +++++++ .../controllers/v1/ObservableTypeCtrl.scala | 65 +++++++++++++++++++ .../controllers/v1/TheHiveQueryExecutor.scala | 5 +- 4 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 dto/src/main/scala/org/thp/thehive/dto/v1/ObservableType.scala create mode 100644 thehive/app/org/thp/thehive/controllers/v1/ObservableTypeCtrl.scala diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/ObservableType.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/ObservableType.scala new file mode 100644 index 0000000000..36940202fa --- /dev/null +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/ObservableType.scala @@ -0,0 +1,26 @@ +package org.thp.thehive.dto.v1 + +import java.util.Date + +import play.api.libs.json.{Json, OFormat, Writes} + +case class InputObservableType(name: String, isAttachment: Option[Boolean]) + +object InputObservableType { + implicit val writes: Writes[InputObservableType] = Json.writes[InputObservableType] +} + +case class OutputObservableType( + _id: String, + _type: String, + _updatedAt: Option[Date] = None, + _updatedBy: Option[String] = None, + _createdAt: Date, + _createdBy: String, + name: String, + isAttachment: Boolean +) + +object OutputObservableType { + implicit val format: OFormat[OutputObservableType] = Json.format[OutputObservableType] +} diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 53c6e2c158..68566d62c1 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -420,4 +420,26 @@ object Conversion { .transform } + implicit val observableTypeOutput: Renderer.Aux[ObservableType with Entity, OutputObservableType] = + Renderer.toJson[ObservableType with Entity, OutputObservableType](observableType => + observableType + .asInstanceOf[ObservableType] + .into[OutputObservableType] + .withFieldConst(_._id, observableType._id.toString) + .withFieldConst(_._updatedAt, observableType._updatedAt) + .withFieldConst(_._updatedBy, observableType._updatedBy) + .withFieldConst(_._createdAt, observableType._createdAt) + .withFieldConst(_._createdBy, observableType._createdBy) + .withFieldConst(_._type, "ObservableType") + .transform + ) + + implicit class InputObservableTypeOps(inputObservableType: InputObservableType) { + def toObservableType: ObservableType = + inputObservableType + .into[ObservableType] + .withFieldComputed(_.isAttachment, _.isAttachment.getOrElse(false)) + .transform + } + } diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableTypeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableTypeCtrl.scala new file mode 100644 index 0000000000..3d7353b0b6 --- /dev/null +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableTypeCtrl.scala @@ -0,0 +1,65 @@ +package org.thp.thehive.controllers.v1 + +import javax.inject.{Inject, Named, Singleton} +import org.thp.scalligraph.EntityIdOrName +import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} +import org.thp.scalligraph.models.{Database, Entity, UMapping} +import org.thp.scalligraph.query._ +import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} +import org.thp.thehive.controllers.v1.Conversion._ +import org.thp.thehive.dto.v1.InputObservableType +import org.thp.thehive.models.{ObservableType, Permissions} +import org.thp.thehive.services.ObservableTypeSrv +import play.api.mvc.{Action, AnyContent, Results} + +@Singleton +class ObservableTypeCtrl @Inject() ( + val entrypoint: Entrypoint, + @Named("with-thehive-schema") db: Database, + observableTypeSrv: ObservableTypeSrv +) extends QueryableCtrl { + override val entityName: String = "ObservableType" + override val initialQuery: Query = + Query.init[Traversal.V[ObservableType]]("listObservableType", (graph, _) => observableTypeSrv.startTraversal(graph)) + override val pageQuery: ParamQuery[OutputParam] = + Query.withParam[OutputParam, Traversal.V[ObservableType], IteratorOutput]( + "page", + FieldsParser[OutputParam], + (range, observableTypeSteps, _) => observableTypeSteps.richPage(range.from, range.to, withTotal = true)(identity) + ) + override val outputQuery: Query = Query.output[ObservableType with Entity] + override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[ObservableType]]( + "getObservableType", + FieldsParser[EntityIdOrName], + (idOrName, graph, _) => observableTypeSrv.get(idOrName)(graph) + ) + override val publicProperties: PublicProperties = PublicPropertyListBuilder[ObservableType] + .property("name", UMapping.string)(_.field.readonly) + .property("isAttachment", UMapping.boolean)(_.field.readonly) + .build + + def get(idOrName: String): Action[AnyContent] = + entrypoint("get observable type").authRoTransaction(db) { _ => implicit graph => + observableTypeSrv + .get(EntityIdOrName(idOrName)) + .getOrFail("Observable") + .map(ot => Results.Ok(ot.toJson)) + } + + def create: Action[AnyContent] = + entrypoint("create observable type") + .extract("observableType", FieldsParser[InputObservableType]) + .authPermittedTransaction(db, Permissions.manageObservableTemplate) { implicit request => implicit graph => + val inputObservableType: InputObservableType = request.body("observableType") + observableTypeSrv + .create(inputObservableType.toObservableType) + .map(observableType => Results.Created(observableType.toJson)) + } + + def delete(idOrName: String): Action[AnyContent] = + entrypoint("delete observable type") + .authPermittedTransaction(db, Permissions.manageObservableTemplate) { _ => implicit graph => + observableTypeSrv.remove(EntityIdOrName(idOrName)).map(_ => Results.NoContent) + } +} diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index a1eef7467c..b22475eb47 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -27,6 +27,7 @@ class TheHiveQueryExecutor @Inject() ( customFieldCtrl: CustomFieldCtrl, logCtrl: LogCtrl, observableCtrl: ObservableCtrl, + observableTypeCtrl: ObservableTypeCtrl, organisationCtrl: OrganisationCtrl, profileCtrl: ProfileCtrl, taskCtrl: TaskCtrl, @@ -42,10 +43,10 @@ class TheHiveQueryExecutor @Inject() ( caseCtrl, caseTemplateCtrl, customFieldCtrl, - // dashboardCtrl, +// dashboardCtrl, logCtrl, observableCtrl, -// observableTypeCtrl, + observableTypeCtrl, organisationCtrl, // pageCtrl, profileCtrl, From e622b42cdbad93916509db14bec882cf866523ad Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 3 Nov 2020 11:49:52 +0100 Subject: [PATCH 118/237] #1599 Add metadata in custom fields --- .../org/thp/thehive/dto/v1/CustomFieldValue.scala | 14 +++++++++++++- .../thp/thehive/controllers/v1/Conversion.scala | 11 +++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index d1651a4262..00159322de 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -14,7 +14,19 @@ object InputCustomField { implicit val writes: Writes[InputCustomField] = Json.writes[InputCustomField] } -case class OutputCustomField(name: String, description: String, `type`: String, options: Seq[JsValue], mandatory: Boolean) +case class OutputCustomField( + _id: String, + _type: String, + _createdBy: String, + _updatedBy: Option[String] = None, + _createdAt: Date, + _updatedAt: Option[Date] = None, + name: String, + description: String, + `type`: String, + options: Seq[JsValue], + mandatory: Boolean +) object OutputCustomField { implicit val format: OFormat[OutputCustomField] = Json.format[OutputCustomField] diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 68566d62c1..e2186c0419 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -166,9 +166,16 @@ object Conversion { ) implicit val customFieldOutput: Renderer.Aux[CustomField with Entity, OutputCustomField] = - Renderer.toJson[CustomField with Entity, OutputCustomField]( - _.asInstanceOf[CustomField] + Renderer.toJson[CustomField with Entity, OutputCustomField](customField => + customField + .asInstanceOf[CustomField] .into[OutputCustomField] + .withFieldConst(_._id, customField._id.toString) + .withFieldConst(_._type, "CustomField") + .withFieldConst(_._createdAt, customField._createdAt) + .withFieldConst(_._createdBy, customField._createdBy) + .withFieldConst(_._updatedAt, customField._updatedAt) + .withFieldConst(_._updatedBy, customField._updatedBy) .withFieldComputed(_.`type`, _.`type`.toString) .withFieldComputed(_.mandatory, _.mandatory) .transform From 05b751bf995fb7bd16b5d8fbc186ec02861a7926 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 3 Nov 2020 13:51:10 +0100 Subject: [PATCH 119/237] #1610 Don't update analyzer tags if job status is not success --- .../connector/cortex/services/JobSrv.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala index 85845bedcc..55d98220ef 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala @@ -180,13 +180,15 @@ class JobSrv @Inject() ( } private def importAnalyzerTags(job: Job with Entity, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Unit] = - db.tryTransaction { implicit graph => - val tags = cortexJob.report.fold[Seq[ReportTag]](Nil)(_.summary.map(_.toAnalyzerTag(job.workerName))) - for { - observable <- get(job).observable.getOrFail("Observable") - _ <- reportTagSrv.updateTags(observable, job.workerName, tags) - } yield () - } + if (cortexJob.status == JobStatus.Success) + db.tryTransaction { implicit graph => + val tags = cortexJob.report.fold[Seq[ReportTag]](Nil)(_.summary.map(_.toAnalyzerTag(job.workerName))) + for { + observable <- get(job).observable.getOrFail("Observable") + _ <- reportTagSrv.updateTags(observable, job.workerName, tags) + } yield () + } + else Success(()) /** * Create observable for each artifact of the job report From 683c9ca9a7e3d8f9447fafcb2f3c4bfae320aead Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 3 Nov 2020 14:01:36 +0100 Subject: [PATCH 120/237] #1490 Limit flow size --- thehive/app/org/thp/thehive/services/FlowActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/services/FlowActor.scala b/thehive/app/org/thp/thehive/services/FlowActor.scala index 593a5956b3..44b6da1d26 100644 --- a/thehive/app/org/thp/thehive/services/FlowActor.scala +++ b/thehive/app/org/thp/thehive/services/FlowActor.scala @@ -62,7 +62,7 @@ class FlowActor extends Actor { organisations.foreach { organisation => val cacheKey = FlowId(organisation, None).toString val ids = cache.get[List[String]](cacheKey).getOrElse(Nil) - cache.set(cacheKey, id :: ids) + cache.set(cacheKey, (id :: ids).take(10)) cases.foreach { caseId => val cacheKey: String = FlowId(organisation, Some(caseId)).toString val ids = cache.get[List[String]](cacheKey).getOrElse(Nil) From 3d1fe3dcb7c75e833cb7e5ea6cbf41c79c6bf6a5 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 4 Nov 2020 15:58:58 +0100 Subject: [PATCH 121/237] #1599 Add metadata in describe API --- ScalliGraph | 2 +- .../thehive/controllers/v0/DescribeCtrl.scala | 44 ++++++++++++++++--- .../thehive/controllers/v1/DescribeCtrl.scala | 34 +++++++++++--- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index d3d2b24006..8e342bb733 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit d3d2b240068fd9feac8bef6846d0df985a72ee07 +Subproject commit 8e342bb7331457ca18aa7a3d5991c3a905d0fe67 diff --git a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala index 8acaf7294b..e7ee191d0d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala @@ -26,13 +26,20 @@ import scala.util.{Failure, Success, Try} class DescribeCtrl @Inject() ( cacheApi: SyncCacheApi, entrypoint: Entrypoint, - caseCtrl: CaseCtrl, - taskCtrl: TaskCtrl, alertCtrl: AlertCtrl, + auditCtrl: AuditCtrl, + caseCtrl: CaseCtrl, + caseTemplateCtrl: CaseTemplateCtrl, + customFieldCtrl: CustomFieldCtrl, + dashboardCtrl: DashboardCtrl, + logCtrl: LogCtrl, observableCtrl: ObservableCtrl, + observableTypeCtrl: ObservableTypeCtrl, + organisationCtrl: OrganisationCtrl, + pageCtrl: PageCtrl, + profileCtrl: ProfileCtrl, + taskCtrl: TaskCtrl, userCtrl: UserCtrl, - logCtrl: LogCtrl, - auditCtrl: AuditCtrl, customFieldSrv: CustomFieldSrv, injector: Injector, @Named("with-thehive-schema") db: Database, @@ -40,12 +47,18 @@ class DescribeCtrl @Inject() ( ) { case class PropertyDescription(name: String, `type`: String, values: Seq[JsValue] = Nil, labels: Seq[String] = Nil) + val metadata = Seq( + PropertyDescription("createdBy", "user"), + PropertyDescription("createdAt", "date"), + PropertyDescription("updatedBy", "user"), + PropertyDescription("updatedAt", "date") + ) case class EntityDescription(label: String, path: String, attributes: Seq[PropertyDescription]) { def toJson: JsObject = Json.obj( "label" -> label, "path" -> path, - "attributes" -> attributes + "attributes" -> (attributes ++ metadata) ) } @@ -88,7 +101,26 @@ class DescribeCtrl @Inject() ( ), EntityDescription("user", "/user", userCtrl.publicData.publicProperties.list.flatMap(propertyToJson("user", _))), EntityDescription("case_task_log", "/case/task/log", logCtrl.publicData.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), - EntityDescription("audit", "/audit", auditCtrl.publicData.publicProperties.list.flatMap(propertyToJson("audit", _))) + EntityDescription("audit", "/audit", auditCtrl.publicData.publicProperties.list.flatMap(propertyToJson("audit", _))), + EntityDescription( + "caseTemplate", + "/caseTemplate", + caseTemplateCtrl.publicData.publicProperties.list.flatMap(propertyToJson("caseTemplate", _)) + ), + EntityDescription("customField", "/customField", customFieldCtrl.publicData.publicProperties.list.flatMap(propertyToJson("customField", _))), + EntityDescription( + "observableType", + "/observableType", + observableTypeCtrl.publicData.publicProperties.list.flatMap(propertyToJson("observableType", _)) + ), + EntityDescription( + "organisation", + "/organisation", + organisationCtrl.publicData.publicProperties.list.flatMap(propertyToJson("organisation", _)) + ), + EntityDescription("profile", "/profile", profileCtrl.publicData.publicProperties.list.flatMap(propertyToJson("profile", _))), + EntityDescription("dashboard", "/dashboard", dashboardCtrl.publicData.publicProperties.list.flatMap(propertyToJson("dashboard", _))), + EntityDescription("page", "/page", pageCtrl.publicData.publicProperties.list.flatMap(propertyToJson("page", _))) ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") } diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index 24c63afeea..8db4d8e430 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -26,13 +26,20 @@ import scala.util.{Failure, Success, Try} class DescribeCtrl @Inject() ( cacheApi: SyncCacheApi, entrypoint: Entrypoint, - caseCtrl: CaseCtrl, - taskCtrl: TaskCtrl, alertCtrl: AlertCtrl, + auditCtrl: AuditCtrl, + caseCtrl: CaseCtrl, + caseTemplateCtrl: CaseTemplateCtrl, + customFieldCtrl: CustomFieldCtrl, +// dashboardCtrl: DashboardCtrl, + logCtrl: LogCtrl, observableCtrl: ObservableCtrl, + observableTypeCtrl: ObservableTypeCtrl, + organisationCtrl: OrganisationCtrl, +// pageCtrl: PageCtrl, + profileCtrl: ProfileCtrl, + taskCtrl: TaskCtrl, userCtrl: UserCtrl, -// logCtrl: LogCtrl, - auditCtrl: AuditCtrl, customFieldSrv: CustomFieldSrv, impactStatusSrv: ImpactStatusSrv, resolutionStatusSrv: ResolutionStatusSrv, @@ -42,11 +49,17 @@ class DescribeCtrl @Inject() ( ) { case class PropertyDescription(name: String, `type`: String, values: Seq[JsValue] = Nil, labels: Seq[String] = Nil) + val metadata = Seq( + PropertyDescription("_createdBy", "user"), + PropertyDescription("_createdAt", "date"), + PropertyDescription("_updatedBy", "user"), + PropertyDescription("_updatedAt", "date") + ) case class EntityDescription(label: String, attributes: Seq[PropertyDescription]) { def toJson: JsObject = Json.obj( "label" -> label, - "attributes" -> attributes + "attributes" -> (attributes ++ metadata) ) } @@ -81,8 +94,15 @@ class DescribeCtrl @Inject() ( EntityDescription("alert", alertCtrl.publicProperties.list.flatMap(propertyToJson("alert", _))), EntityDescription("case_artifact", observableCtrl.publicProperties.list.flatMap(propertyToJson("case_artifact", _))), EntityDescription("user", userCtrl.publicProperties.list.flatMap(propertyToJson("user", _))), - // EntityDescription("case_task_log", logCtrl.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), - EntityDescription("audit", auditCtrl.publicProperties.list.flatMap(propertyToJson("audit", _))) + EntityDescription("case_task_log", logCtrl.publicProperties.list.flatMap(propertyToJson("case_task_log", _))), + EntityDescription("audit", auditCtrl.publicProperties.list.flatMap(propertyToJson("audit", _))), + EntityDescription("caseTemplate", caseTemplateCtrl.publicProperties.list.flatMap(propertyToJson("caseTemplate", _))), + EntityDescription("customField", customFieldCtrl.publicProperties.list.flatMap(propertyToJson("customField", _))), + EntityDescription("observableType", observableTypeCtrl.publicProperties.list.flatMap(propertyToJson("observableType", _))), + EntityDescription("organisation", organisationCtrl.publicProperties.list.flatMap(propertyToJson("organisation", _))), + EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))) +// EntityDescription("dashboard", dashboardCtrl.publicProperties.list.flatMap(propertyToJson("dashboard", _))), +// EntityDescription("page", pageCtrl.publicProperties.list.flatMap(propertyToJson("page", _))) ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") } From 1a063620a63c50fe9ed1d013d3b4985433b1d135 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Wed, 4 Nov 2020 15:43:03 +0100 Subject: [PATCH 122/237] #1588 Fixed customField integer from double --- thehive/app/org/thp/thehive/models/CustomField.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/thehive/app/org/thp/thehive/models/CustomField.scala b/thehive/app/org/thp/thehive/models/CustomField.scala index 00900c32b0..e059a5bef0 100644 --- a/thehive/app/org/thp/thehive/models/CustomField.scala +++ b/thehive/app/org/thp/thehive/models/CustomField.scala @@ -147,6 +147,7 @@ object CustomFieldInteger extends CustomFieldType[Int] { override def setValue[C <: CustomFieldValue[C]](customFieldValue: C, value: Option[Any]): Try[C] = value.getOrElse(JsNull) match { case v: Int => Success(customFieldValue.integerValue = Some(v)) + case v: Double => Success(customFieldValue.integerValue = Some(v.toInt)) case JsNumber(n) => Success(customFieldValue.integerValue = Some(n.toInt)) case JsNull | null => Success(customFieldValue.integerValue = None) case obj: JsObject => From b3ef85716fa390bc73c51ab21ef87f3f6cfd2819 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 4 Nov 2020 18:31:13 +0100 Subject: [PATCH 123/237] #1579 Add support to the new ui settings that defines the default filter of the alert similar cases section --- .../alert/AlertSimilarCaseListCmp.js | 36 ++++-- .../organisation/OrgConfigListCmp.js | 6 +- .../app/scripts/services/api/AlertingSrv.js | 120 +++++++++++++++++- .../components/alert/similarity/toolbar.html | 4 + .../app/views/components/org/config.list.html | 6 +- 5 files changed, 154 insertions(+), 18 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js index 8f076e9659..c557b16bca 100644 --- a/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js +++ b/frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js @@ -3,7 +3,7 @@ angular.module('theHiveComponents') .component('alertSimilarCaseList', { - controller: function($scope, FilteringSrv, PaginatedQuerySrv, CaseResolutionStatus, UiSettingsSrv) { + controller: function($scope, AlertingSrv, FilteringSrv, PaginatedQuerySrv, CaseResolutionStatus, UiSettingsSrv) { var self = this; self.CaseResolutionStatus = CaseResolutionStatus; @@ -49,19 +49,12 @@ self.filtering.initContext('alert.dialog.similar-cases') .then(function() { - var defaultFilter = { - field: 'status', - type: 'enumeration', - value: { - list: [{ - text: 'Open', - label: 'Open' - }] - } - }; - - if(_.isEmpty(self.filtering.context.filters)) { - self.filtering.addFilter(defaultFilter); + var defaultFilter = AlertingSrv.getSimilarityFilter(self.state.defaultAlertSimilarCaseFilter); + + if(_.isEmpty(self.filtering.context.filters) && defaultFilter && defaultFilter.length > 0) { + _.each(defaultFilter, function(item) { + self.filtering.addFilter(item); + }); } self.load(); @@ -173,6 +166,21 @@ }); }; + this.applyDefaultFilter = function() { + self.filtering.clearFilters() + .then(function(){ + var defaultFilter = AlertingSrv.getSimilarityFilter(self.state.defaultAlertSimilarCaseFilter); + + if(defaultFilter && defaultFilter.length > 0) { + _.each(defaultFilter, function(item) { + self.filtering.addFilter(item); + }); + + self.search(); + } + }); + }; + this.filterSimilarities = function(data) { return data; }; diff --git a/frontend/app/scripts/components/organisation/OrgConfigListCmp.js b/frontend/app/scripts/components/organisation/OrgConfigListCmp.js index 143bc820ac..f96da1fade 100644 --- a/frontend/app/scripts/components/organisation/OrgConfigListCmp.js +++ b/frontend/app/scripts/components/organisation/OrgConfigListCmp.js @@ -3,9 +3,11 @@ angular.module('theHiveComponents') .component('orgConfigList', { - controller: function($scope, $q, NotificationSrv, UiSettingsSrv) { + controller: function($scope, $q, NotificationSrv, AlertingSrv, UiSettingsSrv) { var self = this; + self.alertSimilarityFilters = []; + self.isDirtySetting = function(key, newValue) { return newValue !== self.currentSettings[key]; }; @@ -62,6 +64,8 @@ self.$onInit = function() { self.loadSettings(this.uiConfig); + + self.alertSimilarityFilters = AlertingSrv.getSimilarityFilters(); }; }, controllerAs: '$ctrl', diff --git a/frontend/app/scripts/services/api/AlertingSrv.js b/frontend/app/scripts/services/api/AlertingSrv.js index f661565cc1..954644aec3 100644 --- a/frontend/app/scripts/services/api/AlertingSrv.js +++ b/frontend/app/scripts/services/api/AlertingSrv.js @@ -5,8 +5,126 @@ var baseUrl = './api/alert'; - var factory = { + var similarityFilters = { + 'open-cases': { + label: 'Open Cases', + filters: [{ + field: 'status', + type: 'enumeration', + value: { + list: [{ + text: 'Open', + label: 'Open' + }] + } + }] + }, + 'open-cases-last-7days': { + label: 'Open Cases in the last 7 days', + filters: [{ + field: 'status', + type: 'enumeration', + value: { + list: [{ + text: 'Open', + label: 'Open' + }] + } + }, { + field: '_createdAt', + type: 'date', + value: { + operator: 'last7days', + from: null, + to: null + } + }] + }, + 'open-cases-last-30days': { + label: 'Open Cases in the last 30 days', + filters: [{ + field: 'status', + type: 'enumeration', + value: { + list: [{ + text: 'Open', + label: 'Open' + }] + } + }, { + field: '_createdAt', + type: 'date', + value: { + operator: 'last30days', + from: null, + to: null + } + }] + }, + 'open-cases-last-3months': { + label: 'Open Cases in the last 3 months', + filters: [{ + field: 'status', + type: 'enumeration', + value: { + list: [{ + text: 'Open', + label: 'Open' + }] + } + }, { + field: '_createdAt', + type: 'date', + value: { + operator: 'last3months', + from: null, + to: null + } + }] + }, + 'open-cases-last-year': { + label: 'Open Cases in the last year', + filters: [{ + field: 'status', + type: 'enumeration', + value: { + list: [{ + text: 'Open', + label: 'Open' + }] + } + }, { + field: '_createdAt', + type: 'date', + value: { + operator: 'lastyear', + from: null, + to: null + } + }] + }, + 'resolved-cases': { + label: 'Resolved cases', + filters: [{ + field: 'status', + type: 'enumeration', + value: { + list: [{ + text: 'Resolved', + label: 'Resolved' + }] + } + }] + } + }; + var factory = { + getSimilarityFilters: function() { + return similarityFilters; + }, + getSimilarityFilter: function(name) { + return (similarityFilters[name] || {}).filters; + }, list: function(config, callback) { return new PaginatedQuerySrv({ name: 'alerts', diff --git a/frontend/app/views/components/alert/similarity/toolbar.html b/frontend/app/views/components/alert/similarity/toolbar.html index 6f10e21bbb..de0bc3b8d1 100644 --- a/frontend/app/views/components/alert/similarity/toolbar.html +++ b/frontend/app/views/components/alert/similarity/toolbar.html @@ -9,6 +9,10 @@
        +
      • + Default filter +
        • +
      • Open cases
        • diff --git a/frontend/app/views/components/org/config.list.html b/frontend/app/views/components/org/config.list.html index b083028c9d..7bcbfe2df3 100644 --- a/frontend/app/views/components/org/config.list.html +++ b/frontend/app/views/components/org/config.list.html @@ -25,9 +25,11 @@
      - +
      -
      From 080c6bbf6c8bbdafae1e35f6f7e4c945ccee2946 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 4 Nov 2020 18:35:55 +0100 Subject: [PATCH 124/237] #1599 Add case metadata properties for alert similarity --- .../org/thp/thehive/controllers/v1/AlertCtrl.scala | 7 ++++--- .../org/thp/thehive/controllers/v1/Properties.scala | 9 +++++++++ .../thehive/controllers/v1/TheHiveQueryExecutor.scala | 11 ++--------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index d4ca260509..4c1e5afc18 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -53,9 +53,10 @@ class AlertCtrl @Inject() ( _.richAlertWithCustomRenderer(alertStatsRenderer(range.extraData)(authContext)) ) ) - override val outputQuery: Query = Query.output[RichAlert, Traversal.V[Alert]](_.richAlert) + override val outputQuery: Query = Query.output[RichAlert, Traversal.V[Alert]](_.richAlert) + val caseProperties: PublicProperties = properties.`case` ++ properties.metaProperties val caseFilterParser: FieldsParser[Option[InputQuery[Traversal.Unk, Traversal.Unk]]] = - FilterQuery.default(db, properties.`case`).paramParser(ru.typeOf[Traversal.V[Case]]).optional.on("caseFilter") + FilterQuery.default(db, caseProperties).paramParser(ru.typeOf[Traversal.V[Case]]).optional.on("caseFilter") override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Alert], Traversal.V[Observable]]("observables", (alertSteps, _) => alertSteps.observables), Query[Traversal.V[Alert], Traversal.V[Case]]("case", (alertSteps, _) => alertSteps.`case`), @@ -68,7 +69,7 @@ class AlertCtrl @Inject() ( caseFilterParser, { (maybeCaseFilterQuery, alertSteps, authContext) => val maybeCaseFilter: Option[Traversal.V[Case] => Traversal.V[Case]] = - maybeCaseFilterQuery.map(f => cases => f(db, properties.`case`, ru.typeOf[Traversal.V[Case]], cases.cast, authContext).cast) + maybeCaseFilterQuery.map(f => cases => f(db, caseProperties, ru.typeOf[Traversal.V[Case]], cases.cast, authContext).cast) alertSteps.similarCases(maybeCaseFilter)(authContext).domainMap(Json.toJson(_)) } ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 5151fb2b4e..a5398a2b62 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -37,6 +37,15 @@ class Properties @Inject() ( @Named("with-thehive-schema") db: Database ) { + lazy val metaProperties: PublicProperties = + PublicPropertyListBuilder + .forType[Product](_ => true) + .property("_createdBy", UMapping.string)(_.field.readonly) + .property("_createdAt", UMapping.date)(_.field.readonly) + .property("_updatedBy", UMapping.string.optional)(_.field.readonly) + .property("_updatedAt", UMapping.date.optional)(_.field.readonly) + .build + lazy val alert: PublicProperties = PublicPropertyListBuilder[Alert] .property("type", UMapping.string)(_.field.updatable) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index b22475eb47..2ee8a82cf2 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -33,6 +33,7 @@ class TheHiveQueryExecutor @Inject() ( taskCtrl: TaskCtrl, userCtrl: UserCtrl, // dashboardCtrl: DashboardCtrl, + properties: Properties, @Named("with-thehive-schema") implicit val db: Database ) extends QueryExecutor { @@ -57,15 +58,7 @@ class TheHiveQueryExecutor @Inject() ( override val version: (Int, Int) = 1 -> 1 - def metaProperties: PublicProperties = - PublicPropertyListBuilder[Product] - .property("_createdBy", UMapping.string)(_.field.readonly) - .property("_createdAt", UMapping.date)(_.field.readonly) - .property("_updatedBy", UMapping.string.optional)(_.field.readonly) - .property("_updatedAt", UMapping.date.optional)(_.field.readonly) - .build - - override lazy val publicProperties: PublicProperties = controllers.foldLeft(metaProperties)(_ ++ _.publicProperties) + override lazy val publicProperties: PublicProperties = controllers.foldLeft(properties.metaProperties)(_ ++ _.publicProperties) override lazy val queries: Seq[ParamQuery[_]] = controllers.map(_.initialQuery) ++ From efc2133e1d852aad6512558f45685e0a1a1c4cc7 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 4 Nov 2020 19:03:01 +0100 Subject: [PATCH 125/237] #1612 Fix the default filter used in my tasks page --- .../app/scripts/controllers/MainPageCtrl.js | 22 ++++++++++++------- frontend/app/scripts/controllers/RootCtrl.js | 13 ++++++++++- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/frontend/app/scripts/controllers/MainPageCtrl.js b/frontend/app/scripts/controllers/MainPageCtrl.js index 9d293eafde..116f9e1579 100644 --- a/frontend/app/scripts/controllers/MainPageCtrl.js +++ b/frontend/app/scripts/controllers/MainPageCtrl.js @@ -9,17 +9,23 @@ self.view = {}; self.defaultFilter = { - '_ne': { - '_field': 'status', - '_value': 'Completed' - } + _or: [ + { + _field: 'status', + _value: 'Waiting' + }, + { + _field: 'status', + _value: 'InProgress' + } + ] }; self.queryOperations = view === 'mytasks' ? [ - {"_name": "currentUser"}, - {"_name": "tasks"} + {_name: 'currentUser'}, + {_name: 'tasks'} ] : [ - {"_name": "waitingTask"} + {_name: 'waitingTask'} ]; if ($stateParams.viewId === 'mytasks') { @@ -61,7 +67,7 @@ loadAll: false, pageSize: self.filtering.context.pageSize, filter: self.filtering.buildQuery(), - baseFilter: self.defaultFilter, + baseFilter: view === 'mytasks' ? self.defaultFilter : [], operations: self.queryOperations, extraData: ['case'], name: $stateParams.viewId diff --git a/frontend/app/scripts/controllers/RootCtrl.js b/frontend/app/scripts/controllers/RootCtrl.js index 93057a7b06..26e6982dc3 100644 --- a/frontend/app/scripts/controllers/RootCtrl.js +++ b/frontend/app/scripts/controllers/RootCtrl.js @@ -57,7 +57,18 @@ angular.module('theHiveControllers').controller('RootCtrl', StreamQuerySrv('v1', [ {_name: 'currentUser'}, {_name: 'tasks'}, - {_name: 'filter', _ne: {_field: 'status', _value: 'Completed'}}, + {_name: 'filter', + _or: [ + { + _field: 'status', + _value: 'Waiting' + }, + { + _field: 'status', + _value: 'InProgress' + } + ] + }, {_name: 'count'} ], { scope: $scope, From b38f54c6ca09712a475b4d4df5431e111ee54bcf Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 4 Nov 2020 19:10:17 +0100 Subject: [PATCH 126/237] #1547 Display the task log date --- frontend/app/views/directives/log-entry.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/app/views/directives/log-entry.html b/frontend/app/views/directives/log-entry.html index 7c4852d7b4..0b52b59f9e 100644 --- a/frontend/app/views/directives/log-entry.html +++ b/frontend/app/views/directives/log-entry.html @@ -30,7 +30,10 @@ Delete - + + + {{log.date | shortDate}} +
      From adb032a861817c4ec167d7136f4c6d39154a07ba Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 5 Nov 2020 14:25:56 +0100 Subject: [PATCH 127/237] #1619 Fix cyclic reference on EventFilter reads --- .../notification/triggers/FilteredEvent.scala | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/notification/triggers/FilteredEvent.scala b/thehive/app/org/thp/thehive/services/notification/triggers/FilteredEvent.scala index d7ff6db5d7..7328a31e5f 100644 --- a/thehive/app/org/thp/thehive/services/notification/triggers/FilteredEvent.scala +++ b/thehive/app/org/thp/thehive/services/notification/triggers/FilteredEvent.scala @@ -68,12 +68,13 @@ case class ContainsEventFilter(field: String, value: String) extends EventFilter case class LikeEventFilter(field: String, value: String) extends EventFilter { lazy val s: Boolean = value.headOption.contains('*') lazy val e: Boolean = value.lastOption.contains('*') - override def apply(event: JsObject): Boolean = getField[String](event, field).fold(false) { - case v if s && e => v.contains(value.tail.dropRight(1)) - case v if s => v.endsWith(value) - case v if e => v.startsWith(value) - case v => v == value - } + override def apply(event: JsObject): Boolean = + getField[String](event, field).fold(false) { + case v if s && e => v.contains(value.tail.dropRight(1)) + case v if s => v.endsWith(value) + case v if e => v.startsWith(value) + case v => v == value + } } @@ -111,9 +112,9 @@ object EventFilter { implicit lazy val reads: Reads[EventFilter] = (JsPath \ "_any").read[JsValue].map(_ => AnyEventFilter.asInstanceOf[EventFilter]) orElse - (JsPath \ "_and").read[Seq[EventFilter]].map(AndEventFilter) orElse - (JsPath \ "_or").read[Seq[EventFilter]].map(OrEventFilter) orElse - (JsPath \ "_not").read[EventFilter](reads).map(NotEventFilter) orElse + (JsPath \ "_and").lazyRead[Seq[EventFilter]](Reads.seq(reads)).map(AndEventFilter) orElse + (JsPath \ "_or").lazyRead[Seq[EventFilter]](Reads.seq(reads)).map(OrEventFilter) orElse + (JsPath \ "_not").lazyRead[EventFilter](reads).map(NotEventFilter) orElse (JsPath \ "_lt").read[(String, BigDecimal)].map(fv => LtEventFilter(fv._1, fv._2)) orElse (JsPath \ "_gt").read[(String, BigDecimal)].map(fv => GtEventFilter(fv._1, fv._2)) orElse (JsPath \ "_lte").read[(String, BigDecimal)].map(fv => LteEventFilter(fv._1, fv._2)) orElse @@ -140,45 +141,43 @@ class FilteredEvent(eventFilter: EventFilter) extends Trigger { override val name: String = "FilteredEvent" override def preFilter(audit: Audit with Entity, context: Option[Entity], organisation: Organisation with Entity): Boolean = - try { - eventFilter( - Json.obj( - "requestId" -> audit.requestId, - "action" -> audit.action, - "mainAction" -> audit.mainAction, - "objectId" -> audit.objectId, - "objectType" -> audit.objectType, - "details" -> audit.details, - "_createdBy" -> audit._createdBy, - "_updatedBy" -> audit._updatedBy, - "_createdAt" -> audit._createdAt, - "_updatedAt" -> audit._updatedAt - ) + try eventFilter( + Json.obj( + "requestId" -> audit.requestId, + "action" -> audit.action, + "mainAction" -> audit.mainAction, + "objectId" -> audit.objectId, + "objectType" -> audit.objectType, + "details" -> audit.details, + "_createdBy" -> audit._createdBy, + "_updatedBy" -> audit._updatedBy, + "_createdAt" -> audit._createdAt, + "_updatedAt" -> audit._updatedAt ) - } catch { + ) + catch { case EventFilterOnMissingUser => true } override def filter(audit: Audit with Entity, context: Option[Entity], organisation: Organisation with Entity, user: Option[User with Entity])( implicit graph: Graph ): Boolean = - try { - super.filter(audit, context, organisation, user) && eventFilter( - Json.obj( - "requestId" -> audit.requestId, - "action" -> audit.action, - "mainAction" -> audit.mainAction, - "objectId" -> audit.objectId, - "objectType" -> audit.objectType, - "details" -> audit.details, - "_createdBy" -> audit._createdBy, - "_updatedBy" -> audit._updatedBy, - "_createdAt" -> audit._createdAt, - "_updatedAt" -> audit._updatedAt, - "user" -> user.map(_.login) - ) + try super.filter(audit, context, organisation, user) && eventFilter( + Json.obj( + "requestId" -> audit.requestId, + "action" -> audit.action, + "mainAction" -> audit.mainAction, + "objectId" -> audit.objectId, + "objectType" -> audit.objectType, + "details" -> audit.details, + "_createdBy" -> audit._createdBy, + "_updatedBy" -> audit._updatedBy, + "_createdAt" -> audit._createdAt, + "_updatedAt" -> audit._updatedAt, + "user" -> user.map(_.login) ) - } catch { + ) + catch { case EventFilterOnMissingUser => false } } From 58fb2d38a3e5fc3e74060ae651c73ad31d2bfde1 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 5 Nov 2020 14:27:41 +0100 Subject: [PATCH 128/237] #1619 Add the trigger "CaseShared" and the responders "RunAnalyzer" and "RunResponder" --- .../connector/cortex/CortexModule.scala | 6 ++ .../cortex/services/AnalyzerSrv.scala | 24 ++++- .../cortex/services/ResponderSrv.scala | 12 ++- .../notification/notifiers/RunAnalyzer.scala | 83 +++++++++++++++ .../notification/notifiers/RunResponder.scala | 100 ++++++++++++++++++ .../app/org/thp/thehive/TheHiveModule.scala | 1 + .../notification/NotificationActor.scala | 4 +- .../notification/triggers/CaseShared.scala | 24 +++++ 8 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunAnalyzer.scala create mode 100644 cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunResponder.scala create mode 100644 thehive/app/org/thp/thehive/services/notification/triggers/CaseShared.scala diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/CortexModule.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/CortexModule.scala index 7366bfcb1e..ccbdf83079 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/CortexModule.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/CortexModule.scala @@ -6,7 +6,9 @@ import org.thp.scalligraph.models.{Database, Schema} import org.thp.scalligraph.query.QueryExecutor import org.thp.thehive.connector.cortex.controllers.v0.{CortexQueryExecutor => CortexQueryExecutorV0} import org.thp.thehive.connector.cortex.models.{CortexSchemaDefinition, DatabaseProvider} +import org.thp.thehive.connector.cortex.services.notification.notifiers.{RunAnalyzerProvider, RunResponderProvider} import org.thp.thehive.connector.cortex.services.{Connector, CortexActor} +import org.thp.thehive.services.notification.notifiers.NotifierProvider import org.thp.thehive.services.{Connector => TheHiveConnector} import play.api.libs.concurrent.AkkaGuiceSupport import play.api.routing.{Router => PlayRouter} @@ -25,6 +27,10 @@ class CortexModule(environment: Environment, configuration: Configuration) exten val schemaBindings = ScalaMultibinder.newSetBinder[Schema](binder) schemaBindings.addBinding.to[CortexSchemaDefinition] + val notifierBindings = ScalaMultibinder.newSetBinder[NotifierProvider](binder) + notifierBindings.addBinding.to[RunResponderProvider] + notifierBindings.addBinding.to[RunAnalyzerProvider] + bind[Database].annotatedWithName("with-thehive-cortex-schema").toProvider[DatabaseProvider] bindActor[CortexActor]("cortex-actor") () diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerSrv.scala index a319024ef7..f4eb6ebf74 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerSrv.scala @@ -2,9 +2,11 @@ package org.thp.thehive.connector.cortex.services import javax.inject.{Inject, Singleton} import org.thp.cortex.dto.v0.{OutputWorker => CortexWorker} -import org.thp.scalligraph.NotFoundError +import org.thp.scalligraph.{EntityIdOrName, NotFoundError} import org.thp.scalligraph.auth.AuthContext +import org.thp.cortex.dto.v0.OutputWorker import play.api.Logger +import play.api.libs.json.{JsObject, Json} import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} @@ -62,4 +64,24 @@ class AnalyzerSrv @Inject() (connector: Connector, serviceHelper: ServiceHelper, .headOption .fold[Future[(CortexWorker, Seq[String])]](Future.failed(NotFoundError(s"Analyzer $id not found")))(Future.successful) } + + def getAnalyzerByName(analyzerName: String, organisation: EntityIdOrName): Future[Map[CortexWorker, Seq[String]]] = + searchAnalyzers(Json.obj("query" -> Json.obj("_field" -> "name", "_value" -> analyzerName)), organisation) + + def searchAnalyzers(query: JsObject)(implicit authContext: AuthContext): Future[Map[OutputWorker, Seq[String]]] = + searchAnalyzers(query, authContext.organisation) + + def searchAnalyzers(query: JsObject, organisation: EntityIdOrName): Future[Map[OutputWorker, Seq[String]]] = + Future + .traverse(serviceHelper.availableCortexClients(connector.clients, organisation)) { client => + client + .searchResponders(query) + .transform { + case Success(analyzers) => Success(analyzers.map(_ -> client.name)) + case Failure(error) => + logger.error(s"List Cortex analyzers fails on ${client.name}", error) + Success(Nil) + } + } + .map(serviceHelper.flattenList) } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala index e350a2c8f6..18ded91397 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ResponderSrv.scala @@ -9,7 +9,7 @@ import org.thp.scalligraph.models.Database import org.thp.thehive.controllers.v0.Conversion.toObjectType import org.thp.thehive.models.Permissions import play.api.Logger -import play.api.libs.json.JsObject +import play.api.libs.json.{JsObject, Json} import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} @@ -56,15 +56,21 @@ class ResponderSrv @Inject() ( ) } yield serviceHelper.flattenList(responders).filter { case (w, _) => w.maxTlp >= tlp && w.maxPap >= pap } + def getRespondersByName(responderName: String, organisation: EntityIdOrName): Future[Map[OutputWorker, Seq[String]]] = + searchResponders(Json.obj("query" -> Json.obj("_field" -> "name", "_value" -> responderName)), organisation) + /** - * Search responders, not used as of 08/19 + * Search responders * @param query the raw query from frontend * @param authContext auth context for organisation filter * @return */ def searchResponders(query: JsObject)(implicit authContext: AuthContext): Future[Map[OutputWorker, Seq[String]]] = + searchResponders(query, authContext.organisation) + + def searchResponders(query: JsObject, organisation: EntityIdOrName): Future[Map[OutputWorker, Seq[String]]] = Future - .traverse(serviceHelper.availableCortexClients(connector.clients, authContext.organisation)) { client => + .traverse(serviceHelper.availableCortexClients(connector.clients, organisation)) { client => client .searchResponders(query) .transform { diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunAnalyzer.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunAnalyzer.scala new file mode 100644 index 0000000000..c292c513a9 --- /dev/null +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunAnalyzer.scala @@ -0,0 +1,83 @@ +package org.thp.thehive.connector.cortex.services.notification.notifiers + +import javax.inject.{Inject, Singleton} +import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.models.Entity +import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{BadConfigurationError, NotFoundError, RichOption} +import org.thp.thehive.connector.cortex.services.{AnalyzerSrv, JobSrv} +import org.thp.thehive.controllers.v0.AuditRenderer +import org.thp.thehive.models._ +import org.thp.thehive.services.ObservableOps._ +import org.thp.thehive.services._ +import org.thp.thehive.services.notification.notifiers.{Notifier, NotifierProvider} +import play.api.Configuration + +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Try} + +@Singleton +class RunAnalyzerProvider @Inject() ( + analyzerSrv: AnalyzerSrv, + jobSrv: JobSrv, + caseSrv: CaseSrv, + observableSrv: ObservableSrv, + ec: ExecutionContext +) extends NotifierProvider { + override val name: String = "RunAnalyzer" + + override def apply(config: Configuration): Try[Notifier] = + config.getOrFail[String]("analyzerName").map { responderName => + new RunAnalyzer( + responderName, + analyzerSrv, + jobSrv, + caseSrv, + observableSrv, + ec + ) + } +} + +class RunAnalyzer( + analyzerName: String, + analyzerSrv: AnalyzerSrv, + jobSrv: JobSrv, + caseSrv: CaseSrv, + observableSrv: ObservableSrv, + implicit val ec: ExecutionContext +) extends Notifier + with AuditRenderer { + override val name: String = "RunAnalyzer" + + def getObservable(`object`: Option[Entity])(implicit graph: Graph): Future[RichObservable] = + `object` match { + case Some(o) if o._label == "Observable" => Future.fromTry(observableSrv.get(o._id).richObservable.getOrFail("Observable")) + case _ => Future.failed(NotFoundError("Audit object is not an observable")) + } + + def getCase(context: Option[Entity])(implicit graph: Graph): Future[Case with Entity] = + context match { + case Some(c) if c._label == "Case" => Future.fromTry(caseSrv.getOrFail(c._id)) + case _ => Future.failed(NotFoundError("Audit context is not a case")) + } + + override def execute( + audit: Audit with Entity, + context: Option[Entity], + `object`: Option[Entity], + organisation: Organisation with Entity, + user: Option[User with Entity] + )(implicit graph: Graph): Future[Unit] = + if (user.isDefined) + Future.failed(BadConfigurationError("The notification runAnalyzer must not be applied on user")) + else + for { + observable <- getObservable(`object`) + case0 <- getCase(context) + workers <- analyzerSrv.getAnalyzerByName(analyzerName, organisation._id) + (worker, cortexIds) <- Future.fromTry(workers.headOption.toTry(Failure(NotFoundError(s"Analyzer $analyzerName not found")))) + authContext = LocalUserSrv.getSystemAuthContext.changeOrganisation(organisation._id, Permissions.all) + _ <- jobSrv.submit(cortexIds.head, worker.id, observable, case0)(authContext) + } yield () +} diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunResponder.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunResponder.scala new file mode 100644 index 0000000000..8d50288980 --- /dev/null +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunResponder.scala @@ -0,0 +1,100 @@ +package org.thp.thehive.connector.cortex.services.notification.notifiers + +import com.typesafe.config.ConfigRenderOptions +import javax.inject.{Inject, Singleton} +import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.models.Entity +import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.scalligraph.{BadConfigurationError, NotFoundError, RichOption} +import org.thp.thehive.connector.cortex.services.{ActionSrv, ResponderSrv} +import org.thp.thehive.controllers.v0.AuditRenderer +import org.thp.thehive.models.{Audit, Organisation, Permissions, User} +import org.thp.thehive.services._ +import org.thp.thehive.services.notification.notifiers.{Notifier, NotifierProvider} +import play.api.Configuration +import play.api.libs.json.{JsObject, Json, OWrites} + +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Try} + +@Singleton +class RunResponderProvider @Inject() ( + responderSrv: ResponderSrv, + actionSrv: ActionSrv, + taskSrv: TaskSrv, + caseSrv: CaseSrv, + observableSrv: ObservableSrv, + logSrv: LogSrv, + alertSrv: AlertSrv, + ec: ExecutionContext +) extends NotifierProvider { + override val name: String = "RunResponder" + + override def apply(config: Configuration): Try[Notifier] = { + + val parameters = Try(Json.parse(config.underlying.getValue("parameters").render(ConfigRenderOptions.concise())).as[JsObject]).toOption + config.getOrFail[String]("responderName").map { responderName => + new RunResponder( + responderName, + parameters.getOrElse(JsObject.empty), + responderSrv, + actionSrv, + taskSrv, + caseSrv, + observableSrv, + logSrv, + alertSrv, + ec + ) + } + } +} + +class RunResponder( + responderName: String, + parameters: JsObject, + responderSrv: ResponderSrv, + actionSrv: ActionSrv, + taskSrv: TaskSrv, + caseSrv: CaseSrv, + observableSrv: ObservableSrv, + logSrv: LogSrv, + alertSrv: AlertSrv, + implicit val ec: ExecutionContext +) extends Notifier + with AuditRenderer { + override val name: String = "RunResponder" + + def getEntity(audit: Audit)(implicit graph: Graph): Try[(Product with Entity, JsObject)] = + audit + .objectEntityId + .flatMap { objectId => + audit.objectType.map { + case "Task" => taskSrv.get(objectId).project(_.by.by(taskToJson)).getOrFail("Task") + case "Case" => caseSrv.get(objectId).project(_.by.by(caseToJson)).getOrFail("Case") + case "Observable" => observableSrv.get(objectId).project(_.by.by(observableToJson)).getOrFail("Observable") + case "Log" => logSrv.get(objectId).project(_.by.by(logToJson)).getOrFail("Log") + case "Alert" => alertSrv.get(objectId).project(_.by.by(alertToJson)).getOrFail("Alert") + case objectType => Failure(NotFoundError(s"objectType $objectType is not recognised")) + } + } + .getOrElse(Failure(NotFoundError("Object not present in the audit"))) + + override def execute( + audit: Audit with Entity, + context: Option[Entity], + `object`: Option[Entity], + organisation: Organisation with Entity, + user: Option[User with Entity] + )(implicit graph: Graph): Future[Unit] = + if (user.isDefined) + Future.failed(BadConfigurationError("The notification runResponder must not be applied on user")) + else + for { + (entity, entityJson) <- Future.fromTry(getEntity(audit)) + workers <- responderSrv.getRespondersByName(responderName, organisation._id) + (worker, cortexIds) <- Future.fromTry(workers.headOption.toTry(Failure(NotFoundError(s"Responder $responderName not found")))) + authContext = LocalUserSrv.getSystemAuthContext.changeOrganisation(organisation._id, Permissions.all) + _ <- actionSrv.execute(entity, cortexIds.headOption, worker.id, parameters)(OWrites[Entity](_ => entityJson), authContext) + } yield () +} diff --git a/thehive/app/org/thp/thehive/TheHiveModule.scala b/thehive/app/org/thp/thehive/TheHiveModule.scala index 8f44a0273b..4c752f9926 100644 --- a/thehive/app/org/thp/thehive/TheHiveModule.scala +++ b/thehive/app/org/thp/thehive/TheHiveModule.scala @@ -54,6 +54,7 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte triggerBindings.addBinding.to[JobFinishedProvider] triggerBindings.addBinding.to[LogInMyTaskProvider] triggerBindings.addBinding.to[TaskAssignedProvider] + triggerBindings.addBinding.to[CaseShareProvider] val notifierBindings = ScalaMultibinder.newSetBinder[NotifierProvider](binder) notifierBindings.addBinding.to[AppendToFileProvider] diff --git a/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala b/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala index a49724452e..390ba0fdaa 100644 --- a/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala +++ b/thehive/app/org/thp/thehive/services/notification/NotificationActor.scala @@ -162,7 +162,7 @@ class NotificationActor @Inject() ( .getOrElse(organisation._id, Map.empty) .foreach { case (trigger, (inOrg, userIds)) if trigger.preFilter(audit, context, organisation) => - logger.debug(s"Notification trigger ${trigger.name} is applicable") + logger.debug(s"Notification trigger ${trigger.name} is applicable for $audit") if (userIds.nonEmpty) userSrv .getByIds(userIds: _*) @@ -198,7 +198,7 @@ class NotificationActor @Inject() ( } executeNotification(None, orgConfig, audit, context, obj, organisation) } - case (trigger, _) => logger.debug(s"Notification trigger ${trigger.name} is NOT applicable") + case (trigger, _) => logger.debug(s"Notification trigger ${trigger.name} is NOT applicable for $audit") } case _ => } diff --git a/thehive/app/org/thp/thehive/services/notification/triggers/CaseShared.scala b/thehive/app/org/thp/thehive/services/notification/triggers/CaseShared.scala new file mode 100644 index 0000000000..d439e38554 --- /dev/null +++ b/thehive/app/org/thp/thehive/services/notification/triggers/CaseShared.scala @@ -0,0 +1,24 @@ +package org.thp.thehive.services.notification.triggers + +import javax.inject.{Inject, Singleton} +import org.thp.scalligraph.models.Entity +import org.thp.thehive.models.{Audit, Organisation} +import play.api.Configuration +import play.api.libs.json.Json + +import scala.util.{Success, Try} + +@Singleton +class CaseShareProvider @Inject() extends TriggerProvider { + override val name: String = "CaseShared" + override def apply(config: Configuration): Try[Trigger] = Success(new CaseShared()) +} + +class CaseShared() extends Trigger { + override val name: String = "CaseShared" + + override def preFilter(audit: Audit with Entity, context: Option[Entity], organisation: Organisation with Entity): Boolean = + audit.action == Audit.update && audit + .objectType + .contains("Case") && audit.details.flatMap(d => Try(Json.parse(d)).toOption).exists(d => (d \ "share").isDefined) +} From 53a8bb4584c353e114fc388cec17d7da27ae110f Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 5 Nov 2020 16:52:28 +0100 Subject: [PATCH 129/237] #1611 Fix text containing predicate (don't tokenize) --- ScalliGraph | 2 +- .../thp/thehive/controllers/v0/TagCtrl.scala | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 8e342bb733..b724b88971 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 8e342bb7331457ca18aa7a3d5991c3a905d0fe67 +Subproject commit b724b889717c75855e0cdfdb9df0c5221d64c1f9 diff --git a/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala index dd5f388911..636bd0e277 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala @@ -132,6 +132,22 @@ class PublicTag @Inject() (tagSrv: TagSrv) extends PublicData { .property("predicate", UMapping.string)(_.field.readonly) .property("value", UMapping.string.optional)(_.field.readonly) .property("description", UMapping.string.optional)(_.field.readonly) - .property("text", UMapping.string)(_.select(_.displayName).readonly) + .property("text", UMapping.string)( + _.select(_.displayName) + .filter((_, tags) => + tags + .graphMap[String, String, Converter.Identity[String]]( + { v => + val namespace = UMapping.string.getProperty(v, "namespace") + val predicate = UMapping.string.getProperty(v, "predicate") + val value = UMapping.string.optional.getProperty(v, "value") + Tag(namespace, predicate, value, None, 0).toString + }, + Converter.identity[String] + ) + ) + .converter(_ => Converter.identity[String]) + .readonly + ) .build } From acce93d6ad19f90a5db01ff7280e5ef84c80c441 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 5 Nov 2020 17:09:22 +0100 Subject: [PATCH 130/237] #1617 Fix within predicate filter --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index b724b88971..1d8f5e930e 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit b724b889717c75855e0cdfdb9df0c5221d64c1f9 +Subproject commit 1d8f5e930e59cfc71921e2b6e4ac8f6ba9b14a49 From ac0421b6d140c1ac6d2d84d6103c06815ffa9225 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 6 Nov 2020 08:36:37 +0100 Subject: [PATCH 131/237] #1621 Add properties "viewingOrganisation" and "owningOrganisation" to case --- ScalliGraph | 2 +- .../org/thp/thehive/controllers/v0/CaseCtrl.scala | 6 ++++++ .../org/thp/thehive/services/th3/Aggregation.scala | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 1d8f5e930e..5cf2957a52 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 1d8f5e930e59cfc71921e2b6e4ac8f6ba9b14a49 +Subproject commit 5cf2957a52bdc42e0fc73f95a6751ccec2997ef8 diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index 9cd28662a3..62015e054d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -352,5 +352,11 @@ class PublicCase @Inject() ( ) ).readonly ) + .property("viewingOrganisation", UMapping.string)( + _.authSelect((cases, authContext) => cases.organisations.visible(authContext).value(_.name)).readonly + ) + .property("owningOrganisation", UMapping.string)( + _.authSelect((cases, authContext) => cases.origin.visible(authContext).value(_.name)).readonly + ) .build } diff --git a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala index 0613df53a5..90032f84dc 100644 --- a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala +++ b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala @@ -161,7 +161,7 @@ case class AggSum(aggName: Option[String], fieldName: String) extends Aggregatio traversal.coalesce( t => property - .select(fieldPath, t) + .select(fieldPath, t, authContext) .sum .domainMap(sum => Output(Json.obj(name -> JsNumber(BigDecimal(sum.toString))))) .castDomain[Output[_]], @@ -184,7 +184,7 @@ case class AggAvg(aggName: Option[String], fieldName: String) extends Aggregatio traversal.coalesce( t => property - .select(fieldPath, t) + .select(fieldPath, t, authContext) .mean .domainMap(avg => Output(Json.obj(name -> avg.asInstanceOf[Double]))), Output(Json.obj(name -> JsNull)) @@ -207,7 +207,7 @@ case class AggMin(aggName: Option[String], fieldName: String) extends Aggregatio traversal.coalesce( t => property - .select(fieldPath, t) + .select(fieldPath, t, authContext) .min .domainMap(min => Output(Json.obj(name -> property.mapping.selectRenderer.toJson(min)))), Output(Json.obj(name -> JsNull)) @@ -230,7 +230,7 @@ case class AggMax(aggName: Option[String], fieldName: String) extends Aggregatio traversal.coalesce( t => property - .select(fieldPath, t) + .select(fieldPath, t, authContext) .max .domainMap(max => Output(Json.obj(name -> property.mapping.selectRenderer.toJson(max)))), Output(Json.obj(name -> JsNull)) @@ -275,7 +275,7 @@ case class FieldAggregation( val property = publicProperties .get[Traversal.UnkD, Traversal.UnkDU](fieldPath, traversalType) .getOrElse(throw BadRequestError(s"Property $fieldName for type $traversalType not found")) - val groupedVertices = property.select(fieldPath, traversal.as(label)).group(_.by, _.by(_.select(label).fold)).unfold + val groupedVertices = property.select(fieldPath, traversal.as(label), authContext).group(_.by, _.by(_.select(label).fold)).unfold val sortedAndGroupedVertex = orders .map { case order if order.headOption.contains('-') => order.tail -> Order.desc @@ -377,7 +377,7 @@ case class TimeAggregation( .getOrElse(throw BadRequestError(s"Property $fieldName for type $traversalType not found")) val label = StepLabel[Traversal.UnkD, Traversal.UnkG, Converter[Traversal.UnkD, Traversal.UnkG]] val groupedVertex = property - .select(fieldPath, traversal.as(label)) + .select(fieldPath, traversal.as(label), authContext) .cast[Date, Date] .graphMap[Long, JLong, Converter[Long, JLong]](dateToKey, Converter.long) .group(_.by, _.by(_.select(label).fold)) From 2f7af39bc6750bcf0fcc09e1cbe40f397ec51f6e Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 6 Nov 2020 08:39:03 +0100 Subject: [PATCH 132/237] Add VSCode files in gitignore --- .gitignore | 6 ++++++ ScalliGraph | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a8a3588a45..d78ccf369e 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,9 @@ tmp /.idea/** !/.idea/runConfigurations/ !/.idea/runConfigurations/* + +# VSCode +.vscode/ +.bloop/ +.metals/ +metals.sbt diff --git a/ScalliGraph b/ScalliGraph index 5cf2957a52..e95b44aafa 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 5cf2957a52bdc42e0fc73f95a6751ccec2997ef8 +Subproject commit e95b44aafa9269a723903204f9a1676fbbfab698 From 4c57fec14617cf0266547ed6a0404dd93524808e Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 6 Nov 2020 11:55:31 +0100 Subject: [PATCH 133/237] #1625 Add "ignoreSimilarity" attribute in observable --- .../cortex/services/ActionOperationSrv.scala | 2 +- .../cortex/services/Conversion.scala | 1 + .../org/thp/thehive/dto/v0/Observable.scala | 6 ++- .../org/thp/thehive/dto/v1/Observable.scala | 6 ++- .../thehive/migration/th3/Conversion.scala | 6 +-- .../misp/services/MispImportSrv.scala | 6 +-- .../controllers/v0/ObservableCtrl.scala | 2 +- .../controllers/v0/ObservableRenderer.scala | 2 +- .../controllers/v1/ObservableCtrl.scala | 2 +- .../controllers/v1/ObservableRenderer.scala | 2 +- .../org/thp/thehive/models/Observable.scala | 3 +- .../models/TheHiveSchemaDefinition.scala | 2 + .../org/thp/thehive/services/AlertSrv.scala | 49 +++++++++---------- .../org/thp/thehive/services/CaseSrv.scala | 2 +- .../thp/thehive/services/ObservableSrv.scala | 12 +++-- .../thp/thehive/services/AlertSrvTest.scala | 2 +- .../thp/thehive/services/CaseSrvTest.scala | 2 +- .../thp/thehive/services/DataSrvTest.scala | 2 +- 18 files changed, 61 insertions(+), 48 deletions(-) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala index 69cf1d92fc..2e2722b805 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala @@ -98,7 +98,7 @@ class ActionOperationSrv @Inject() ( c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddArtifactToCase without case")))(Success(_)) obsType <- observableTypeSrv.getOrFail(EntityIdOrName(dataType)) richObservable <- observableSrv.create( - Observable(Some(dataMessage), 2, ioc = false, sighted = false), + Observable(Some(dataMessage), 2, ioc = false, sighted = false, ignoreSimilarity = None), obsType, dataMessage, Set.empty[String], diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala index 1a1608bd07..5f1256fd91 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala @@ -28,6 +28,7 @@ object Conversion { .withFieldComputed(_.tlp, _.tlp) .withFieldConst(_.ioc, false) .withFieldConst(_.sighted, false) + .withFieldConst(_.ignoreSimilarity, None) .transform } diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala index c63f0b01de..369ead4aa9 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala @@ -17,7 +17,8 @@ case class InputObservable( tlp: Option[Int] = None, tags: Set[String] = Set.empty, ioc: Option[Boolean] = None, - sighted: Option[Boolean] = None + sighted: Option[Boolean] = None, + ignoreSimilarity: Option[Boolean] = None ) object InputObservable { @@ -51,7 +52,8 @@ case class OutputObservable( reports: JsObject, stats: JsObject, seen: Option[Boolean], - `case`: Option[OutputCase] + `case`: Option[OutputCase], + ignoreSimilarity: Option[Boolean] ) object OutputObservable { diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala index e94bcf09c8..4211b762f9 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala @@ -17,7 +17,8 @@ case class InputObservable( tlp: Option[Int] = None, tags: Set[String] = Set.empty, ioc: Option[Boolean] = None, - sighted: Option[Boolean] = None + sighted: Option[Boolean] = None, + ignoreSimilarity: Option[Boolean] = None ) object InputObservable { @@ -48,7 +49,8 @@ case class OutputObservable( sighted: Boolean, reports: JsObject, message: Option[String], - extraData: JsObject + extraData: JsObject, + ignoreSimilarity: Option[Boolean] ) object OutputObservable { diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index 880f31d881..77b45faac6 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -107,7 +107,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp, ioc, sighted), + Observable(message, tlp, ioc, sighted, None), Seq(mainOrganisation), dataType, tags, @@ -228,7 +228,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false), + Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false, ignoreSimilarity = None), Nil, dataType, tags, @@ -448,7 +448,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp, ioc, sighted), + Observable(message, tlp, ioc, sighted, ignoreSimilarity = None), Seq(mainOrganisation), dataType, tags, diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala index 1f57a57a3d..95e8d64ecd 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala @@ -110,7 +110,7 @@ class MispImportSrv @Inject() ( ) List( ( - Observable(attribute.comment, 0, ioc = false, sighted = false), + Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Right(attribute.data.get) @@ -122,7 +122,7 @@ class MispImportSrv @Inject() ( ) List( ( - Observable(attribute.comment, 0, ioc = false, sighted = false), + Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Left(attribute.value) @@ -140,7 +140,7 @@ class MispImportSrv @Inject() ( s"attribute ${attribute.category}:${attribute.`type`} (${attribute.tags}) is converted to observable $observableType with tags $additionalTags" ) ( - Observable(attribute.comment, 0, ioc = false, sighted = false), + Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Left(value) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 8278610233..407a2604a0 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -123,7 +123,7 @@ class ObservableCtrl @Inject() ( val observables = observableSrv .get(EntityIdOrName(observableId)) .visible - .similar + .filteredSimilar .visible .richObservableWithCustomRenderer(observableLinkRenderer) .toSeq diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala index 6589aa9d9b..5076cdac72 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala @@ -19,7 +19,7 @@ trait ObservableRenderer { def observableStatsRenderer(implicit authContext: AuthContext ): Traversal.V[Observable] => Traversal[JsObject, JMap[JBoolean, JLong], Converter[JsObject, JMap[JBoolean, JLong]]] = - _.similar + _.filteredSimilar .visible .groupCount(_.byValue(_.ioc)) .domainMap { stats => diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index c4d7fe18d6..910afb8f73 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -62,7 +62,7 @@ class ObservableCtrl @Inject() ( ), Query[Traversal.V[Observable], Traversal.V[Observable]]( "similar", - (observableSteps, authContext) => observableSteps.similar.visible(authContext) + (observableSteps, authContext) => observableSteps.filteredSimilar.visible(authContext) ), Query[Traversal.V[Observable], Traversal.V[Case]]("case", (observableSteps, _) => observableSteps.`case`) ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala index 54b9e3cfd0..8bef007b51 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala @@ -20,7 +20,7 @@ trait ObservableRenderer { def seenStats(implicit authContext: AuthContext ): Traversal.V[Observable] => Traversal[JsValue, JMap[JBoolean, JLong], Converter[JsValue, JMap[JBoolean, JLong]]] = - _.similar + _.filteredSimilar .visible .groupCount(_.byValue(_.ioc)) .domainMap { stats => diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index 31dd2d149a..ae4d2715ce 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -18,7 +18,7 @@ case class ObservableData() case class ObservableTag() @BuildVertexEntity -case class Observable(message: Option[String], tlp: Int, ioc: Boolean, sighted: Boolean) +case class Observable(message: Option[String], tlp: Int, ioc: Boolean, sighted: Boolean, ignoreSimilarity: Option[Boolean]) case class RichObservable( observable: Observable with Entity, @@ -39,6 +39,7 @@ case class RichObservable( def tlp: Int = observable.tlp def ioc: Boolean = observable.ioc def sighted: Boolean = observable.sighted + def ignoreSimilarity: Option[Boolean] = observable.ignoreSimilarity def dataOrAttachment: Either[Data with Entity, Attachment with Entity] = data.toLeft(attachment.get) } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index f994c29280..62683434d6 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -68,10 +68,12 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .noop // .addIndex("Tag", IndexType.unique, "namespace", "predicate", "value") .noop // .addIndex("Audit", IndexType.basic, "requestId", "mainAction") .rebuildIndexes + // release 4.0.0 .updateGraph("Remove cases with a Deleted status", "Case") { traversal => traversal.unsafeHas("status", "Deleted").remove() Success(()) } + .addProperty[Option[Boolean]]("Observable", "ignoreSimilarity") val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 0ac7644654..b4f40a6036 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -271,28 +271,30 @@ class AlertSrv @Inject() ( updatedCase <- mergeInCase(alert, case0) } yield updatedCase - def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = { - auditSrv.mergeAudits { - val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" + def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = + auditSrv + .mergeAudits { + val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" - for { - _ <- markAsRead(alert._id) - _ <- importObservables(alert, `case`) - _ <- importCustomFields(alert, `case`) - _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") - _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) - // No audit for markAsRead and observables - // Audits for customFields, description and tags - c <- caseSrv.getOrFail(`case`._id) - details <- Success(Json.obj( - "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), - "description" -> c.description, - "tags" -> caseSrv.get(`case`).tags.toSeq.map(_.toString)) - ) - } yield details - } (details => auditSrv.alertToCase.merge(alert, `case`, Some(details))) + for { + _ <- markAsRead(alert._id) + _ <- importObservables(alert, `case`) + _ <- importCustomFields(alert, `case`) + _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") + _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) + // No audit for markAsRead and observables + // Audits for customFields, description and tags + c <- caseSrv.getOrFail(`case`._id) + details <- Success( + Json.obj( + "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), + "description" -> c.description, + "tags" -> caseSrv.get(`case`).tags.toSeq.map(_.toString) + ) + ) + } yield details + }(details => auditSrv.alertToCase.merge(alert, `case`, Some(details))) .flatMap(_ => caseSrv.getOrFail(`case`._id)) - } def importObservables(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, @@ -335,10 +337,7 @@ class AlertSrv @Inject() ( .toIterator .toTry { richCustomField => caseSrv - .setOrCreateCustomField(`case`, - richCustomField.customField._id, - richCustomField.value, - richCustomField.customFieldValue.order) + .setOrCreateCustomField(`case`, richCustomField.customField._id, richCustomField.value, richCustomField.customFieldValue.order) } .map(_ => ()) @@ -401,7 +400,7 @@ object AlertOps { authContext: AuthContext ): Traversal[(RichCase, SimilarStats), JMap[String, Any], Converter[(RichCase, SimilarStats), JMap[String, Any]]] = { val similarObservables = observables - .similar + .filteredSimilar .visible maybeCaseFilter .fold(similarObservables)(caseFilter => similarObservables.filter(o => caseFilter(o.`case`))) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 0c0caf292f..8c6a22b303 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -162,7 +162,7 @@ class CaseSrv @Inject() ( ): Try[Unit] = { val alreadyExistInThatCase = observableSrv .get(richObservable.observable) - .similar + .filteredSimilar .visible .`case` .hasId(`case`._id) diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index fa61aa98e0..2a156ec43f 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -274,7 +274,7 @@ object ObservableOps { .by(_.data.fold) .by(_.attachments.fold) .by(_.tags.fold) - .by(_.similar.visible.limit(1).count) + .by(_.filteredSimilar.visible.limit(1).count) .by(_.keyValues.fold) .by(_.reportTags.fold) ) @@ -302,7 +302,7 @@ object ObservableOps { .by(_.data.fold) .by(_.attachments.fold) .by(_.tags.fold) - .by(_.similar.visible.limit(1).count) + .by(_.filteredSimilar.visible.limit(1).count) .by(_.keyValues.fold) .by(_.reportTags.fold) .by(entityRenderer) @@ -333,6 +333,12 @@ object ObservableOps { if (tags.nonEmpty) traversal.outE[ObservableTag].filter(_.otherV.hasId(tags.map(_._id).toSeq: _*)).remove() + def filteredSimilar: Traversal.V[Observable] = + traversal + .hasNot(_.ignoreSimilarity, true) + .similar + .hasNot(_.ignoreSimilarity, true) + def similar: Traversal.V[Observable] = { val originLabel = StepLabel.v[Observable] traversal @@ -341,7 +347,7 @@ object ObservableOps { _.out[ObservableData] .in[ObservableData], _.out[ObservableAttachment] - .in[ObservableAttachment] + .in[ObservableAttachment] // FIXME this doesn't work. Link must be done with attachmentId ) .where(JP.without(originLabel.name)) .dedup diff --git a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala index f63bc0e3e6..3af6601d61 100644 --- a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala @@ -103,7 +103,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { for { observableType <- app[ObservableTypeSrv].getOrFail(EntityName("domain")) observable <- app[ObservableSrv].create( - observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true), + observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None), `type` = observableType, dataValue = "perdu.com", tagNames = Set("tag10"), diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index 75a105d0b9..de36479d95 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -279,7 +279,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { val newObs = app[Database].tryTransaction { implicit graph => app[ObservableSrv].create( - Observable(Some("if you feel lost"), 1, ioc = false, sighted = true), + Observable(Some("if you feel lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None), app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Case").get, "lost.com", Set[String](), diff --git a/thehive/test/org/thp/thehive/services/DataSrvTest.scala b/thehive/test/org/thp/thehive/services/DataSrvTest.scala index 054b578e89..922926dd91 100644 --- a/thehive/test/org/thp/thehive/services/DataSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/DataSrvTest.scala @@ -22,7 +22,7 @@ class DataSrvTest extends PlaySpecification with TestAppBuilder { "get related observables" in testApp { app => app[Database].tryTransaction { implicit graph => app[ObservableSrv].create( - Observable(Some("love"), 1, ioc = false, sighted = true), + Observable(Some("love"), 1, ioc = false, sighted = true, ignoreSimilarity = None), app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Observable").get, "love.com", Set("tagX"), From 0e5911eceb568fe9eb9e1f730d5f468fc2ff1a4a Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 5 Nov 2020 11:39:05 +0100 Subject: [PATCH 134/237] #1552 Fixed order when importing customFields --- .../thp/thehive/dto/v0/CustomFieldValue.scala | 15 ++++++------- .../thp/thehive/dto/v1/CustomFieldValue.scala | 9 +++++--- .../thp/thehive/migration/th4/Output.scala | 2 +- .../misp/services/MispExportSrv.scala | 2 +- .../misp/services/MispImportSrv.scala | 2 +- .../thehive/controllers/v0/AlertCtrl.scala | 4 ++-- .../thehive/controllers/v1/AlertCtrl.scala | 2 +- .../thehive/controllers/v1/Properties.scala | 2 +- .../org/thp/thehive/services/AlertSrv.scala | 21 ++++++++++--------- .../thp/thehive/services/AlertSrvTest.scala | 2 +- 10 files changed, 33 insertions(+), 28 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala index 3931d5b4dd..d7eb91b277 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala @@ -70,20 +70,21 @@ object InputCustomFieldValue { case (_, FObject(fields)) => fields .toSeq + .zipWithIndex .validatedBy { - case (name, FString(value)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FNumber(value)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FBoolean(value)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FAny(value :: _)) => Good(InputCustomFieldValue(name, Some(value), None)) - case (name, FNull) => Good(InputCustomFieldValue(name, None, None)) - case (name, obj: FObject) => + case ((name, FString(value)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) + case ((name, FNumber(value)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) + case ((name, FBoolean(value)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) + case ((name, FAny(value :: _)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) + case ((name, FNull), i) => Good(InputCustomFieldValue(name, None, Some(i))) + case ((name, obj: FObject), i) => getStringCustomField(name, obj) orElse getIntegerCustomField(name, obj) orElse getFloatCustomField(name, obj) orElse getDateCustomField(name, obj) orElse getBooleanCustomField(name, obj) getOrElse Good(InputCustomFieldValue(name, None, None)) - case (name, other) => + case ((name, other), i) => Bad( One( InvalidFormatAttributeError(name, "CustomFieldValue", Set("field: string", "field: number", "field: boolean", "field: date"), other) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index 00159322de..17888b43e4 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -48,12 +48,15 @@ object InputCustomFieldValue { case (_, FObject(fields)) => fields .toSeq + .zipWithIndex .validatedBy { - case (name, valueField) => valueParser(valueField).map(v => InputCustomFieldValue(name, v, None)) + case ((name, valueField), i) => valueParser(valueField).map(v => InputCustomFieldValue(name, v, Some(i))) } .map(_.toSeq) case (_, FSeq(list)) => - list.zipWithIndex.validatedBy { + list + .zipWithIndex + .validatedBy { case (cf: FObject, i) => val order = FieldsParser.int(cf.get("order")).getOrElse(i) for { @@ -63,7 +66,7 @@ object InputCustomFieldValue { case (other, i) => Bad( One( - InvalidFormatAttributeError(s"customFild[$i]", "CustomFieldValue", Set.empty, other) + InvalidFormatAttributeError(s"customField[$i]", "CustomFieldValue", Set.empty, other) ) ) } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index d320cbc0d8..5323d43318 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -678,7 +678,7 @@ class Output @Inject() ( _ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), alert, organisation) _ <- caseTemplate.map(ct => alertSrv.alertCaseTemplateSrv.create(AlertCaseTemplate(), alert, ct)).flip _ <- tags.toTry(t => alertSrv.alertTagSrv.create(AlertTag(), alert, t)) - _ <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, name, value) } + _ <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, name, value, None) } _ = updateMetaData(alert, inputAlert.metaData) _ = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert, _)) } yield IdMapping(inputAlert.metaData.id, alert._id) diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala index 58c00318cd..ee69a055e2 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala @@ -150,7 +150,7 @@ class MispExportSrv @Inject() ( ) } org <- organisationSrv.getOrFail(authContext.organisation) - createdAlert <- alertSrv.create(alert.copy(lastSyncDate = new Date(0L)), org, Seq.empty[Tag with Entity], Map.empty[String, Option[Any]], None) + createdAlert <- alertSrv.create(alert.copy(lastSyncDate = new Date(0L)), org, Seq.empty[Tag with Entity], Seq(), None) _ <- alertSrv.alertCaseSrv.create(AlertCase(), createdAlert.alert, `case`) } yield createdAlert diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala index 95e8d64ecd..4318dd1ad4 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala @@ -354,7 +354,7 @@ class MispImportSrv @Inject() ( case None => // if the related alert doesn't exist, create it logger.debug(s"Event ${client.name}#${event.id} has no related alert for organisation ${organisation.name}") alertSrv - .create(alert, organisation, event.tags.map(_.name).toSet, Map.empty[String, Option[Any]], caseTemplate) + .create(alert, organisation, event.tags.map(_.name).toSet, Seq(), caseTemplate) .map(_.alert) case Some(richAlert) => logger.debug(s"Event ${client.name}#${event.id} have already been imported for organisation ${organisation.name}, updating the alert") diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 37662eaaa4..565cb287b5 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -52,7 +52,7 @@ class AlertCtrl @Inject() ( val caseTemplateName: Option[String] = request.body("caseTemplate") val inputAlert: InputAlert = request.body("alert") val observables: Seq[InputObservable] = request.body("observables") - val customFields = inputAlert.customFields.map(c => c.name -> c.value).toMap + val customFields = inputAlert.customFields.map(c => (c.name, c.value, c.order)) val caseTemplate = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption) for { organisation <- @@ -428,7 +428,7 @@ class PublicAlert @Inject() ( case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) => for { c <- alertSrv.getByIds(EntityId(vertex.id))(graph).getOrFail("Alert") - _ <- alertSrv.setOrCreateCustomField(c, name, Some(value))(graph, authContext) + _ <- alertSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) } yield Json.obj(s"customField.$name" -> value) case (FPathElem(_, FPathEmpty), values: JsObject, vertex, _, graph, authContext) => for { diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 4c1e5afc18..2eae69c298 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -85,7 +85,7 @@ class AlertCtrl @Inject() ( val caseTemplate = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption) for { organisation <- userSrv.current.organisations(Permissions.manageAlert).getOrFail("Organisation") - customFields = inputAlert.customFieldValue.map(cf => cf.name -> cf.value).toMap + customFields = inputAlert.customFieldValue.map(cf => (cf.name, cf.value, cf.order)) richAlert <- alertSrv.create(inputAlert.toAlert, organisation, inputAlert.tags, customFields, caseTemplate) } yield Results.Created(richAlert.toJson) } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index a5398a2b62..090cbd34aa 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -96,7 +96,7 @@ class Properties @Inject() ( case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) => for { c <- alertSrv.getOrFail(vertex)(graph) - _ <- alertSrv.setOrCreateCustomField(c, name, Some(value))(graph, authContext) + _ <- alertSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) } yield Json.obj(s"customField.$name" -> value) case _ => Failure(BadRequestError("Invalid custom fields format")) }) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index b4f40a6036..1f2e28a947 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -55,7 +55,7 @@ class AlertSrv @Inject() ( alert: Alert, organisation: Organisation with Entity, tagNames: Set[String], - customFields: Map[String, Option[Any]], + customFields: Seq[(String, Option[Any], Option[Int])], caseTemplate: Option[CaseTemplate with Entity] )(implicit graph: Graph, @@ -67,7 +67,7 @@ class AlertSrv @Inject() ( alert: Alert, organisation: Organisation with Entity, tags: Seq[Tag with Entity], - customFields: Map[String, Option[Any]], + customFields: Seq[(String, Option[Any], Option[Int])], caseTemplate: Option[CaseTemplate with Entity] )(implicit graph: Graph, @@ -82,7 +82,7 @@ class AlertSrv @Inject() ( _ <- alertOrganisationSrv.create(AlertOrganisation(), createdAlert, organisation) _ <- caseTemplate.map(ct => alertCaseTemplateSrv.create(AlertCaseTemplate(), createdAlert, ct)).flip _ <- tags.toTry(t => alertTagSrv.create(AlertTag(), createdAlert, t)) - cfs <- customFields.toTry { case (name, value) => createCustomField(createdAlert, name, value) } + cfs <- customFields.toTry { case (name, value, order) => createCustomField(createdAlert, name, value, order) } richAlert = RichAlert(createdAlert, organisation.name, tags, cfs, None, caseTemplate.map(_.name), 0) _ <- auditSrv.alert.create(createdAlert, richAlert.toJson) } yield richAlert @@ -170,16 +170,17 @@ class AlertSrv @Inject() ( def createCustomField( alert: Alert with Entity, - customFieldName: String, - customFieldValue: Option[Any] + name: String, + value: Option[Any], + order: Option[Int] )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { - cf <- customFieldSrv.getOrFail(EntityIdOrName(customFieldName)) - ccf <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), customFieldValue) + cf <- customFieldSrv.getOrFail(EntityIdOrName(name)) + ccf <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), value).map(_.order_=(order)) ccfe <- alertCustomFieldSrv.create(ccf, alert, cf) } yield RichCustomField(cf, ccfe) - def setOrCreateCustomField(alert: Alert with Entity, customFieldName: String, value: Option[Any])(implicit + def setOrCreateCustomField(alert: Alert with Entity, customFieldName: String, value: Option[Any], order: Option[Int])(implicit graph: Graph, authContext: AuthContext ): Try[Unit] = { @@ -187,7 +188,7 @@ class AlertSrv @Inject() ( if (cfv.clone().exists) cfv.setValue(value) else - createCustomField(alert, customFieldName, value).map(_ => ()) + createCustomField(alert, customFieldName, value, order).map(_ => ()) } def getCustomField(alert: Alert with Entity, customFieldName: String)(implicit graph: Graph): Option[RichCustomField] = @@ -205,7 +206,7 @@ class AlertSrv @Inject() ( .filterNot(rcf => customFieldNames.contains(rcf.name)) .foreach(rcf => get(alert).customFields(rcf.name).remove()) customFieldValues - .toTry { case (cf, v) => setOrCreateCustomField(alert, cf.name, Some(v)) } + .toTry { case (cf, v) => setOrCreateCustomField(alert, cf.name, Some(v), None) } .map(_ => ()) } diff --git a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala index 3af6601d61..8e10804106 100644 --- a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala @@ -39,7 +39,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { ), app[OrganisationSrv].getOrFail(EntityName("cert")).get, Set("tag1", "tag2"), - Map("string1" -> Some("lol")), + Seq(("string1", Some("lol"), None)), Some(app[CaseTemplateSrv].getOrFail(EntityName("spam")).get) ) } From a031e7aa02bec13f2feca374cf2a310634bea4f4 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Thu, 5 Nov 2020 13:52:54 +0100 Subject: [PATCH 135/237] #1552 #1557 Removed duplicate customFields when importing alert with a caseTemplate --- thehive/app/org/thp/thehive/services/CaseSrv.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 8c6a22b303..ecb996c201 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -80,7 +80,10 @@ class CaseSrv @Inject() ( caseTemplate .fold[Seq[RichCustomField]](Nil)(_.customFields) .map(cf => (cf.name, cf.value, cf.order)) - cfs <- (caseTemplateCustomFields ++ customFields).toTry { + uniqueFields = caseTemplateCustomFields.filter { + case (name, _, _) => !customFields.map(c => c._1).contains(name) + } + cfs <- (uniqueFields ++ customFields).toTry { case (name, value, order) => createCustomField(createdCase, EntityIdOrName(name), value, order) } caseTemplateTags = caseTemplate.fold[Seq[Tag with Entity]](Nil)(_.tags) From 359465b65107a1211e04c38c5b0cea25290dd78f Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Fri, 6 Nov 2020 11:34:31 +0100 Subject: [PATCH 136/237] #1552 WIP Cleaned customFields ordering --- .../scala/org/thp/thehive/dto/v1/Case.scala | 2 +- .../thp/thehive/dto/v1/CustomFieldValue.scala | 15 +++++---- .../migration/dto/InputCaseTemplate.scala | 3 +- .../thehive/migration/th3/Conversion.scala | 5 +-- .../thp/thehive/migration/th4/Output.scala | 5 +-- .../thehive/controllers/v0/AlertCtrl.scala | 5 +-- .../thp/thehive/controllers/v0/CaseCtrl.scala | 3 +- .../thehive/controllers/v1/AlertCtrl.scala | 4 +-- .../thp/thehive/controllers/v1/CaseCtrl.scala | 3 +- .../thehive/controllers/v1/Conversion.scala | 2 +- .../thehive/controllers/v1/Properties.scala | 3 +- .../org/thp/thehive/services/AlertSrv.scala | 27 ++++++++-------- .../org/thp/thehive/services/CaseSrv.scala | 31 +++++++++++++------ .../controllers/v0/AlertCtrlTest.scala | 4 +-- .../thehive/controllers/v0/CaseCtrlTest.scala | 6 ++-- .../thehive/controllers/v1/CaseCtrlTest.scala | 4 +-- .../thp/thehive/services/AlertSrvTest.scala | 3 +- 17 files changed, 70 insertions(+), 55 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala index 464a15864e..bd2fa69f4b 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala @@ -19,7 +19,7 @@ case class InputCase( summary: Option[String] = None, user: Option[String] = None, @WithParser(InputCustomFieldValue.parser) - customFieldValue: Seq[InputCustomFieldValue] = Nil + customFieldValues: Seq[InputCustomFieldValue] = Nil ) object InputCase { diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala index 17888b43e4..6e72438d06 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala @@ -48,25 +48,23 @@ object InputCustomFieldValue { case (_, FObject(fields)) => fields .toSeq - .zipWithIndex .validatedBy { - case ((name, valueField), i) => valueParser(valueField).map(v => InputCustomFieldValue(name, v, Some(i))) + case (name, valueField) => valueParser(valueField).map(v => InputCustomFieldValue(name, v, None)) } .map(_.toSeq) case (_, FSeq(list)) => list - .zipWithIndex .validatedBy { - case (cf: FObject, i) => - val order = FieldsParser.int(cf.get("order")).getOrElse(i) + case cf: FObject => + val order = FieldsParser.int(cf.get("order")).toOption for { name <- FieldsParser.string(cf.get("name")) value <- valueParser(cf.get("value")) - } yield InputCustomFieldValue(name, value, Some(order)) - case (other, i) => + } yield InputCustomFieldValue(name, value, order) + case other => Bad( One( - InvalidFormatAttributeError(s"customField[$i]", "CustomFieldValue", Set.empty, other) + InvalidFormatAttributeError(s"customField", "CustomFieldValue", Set.empty, other) ) ) } @@ -82,6 +80,7 @@ object InputCustomFieldValue { case InputCustomFieldValue(name, None, _) => name -> JsNull case InputCustomFieldValue(name, other, _) => sys.error(s"The custom field $name has invalid value: $other (${other.getClass})") } + // TODO Change JsObject to JsArray ? JsObject(fields) } } diff --git a/migration/src/main/scala/org/thp/thehive/migration/dto/InputCaseTemplate.scala b/migration/src/main/scala/org/thp/thehive/migration/dto/InputCaseTemplate.scala index 85103cb796..76138e032c 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/dto/InputCaseTemplate.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/dto/InputCaseTemplate.scala @@ -1,5 +1,6 @@ package org.thp.thehive.migration.dto +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models.CaseTemplate case class InputCaseTemplate( @@ -7,5 +8,5 @@ case class InputCaseTemplate( caseTemplate: CaseTemplate, organisation: String, tags: Set[String], - customFields: Seq[(String, Option[Any], Option[Int])] + customFields: Seq[InputCustomFieldValue] ) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index 77b45faac6..2694c96c7b 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -8,6 +8,7 @@ import akka.util.ByteString import org.thp.scalligraph.utils.Hash import org.thp.thehive.connector.cortex.models.{Action, Job, JobStatus} import org.thp.thehive.controllers.v0 +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.migration.dto._ import org.thp.thehive.models._ import play.api.libs.functional.syntax._ @@ -338,12 +339,12 @@ trait Conversion { tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty) metrics = (json \ "metrics").asOpt[JsObject].getOrElse(JsObject.empty) metricsValue = metrics.value.map { - case (name, value) => (name, Some(value), None) + case (name, value) => InputCustomFieldValue(name, Some(value), None) } customFields <- (json \ "customFields").validateOpt[JsObject] customFieldsValue = customFields.getOrElse(JsObject.empty).value.map { case (name, value) => - ( + InputCustomFieldValue( name, Some((value \ "string") orElse (value \ "boolean") orElse (value \ "number") orElse (value \ "date") getOrElse JsNull), (value \ "order").asOpt[Int] diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 5323d43318..836bf568a9 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -16,6 +16,7 @@ import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.connector.cortex.models.{CortexSchemaDefinition, TheHiveCortexSchemaProvider} import org.thp.thehive.connector.cortex.services.{ActionSrv, JobSrv} +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.migration import org.thp.thehive.migration.IdMapping import org.thp.thehive.migration.dto._ @@ -467,7 +468,7 @@ class Output @Inject() ( richCaseTemplate <- caseTemplateSrv.create(inputCaseTemplate.caseTemplate, organisation, tags, Nil, Nil) _ = updateMetaData(richCaseTemplate.caseTemplate, inputCaseTemplate.metaData) _ = inputCaseTemplate.customFields.foreach { - case (name, value, order) => + case InputCustomFieldValue(name, value, order) => (for { cf <- getCustomField(name) ccf <- CustomFieldType.map(cf.`type`).setValue(CaseTemplateCustomField(order = order), value) @@ -678,7 +679,7 @@ class Output @Inject() ( _ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), alert, organisation) _ <- caseTemplate.map(ct => alertSrv.alertCaseTemplateSrv.create(AlertCaseTemplate(), alert, ct)).flip _ <- tags.toTry(t => alertSrv.alertTagSrv.create(AlertTag(), alert, t)) - _ <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, name, value, None) } + _ <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, InputCustomFieldValue(name, value, None)) } _ = updateMetaData(alert, inputAlert.metaData) _ = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert, _)) } yield IdMapping(inputAlert.metaData.id, alert._id) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 565cb287b5..8c71bea2f8 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -14,6 +14,7 @@ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityId, EntityIdOrName, EntityName, InvalidFormatAttributeError, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase} +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseOps._ @@ -52,7 +53,7 @@ class AlertCtrl @Inject() ( val caseTemplateName: Option[String] = request.body("caseTemplate") val inputAlert: InputAlert = request.body("alert") val observables: Seq[InputObservable] = request.body("observables") - val customFields = inputAlert.customFields.map(c => (c.name, c.value, c.order)) + val customFields = inputAlert.customFields.map(c => InputCustomFieldValue(c.name, c.value, c.order)) val caseTemplate = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption) for { organisation <- @@ -428,7 +429,7 @@ class PublicAlert @Inject() ( case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) => for { c <- alertSrv.getByIds(EntityId(vertex.id))(graph).getOrFail("Alert") - _ <- alertSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) + _ <- alertSrv.setOrCreateCustomField(c, InputCustomFieldValue(name, Some(value), None))(graph, authContext) } yield Json.obj(s"customField.$name" -> value) case (FPathElem(_, FPathEmpty), values: JsObject, vertex, _, graph, authContext) => for { diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index 62015e054d..aa94cfbd82 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -12,6 +12,7 @@ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.scalligraph.{RichSeq, _} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputCase, InputTask} +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CaseTemplateOps._ @@ -46,7 +47,7 @@ class CaseCtrl @Inject() ( val caseTemplateName: Option[String] = request.body("caseTemplate") val inputCase: InputCase = request.body("case") val inputTasks: Seq[InputTask] = request.body("tasks") - val customFields = inputCase.customFields.map(c => (c.name, c.value, c.order)) + val customFields = inputCase.customFields.map(c => InputCustomFieldValue(c.name, c.value, c.order)) for { organisation <- userSrv diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 2eae69c298..0a5477c2e3 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -10,7 +10,7 @@ import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.thehive.controllers.v1.Conversion._ -import org.thp.thehive.dto.v1.InputAlert +import org.thp.thehive.dto.v1.{InputAlert, InputCustomFieldValue} import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseTemplateOps._ @@ -85,7 +85,7 @@ class AlertCtrl @Inject() ( val caseTemplate = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption) for { organisation <- userSrv.current.organisations(Permissions.manageAlert).getOrFail("Organisation") - customFields = inputAlert.customFieldValue.map(cf => (cf.name, cf.value, cf.order)) + customFields = inputAlert.customFieldValue.map(cf => InputCustomFieldValue(cf.name, cf.value, cf.order)) richAlert <- alertSrv.create(inputAlert.toAlert, organisation, inputAlert.tags, customFields, caseTemplate) } yield Results.Created(richAlert.toJson) } diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index 3139f7d6e3..0440346733 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -72,7 +72,6 @@ class CaseCtrl @Inject() ( val inputTasks: Seq[InputTask] = request.body("tasks") for { caseTemplate <- caseTemplateName.map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.richCaseTemplate.getOrFail("CaseTemplate")).flip - customFields = inputCase.customFieldValue.map(cf => (cf.name, cf.value, cf.order)) organisation <- userSrv.current.organisations(Permissions.manageCase).get(request.organisation).getOrFail("Organisation") user <- inputCase.user.fold[Try[Option[User with Entity]]](Success(None))(u => userSrv.getOrFail(EntityIdOrName(u)).map(Some.apply)) tags <- inputCase.tags.toTry(tagSrv.getOrCreate) @@ -81,7 +80,7 @@ class CaseCtrl @Inject() ( user, organisation, tags.toSet, - customFields, + inputCase.customFieldValues, caseTemplate, inputTasks.map(t => t.toTask -> t.assignee.flatMap(u => userSrv.get(EntityIdOrName(u)).headOption)) ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index e2186c0419..5c569fb0d0 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -131,7 +131,7 @@ object Conversion { status = inputCase.status, summary = inputCase.summary orElse caseTemplate.summary, user = inputCase.user, - customFieldValue = inputCase.customFieldValue + customFieldValues = inputCase.customFieldValues ) } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 090cbd34aa..45f59c1313 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -9,6 +9,7 @@ import org.thp.scalligraph.query.{PublicProperties, PublicPropertyListBuilder} import org.thp.scalligraph.traversal.Converter import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{BadRequestError, EntityIdOrName, RichSeq} +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.AuditOps._ @@ -96,7 +97,7 @@ class Properties @Inject() ( case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) => for { c <- alertSrv.getOrFail(vertex)(graph) - _ <- alertSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) + _ <- alertSrv.setOrCreateCustomField(c, InputCustomFieldValue(name, Some(value), None))(graph, authContext) } yield Json.obj(s"customField.$name" -> value) case _ => Failure(BadRequestError("Invalid custom fields format")) }) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 1f2e28a947..c45a656ee6 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -14,6 +14,7 @@ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IdentityConverter, StepLabel, Traversal} import org.thp.scalligraph.{CreateError, EntityId, EntityIdOrName, RichOptionTry, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseOps._ @@ -55,7 +56,7 @@ class AlertSrv @Inject() ( alert: Alert, organisation: Organisation with Entity, tagNames: Set[String], - customFields: Seq[(String, Option[Any], Option[Int])], + customFields: Seq[InputCustomFieldValue], caseTemplate: Option[CaseTemplate with Entity] )(implicit graph: Graph, @@ -67,7 +68,7 @@ class AlertSrv @Inject() ( alert: Alert, organisation: Organisation with Entity, tags: Seq[Tag with Entity], - customFields: Seq[(String, Option[Any], Option[Int])], + customFields: Seq[InputCustomFieldValue], caseTemplate: Option[CaseTemplate with Entity] )(implicit graph: Graph, @@ -82,7 +83,7 @@ class AlertSrv @Inject() ( _ <- alertOrganisationSrv.create(AlertOrganisation(), createdAlert, organisation) _ <- caseTemplate.map(ct => alertCaseTemplateSrv.create(AlertCaseTemplate(), createdAlert, ct)).flip _ <- tags.toTry(t => alertTagSrv.create(AlertTag(), createdAlert, t)) - cfs <- customFields.toTry { case (name, value, order) => createCustomField(createdAlert, name, value, order) } + cfs <- customFields.toTry { simpleCf: InputCustomFieldValue => createCustomField(createdAlert, simpleCf) } richAlert = RichAlert(createdAlert, organisation.name, tags, cfs, None, caseTemplate.map(_.name), 0) _ <- auditSrv.alert.create(createdAlert, richAlert.toJson) } yield richAlert @@ -170,25 +171,23 @@ class AlertSrv @Inject() ( def createCustomField( alert: Alert with Entity, - name: String, - value: Option[Any], - order: Option[Int] + simpleCf: InputCustomFieldValue )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { - cf <- customFieldSrv.getOrFail(EntityIdOrName(name)) - ccf <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), value).map(_.order_=(order)) + cf <- customFieldSrv.getOrFail(EntityIdOrName(simpleCf.name)) + ccf <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), simpleCf.value).map(_.order_=(simpleCf.order)) ccfe <- alertCustomFieldSrv.create(ccf, alert, cf) } yield RichCustomField(cf, ccfe) - def setOrCreateCustomField(alert: Alert with Entity, customFieldName: String, value: Option[Any], order: Option[Int])(implicit + def setOrCreateCustomField(alert: Alert with Entity, simpleCf: InputCustomFieldValue)(implicit graph: Graph, authContext: AuthContext ): Try[Unit] = { - val cfv = get(alert).customFields(customFieldName) + val cfv = get(alert).customFields(simpleCf.name) if (cfv.clone().exists) - cfv.setValue(value) + cfv.setValue(simpleCf.value) else - createCustomField(alert, customFieldName, value, order).map(_ => ()) + createCustomField(alert, simpleCf).map(_ => ()) } def getCustomField(alert: Alert with Entity, customFieldName: String)(implicit graph: Graph): Option[RichCustomField] = @@ -206,7 +205,7 @@ class AlertSrv @Inject() ( .filterNot(rcf => customFieldNames.contains(rcf.name)) .foreach(rcf => get(alert).customFields(rcf.name).remove()) customFieldValues - .toTry { case (cf, v) => setOrCreateCustomField(alert, cf.name, Some(v), None) } + .toTry { case (cf, v) => setOrCreateCustomField(alert, InputCustomFieldValue(cf.name, Some(v), None)) } .map(_ => ()) } @@ -244,7 +243,7 @@ class AlertSrv @Inject() ( .caseTemplate .map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).richCaseTemplate.getOrFail("CaseTemplate")) .flip - customField = alert.customFields.map(f => (f.name, f.value, f.order)) + customField = alert.customFields.map(f => InputCustomFieldValue(f.name, f.value, f.order)) case0 = Case( number = 0, title = caseTemplate.flatMap(_.titlePrefix).getOrElse("") + alert.title, diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index ecb996c201..84123daa92 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -15,6 +15,7 @@ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal} import org.thp.scalligraph.{CreateError, EntityIdOrName, EntityName, RichOptionTry, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CustomFieldOps._ @@ -62,7 +63,7 @@ class CaseSrv @Inject() ( user: Option[User with Entity], organisation: Organisation with Entity, tags: Set[Tag with Entity], - customFields: Seq[(String, Option[Any], Option[Int])], + customFields: Seq[InputCustomFieldValue], caseTemplate: Option[RichCaseTemplate], additionalTasks: Seq[(Task, Option[User with Entity])] )(implicit graph: Graph, authContext: AuthContext): Try[RichCase] = @@ -72,27 +73,37 @@ class CaseSrv @Inject() ( _ <- caseUserSrv.create(CaseUser(), createdCase, assignee) _ <- shareSrv.shareCase(owner = true, createdCase, organisation, profileSrv.orgAdmin) _ <- caseTemplate.map(ct => caseCaseTemplateSrv.create(CaseCaseTemplate(), createdCase, ct.caseTemplate)).flip + createdTasks <- caseTemplate.fold(additionalTasks)(_.tasks.map(t => t.task -> t.assignee)).toTry { case (task, owner) => taskSrv.create(task, owner) } _ <- createdTasks.toTry(t => shareSrv.shareTask(t, createdCase, organisation)) - caseTemplateCustomFields = - caseTemplate - .fold[Seq[RichCustomField]](Nil)(_.customFields) - .map(cf => (cf.name, cf.value, cf.order)) - uniqueFields = caseTemplateCustomFields.filter { - case (name, _, _) => !customFields.map(c => c._1).contains(name) - } - cfs <- (uniqueFields ++ customFields).toTry { - case (name, value, order) => createCustomField(createdCase, EntityIdOrName(name), value, order) + + caseTemplateCf = caseTemplate + .fold[Seq[RichCustomField]](Seq())(_.customFields) + .map(cf => InputCustomFieldValue(cf.name, cf.value, cf.order)) + cfs <- cleanCustomFields(caseTemplateCf, customFields).toTry { + case InputCustomFieldValue(name, value, order) => createCustomField(createdCase, EntityIdOrName(name), value, order) } + caseTemplateTags = caseTemplate.fold[Seq[Tag with Entity]](Nil)(_.tags) allTags = tags ++ caseTemplateTags _ <- allTags.toTry(t => caseTagSrv.create(CaseTag(), createdCase, t)) + richCase = RichCase(createdCase, allTags.toSeq, None, None, Some(assignee.login), cfs, authContext.permissions) _ <- auditSrv.`case`.create(createdCase, richCase.toJson) } yield richCase + private def cleanCustomFields(caseTemplateCf: Seq[InputCustomFieldValue], caseCf: Seq[InputCustomFieldValue]): Seq[InputCustomFieldValue] = { + val uniqueFields = caseTemplateCf.filter { + case InputCustomFieldValue(name, _, _) => !caseCf.map(c => c.name).contains(name) + } + (caseCf ++ uniqueFields) + .sortBy(cf => (cf.order.isEmpty, cf.order)) + .zipWithIndex + .map { case (InputCustomFieldValue(name, value, _), i) => InputCustomFieldValue(name, value, Some(i)) } + } + def nextCaseNumber(implicit graph: Graph): Int = startTraversal.getLast.headOption.fold(0)(_.number) + 1 override def exists(e: Case)(implicit graph: Graph): Boolean = startTraversal.getByNumber(e.number).exists diff --git a/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala index 0741f9acac..fd5da90598 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala @@ -275,8 +275,8 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder { summary = None, owner = Some("certuser@thehive.local"), customFields = Json.obj( - "boolean1" -> Json.obj("boolean" -> JsNull, "order" -> 2), - "string1" -> Json.obj("string" -> "string1 custom field", "order" -> 1) + "boolean1" -> Json.obj("boolean" -> JsNull, "order" -> 1), + "string1" -> Json.obj("string" -> "string1 custom field", "order" -> 0) ), stats = Json.obj() ) diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala index 3d90d757f2..f677c55520 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala @@ -90,9 +90,9 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { summary = None, owner = Some("certuser@thehive.local"), customFields = Json.obj( - "boolean1" -> Json.obj("boolean" -> true, "order" -> JsNull), - "string1" -> Json.obj("string" -> "string1 custom field", "order" -> 1), - "date1" -> Json.obj("date" -> now.getTime, "order" -> JsNull) + "boolean1" -> Json.obj("boolean" -> true, "order" -> 2), + "string1" -> Json.obj("string" -> "string1 custom field", "order" -> 0), + "date1" -> Json.obj("date" -> now.getTime, "order" -> 1) ), stats = Json.obj() ) diff --git a/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala index be0ab07cb3..a7726a858e 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/CaseCtrlTest.scala @@ -132,8 +132,8 @@ class CaseCtrlTest extends PlaySpecification with TestAppBuilder { summary = None, user = Some("certuser@thehive.local"), customFields = Seq( - TestCustomFieldValue("string1", "string custom field", "string", JsString("string1 custom field"), 1), - TestCustomFieldValue("boolean1", "boolean custom field", "boolean", JsNull, 2) + TestCustomFieldValue("string1", "string custom field", "string", JsString("string1 custom field"), 0), + TestCustomFieldValue("boolean1", "boolean custom field", "boolean", JsNull, 1) ) ) diff --git a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala index 8e10804106..5db4963a9a 100644 --- a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala @@ -7,6 +7,7 @@ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.TestAppBuilder +import org.thp.thehive.dto.v1.InputCustomFieldValue import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseOps._ @@ -39,7 +40,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { ), app[OrganisationSrv].getOrFail(EntityName("cert")).get, Set("tag1", "tag2"), - Seq(("string1", Some("lol"), None)), + Seq(InputCustomFieldValue("string1", Some("lol"), None)), Some(app[CaseTemplateSrv].getOrFail(EntityName("spam")).get) ) } From df5e7a02362634c237dc38a86afeca980f617251 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Fri, 6 Nov 2020 12:34:12 +0100 Subject: [PATCH 137/237] #1552 fixed unit test failure --- .../org/thp/thehive/dto/v0/CustomFieldValue.scala | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala index d7eb91b277..3931d5b4dd 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala @@ -70,21 +70,20 @@ object InputCustomFieldValue { case (_, FObject(fields)) => fields .toSeq - .zipWithIndex .validatedBy { - case ((name, FString(value)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) - case ((name, FNumber(value)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) - case ((name, FBoolean(value)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) - case ((name, FAny(value :: _)), i) => Good(InputCustomFieldValue(name, Some(value), Some(i))) - case ((name, FNull), i) => Good(InputCustomFieldValue(name, None, Some(i))) - case ((name, obj: FObject), i) => + case (name, FString(value)) => Good(InputCustomFieldValue(name, Some(value), None)) + case (name, FNumber(value)) => Good(InputCustomFieldValue(name, Some(value), None)) + case (name, FBoolean(value)) => Good(InputCustomFieldValue(name, Some(value), None)) + case (name, FAny(value :: _)) => Good(InputCustomFieldValue(name, Some(value), None)) + case (name, FNull) => Good(InputCustomFieldValue(name, None, None)) + case (name, obj: FObject) => getStringCustomField(name, obj) orElse getIntegerCustomField(name, obj) orElse getFloatCustomField(name, obj) orElse getDateCustomField(name, obj) orElse getBooleanCustomField(name, obj) getOrElse Good(InputCustomFieldValue(name, None, None)) - case ((name, other), i) => + case (name, other) => Bad( One( InvalidFormatAttributeError(name, "CustomFieldValue", Set("field: string", "field: number", "field: boolean", "field: date"), other) From f961fe1b6a28c14fc891a2eb606e0eb5195ebfb7 Mon Sep 17 00:00:00 2001 From: Robin Riclet Date: Fri, 6 Nov 2020 13:59:21 +0100 Subject: [PATCH 138/237] #1552 Renamed val & used contains() --- .../app/org/thp/thehive/services/AlertSrv.scala | 16 ++++++++-------- .../app/org/thp/thehive/services/CaseSrv.scala | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index c45a656ee6..f665692ecc 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -83,7 +83,7 @@ class AlertSrv @Inject() ( _ <- alertOrganisationSrv.create(AlertOrganisation(), createdAlert, organisation) _ <- caseTemplate.map(ct => alertCaseTemplateSrv.create(AlertCaseTemplate(), createdAlert, ct)).flip _ <- tags.toTry(t => alertTagSrv.create(AlertTag(), createdAlert, t)) - cfs <- customFields.toTry { simpleCf: InputCustomFieldValue => createCustomField(createdAlert, simpleCf) } + cfs <- customFields.toTry { cf: InputCustomFieldValue => createCustomField(createdAlert, cf) } richAlert = RichAlert(createdAlert, organisation.name, tags, cfs, None, caseTemplate.map(_.name), 0) _ <- auditSrv.alert.create(createdAlert, richAlert.toJson) } yield richAlert @@ -171,23 +171,23 @@ class AlertSrv @Inject() ( def createCustomField( alert: Alert with Entity, - simpleCf: InputCustomFieldValue + inputCf: InputCustomFieldValue )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { - cf <- customFieldSrv.getOrFail(EntityIdOrName(simpleCf.name)) - ccf <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), simpleCf.value).map(_.order_=(simpleCf.order)) + cf <- customFieldSrv.getOrFail(EntityIdOrName(inputCf.name)) + ccf <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), inputCf.value).map(_.order_=(inputCf.order)) ccfe <- alertCustomFieldSrv.create(ccf, alert, cf) } yield RichCustomField(cf, ccfe) - def setOrCreateCustomField(alert: Alert with Entity, simpleCf: InputCustomFieldValue)(implicit + def setOrCreateCustomField(alert: Alert with Entity, cf: InputCustomFieldValue)(implicit graph: Graph, authContext: AuthContext ): Try[Unit] = { - val cfv = get(alert).customFields(simpleCf.name) + val cfv = get(alert).customFields(cf.name) if (cfv.clone().exists) - cfv.setValue(simpleCf.value) + cfv.setValue(cf.value) else - createCustomField(alert, simpleCf).map(_ => ()) + createCustomField(alert, cf).map(_ => ()) } def getCustomField(alert: Alert with Entity, customFieldName: String)(implicit graph: Graph): Option[RichCustomField] = diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 84123daa92..dfcc74af4c 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -96,7 +96,7 @@ class CaseSrv @Inject() ( private def cleanCustomFields(caseTemplateCf: Seq[InputCustomFieldValue], caseCf: Seq[InputCustomFieldValue]): Seq[InputCustomFieldValue] = { val uniqueFields = caseTemplateCf.filter { - case InputCustomFieldValue(name, _, _) => !caseCf.map(c => c.name).contains(name) + case InputCustomFieldValue(name, _, _) => !caseCf.exists(_.name == name) } (caseCf ++ uniqueFields) .sortBy(cf => (cf.order.isEmpty, cf.order)) From 295e3e758711177c11a2830315df6e8fe80c9322 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 6 Nov 2020 16:01:29 +0100 Subject: [PATCH 139/237] #1627 Add report-template link to admin section --- .../scripts/controllers/admin/AdminAnalyzerTemplatesCtrl.js | 3 ++- frontend/app/views/partials/admin/analyzer-templates.html | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/app/scripts/controllers/admin/AdminAnalyzerTemplatesCtrl.js b/frontend/app/scripts/controllers/admin/AdminAnalyzerTemplatesCtrl.js index 583fdca2cd..1b7e7ea73a 100644 --- a/frontend/app/scripts/controllers/admin/AdminAnalyzerTemplatesCtrl.js +++ b/frontend/app/scripts/controllers/admin/AdminAnalyzerTemplatesCtrl.js @@ -8,9 +8,10 @@ .controller('AdminAnalyzerTemplateDeleteCtrl', AdminAnalyzerTemplateDeleteCtrl); - function AdminAnalyzerTemplatesCtrl($q, $uibModal, AnalyzerSrv, AnalyzerTemplateSrv, NotificationSrv) { + function AdminAnalyzerTemplatesCtrl($q, $uibModal, AnalyzerSrv, AnalyzerTemplateSrv, NotificationSrv, appConfig) { var self = this; + this.appConfig = appConfig; this.templates = []; this.analyzers = []; this.analyzerIds = []; diff --git a/frontend/app/views/partials/admin/analyzer-templates.html b/frontend/app/views/partials/admin/analyzer-templates.html index 66d669c3cf..590205556c 100644 --- a/frontend/app/views/partials/admin/analyzer-templates.html +++ b/frontend/app/views/partials/admin/analyzer-templates.html @@ -15,6 +15,10 @@

      Analyzer template management

      +
      +

      Download the official templates archive

      +

      You can download the latest archive of the official analyzer templates from here

      +
      No analyzer templates found.
      From 56afbc2965b3294096675ca965b652fec977487e Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 6 Nov 2020 16:07:04 +0100 Subject: [PATCH 140/237] #1612 Use _in operator instead of _or --- frontend/app/scripts/controllers/MainPageCtrl.js | 14 ++++---------- frontend/app/scripts/controllers/RootCtrl.js | 14 ++++---------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/frontend/app/scripts/controllers/MainPageCtrl.js b/frontend/app/scripts/controllers/MainPageCtrl.js index 116f9e1579..8320212b90 100644 --- a/frontend/app/scripts/controllers/MainPageCtrl.js +++ b/frontend/app/scripts/controllers/MainPageCtrl.js @@ -9,16 +9,10 @@ self.view = {}; self.defaultFilter = { - _or: [ - { - _field: 'status', - _value: 'Waiting' - }, - { - _field: 'status', - _value: 'InProgress' - } - ] + _in: { + _field: 'status', + _values: ['Waiting', 'InProgress'] + } }; self.queryOperations = view === 'mytasks' ? [ diff --git a/frontend/app/scripts/controllers/RootCtrl.js b/frontend/app/scripts/controllers/RootCtrl.js index 26e6982dc3..15341e81f2 100644 --- a/frontend/app/scripts/controllers/RootCtrl.js +++ b/frontend/app/scripts/controllers/RootCtrl.js @@ -58,16 +58,10 @@ angular.module('theHiveControllers').controller('RootCtrl', {_name: 'currentUser'}, {_name: 'tasks'}, {_name: 'filter', - _or: [ - { - _field: 'status', - _value: 'Waiting' - }, - { - _field: 'status', - _value: 'InProgress' - } - ] + _in: { + _field: 'status', + _values: ['Waiting', 'InProgress'] + } }, {_name: 'count'} ], { From c7170e323b16ebeb942a09368f4a585ef02f4065 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 6 Nov 2020 16:43:24 +0100 Subject: [PATCH 141/237] #1623 Add the API key section in the user's settings page --- .../app/scripts/controllers/SettingsCtrl.js | 40 +++++++++++++++++++ .../app/views/partials/personal-settings.html | 30 ++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/frontend/app/scripts/controllers/SettingsCtrl.js b/frontend/app/scripts/controllers/SettingsCtrl.js index c84b6ef61d..470670ef98 100644 --- a/frontend/app/scripts/controllers/SettingsCtrl.js +++ b/frontend/app/scripts/controllers/SettingsCtrl.js @@ -24,6 +24,8 @@ passwordConfirm: '' }; + $scope.keyData = {}; + $scope.mfaData = { enabled: $scope.currentUser.hasMFA, secret: null, @@ -32,6 +34,7 @@ }; $scope.canChangePass = appConfig.config.capabilities.indexOf('changePassword') !== -1; + $scope.canChangeKey = appConfig.config.capabilities.indexOf('authByKey') !== -1; $scope.canChangeMfa = appConfig.config.capabilities.indexOf('mfa') !== -1; @@ -95,6 +98,43 @@ } }; + $scope.createKey = function(){ + var modalInstance = ModalSrv.confirm( + 'Renew API Key', + 'Are you sure you want to renew your API Key?', { + flavor: 'danger', + okText: 'Yes, renew it' + } + ); + + modalInstance.result + .then(function() { + UserSrv.setKey($scope.currentUser.login); + }) + .then(function() { + delete $scope.keyData.key; + NotificationSrv.success('API key has been successfully renewed.'); + }) + .catch(function(err) { + if (!_.isString(err)) { + NotificationSrv.error('SettingsCtrl', err.data, err.status); + } + }); + }; + + $scope.getKey = function() { + UserSrv.getKey($scope.currentUser.login) + .then(function(key) { + $scope.keyData.key = key; + }); + }; + + $scope.copyKey = function() { + clipboard.copyText($scope.keyData.key); + delete $scope.keyData.key; + NotificationSrv.success('API key has been successfully copied to clipboard.'); + }; + $scope.copySecret = function(secret) { clipboard.copyText(secret); NotificationSrv.success('MFA Secret has been successfully copied to clipboard.'); diff --git a/frontend/app/views/partials/personal-settings.html b/frontend/app/views/partials/personal-settings.html index 8adb4364c6..525cdf7c94 100644 --- a/frontend/app/views/partials/personal-settings.html +++ b/frontend/app/views/partials/personal-settings.html @@ -136,6 +136,36 @@

      + +
      +
      +
      +

      API Key

      +
      +
      +
      +

      You don't have any API key.

      +

      Please contact your organization's administrator.

      +
      +
      +

      You have an API key defined. You can renew it if needed.

      +
      + + Renew + Reveal + + + + + +
      +
      +
      +
      +
      +
      From 5c3a9f1e58ce5c0d6023f5e66b46e70bb35bb87e Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 7 Nov 2020 09:23:09 +0100 Subject: [PATCH 142/237] #1625 Display ignoreSimilarity in case and alert observables list --- .../app/views/components/alert/observable-list.component.html | 3 ++- frontend/app/views/partials/observables/list/observables.html | 4 ++++ thehive/app/org/thp/thehive/services/AlertSrv.scala | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/app/views/components/alert/observable-list.component.html b/frontend/app/views/components/alert/observable-list.component.html index d69db28872..21abc48972 100644 --- a/frontend/app/views/components/alert/observable-list.component.html +++ b/frontend/app/views/components/alert/observable-list.component.html @@ -23,7 +23,7 @@ - + Type Data Date Added @@ -36,6 +36,7 @@ + {{observable.dataType}} diff --git a/frontend/app/views/partials/observables/list/observables.html b/frontend/app/views/partials/observables/list/observables.html index 823c37e8df..1d852c7e4d 100644 --- a/frontend/app/views/partials/observables/list/observables.html +++ b/frontend/app/views/partials/observables/list/observables.html @@ -31,6 +31,7 @@

      + Type @@ -71,6 +72,9 @@

      + + + Date: Sat, 7 Nov 2020 09:53:39 +0100 Subject: [PATCH 143/237] #1625 Add the ignoreSimilarity flag to the observable bulk update dialog --- .../controllers/case/ObservableUpdateCtrl.js | 14 +++++++++++++- .../partials/observables/observable.update.html | 11 +++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/frontend/app/scripts/controllers/case/ObservableUpdateCtrl.js b/frontend/app/scripts/controllers/case/ObservableUpdateCtrl.js index d349dd8ccb..533ad1ef3c 100644 --- a/frontend/app/scripts/controllers/case/ObservableUpdateCtrl.js +++ b/frontend/app/scripts/controllers/case/ObservableUpdateCtrl.js @@ -10,6 +10,7 @@ enableTlp: false, enableIoc: false, enableSighted: false, + enableIgnoreSimilarity: false, enableAddTags: false, enableRemoveTags: false }; @@ -18,6 +19,7 @@ this.params = { ioc: false, sighted: false, + ignoreSimilarity: false, tlp: 2, addTagNames: '', removeTagNames: '' @@ -30,6 +32,7 @@ this.state.enableTlp = this.state.all; this.state.enableIoc = this.state.all; this.state.enableSighted = this.state.all; + this.state.enableIgnoreSimilarity = this.state.all; this.state.enableAddTags = this.state.all; this.state.enableRemoveTags = this.state.all; }; @@ -52,7 +55,7 @@ }; this.buildOperations = function(postData) { - var flags = _.pick(postData, 'ioc', 'sighted', 'tlp'); + var flags = _.pick(postData, 'ioc', 'sighted', 'ignoreSimilarity', 'tlp'); // Handle updates without tag changes if(!postData.addTags && !postData.removeTags) { @@ -110,6 +113,10 @@ postData.sighted = this.params.sighted; } + if(this.state.enableIgnoreSimilarity) { + postData.ignoreSimilarity = this.params.ignoreSimilarity; + } + if(this.state.enableTlp) { postData.tlp = this.params.tlp; } @@ -143,6 +150,11 @@ this.state.enableSighted = true; }; + this.toogleIgnoreSimilarity = function() { + this.params.ignoreSimilarity = !this.params.ignoreSimilarity; + this.state.enableIgnoreSimilarity = true; + }; + this.toggleTlp = function(value) { this.params.tlp = value; this.activeTlp = 'active'; diff --git a/frontend/app/views/partials/observables/observable.update.html b/frontend/app/views/partials/observables/observable.update.html index 1959126add..71e2ba7b67 100644 --- a/frontend/app/views/partials/observables/observable.update.html +++ b/frontend/app/views/partials/observables/observable.update.html @@ -41,6 +41,17 @@

      Update observable(s)

      +
      + +
      +

      + + + +

      +
      +
      +
      From ae5b9101c7c8f9969ff8345e32990db825aa0f27 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 8 Nov 2020 08:16:25 +0100 Subject: [PATCH 144/237] #1625 Make ignoreSimilarity updatable via bulk patch API --- thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 407a2604a0..0cf30e6d08 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -208,6 +208,7 @@ class PublicObservable @Inject() ( .property("startDate", UMapping.date)(_.select(_._createdAt).readonly) .property("ioc", UMapping.boolean)(_.field.updatable) .property("sighted", UMapping.boolean)(_.field.updatable) + .property("ignoreSimilarity", UMapping.boolean)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) .filter((_, cases) => From 7112ed13aaa8b2a30a7a3944ae52d7bbf79cce3a Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 8 Nov 2020 08:20:58 +0100 Subject: [PATCH 145/237] #1625 Add ignoreSimilarity to descrive v1 API --- thehive/app/org/thp/thehive/controllers/v1/Properties.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 45f59c1313..f435c9b3d3 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -345,6 +345,7 @@ class Properties @Inject() ( .property("startDate", UMapping.date)(_.select(_._createdAt).readonly) .property("ioc", UMapping.boolean)(_.field.updatable) .property("sighted", UMapping.boolean)(_.field.updatable) + .property("ignoreSimilarity", UMapping.boolean)(_.field.updatable) .property("tags", UMapping.string.set)( _.select(_.tags.displayName) .filter((_, cases) => From 52309269dda27ad2c64d1e1439d3c1ee9ca7c13b Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 8 Nov 2020 08:43:52 +0100 Subject: [PATCH 146/237] #1625 Use observable ignoreSimilarity flag in observable details page --- .../case/CaseObservablesItemCtrl.js | 60 +++++++++++-------- .../partials/observables/details/summary.html | 17 +++++- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js b/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js index b07205e918..605f994af8 100644 --- a/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js @@ -33,23 +33,6 @@ mode: 'vb' }; - // Add tab - CaseTabsSrv.addTab(observableName, { - name: observableName, - label: artifact.data || artifact.attachment.name, - closable: true, - state: 'app.case.observables-item', - params: { - itemId: artifact.id - } - }); - - // Select tab - $timeout(function() { - CaseTabsSrv.activateTab(observableName); - $('html,body').animate({scrollTop: $('body').offset().top}, 'fast'); - }, 0); - $scope.initScope = function (artifact) { var promise = $scope.analysisEnabled ? AnalyzerSrv.forDataType(artifact.dataType) : $q.resolve([]); @@ -171,14 +154,6 @@ }); }; - CaseArtifactSrv.similar(observableId, { - range: 'all', - sort: ['-startDate'] - }).then(function(response) { - $scope.similarArtifacts = response; - }); - - $scope.openArtifact = function (a) { $state.go('app.case.observables-item', { caseId: a.stats['case']._id, @@ -214,6 +189,10 @@ }) .then(function(response) { $scope.artifact = response.toJSON(); + + if(fieldName === 'ignoreSimilarity' && !!!newValue) { + $scope.getSimilarity(); + } }) .catch(function (response) { NotificationSrv.error('ObservableDetails', response.data, response.status); @@ -367,8 +346,39 @@ }); }; + $scope.getSimilarity = function() { + CaseArtifactSrv.similar(observableId, { + range: 'all', + sort: ['-startDate'] + }).then(function(response) { + $scope.similarArtifacts = response; + }); + }; + this.$onInit = function () { + // Add tab + CaseTabsSrv.addTab(observableName, { + name: observableName, + label: artifact.data || artifact.attachment.name, + closable: true, + state: 'app.case.observables-item', + params: { + itemId: artifact.id + } + }); + + // Select tab + $timeout(function() { + CaseTabsSrv.activateTab(observableName); + $('html,body').animate({scrollTop: $('body').offset().top}, 'fast'); + }, 0); + + // Fetch similar cases + if(!$scope.artifact.ignoreSimilarity) { + $scope.getSimilarity(); + } + if(SecuritySrv.checkPermissions(['manageShare'], $scope.userPermissions)) { $scope.loadShares(); } diff --git a/frontend/app/views/partials/observables/details/summary.html b/frontend/app/views/partials/observables/details/summary.html index 3a5e168b0d..b92960cb92 100644 --- a/frontend/app/views/partials/observables/details/summary.html +++ b/frontend/app/views/partials/observables/details/summary.html @@ -78,6 +78,20 @@

      +
      +
      Ignored for similarity
      +
      + + + +
      +
      + + + +
      +
      +
      Tags
      @@ -101,8 +115,9 @@

      -
      +

      Links

      +
      This observable has not been seen in any other case
      From 1dc0a4fe1c9758b5db244130c1b22c0245434b33 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 8 Nov 2020 08:47:35 +0100 Subject: [PATCH 147/237] #1625 Filter observable list using ignoreSimilarity icon clicks --- frontend/app/views/partials/observables/list/observables.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/views/partials/observables/list/observables.html b/frontend/app/views/partials/observables/list/observables.html index 1d852c7e4d..caaceb3baa 100644 --- a/frontend/app/views/partials/observables/list/observables.html +++ b/frontend/app/views/partials/observables/list/observables.html @@ -73,7 +73,7 @@

      - + Date: Sun, 8 Nov 2020 12:28:03 +0100 Subject: [PATCH 148/237] #1629 Display thehive version in login page --- frontend/app/scripts/controllers/AuthenticationCtrl.js | 2 ++ frontend/app/views/login.html | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/app/scripts/controllers/AuthenticationCtrl.js b/frontend/app/scripts/controllers/AuthenticationCtrl.js index 7b9685e601..30d0d2867b 100644 --- a/frontend/app/scripts/controllers/AuthenticationCtrl.js +++ b/frontend/app/scripts/controllers/AuthenticationCtrl.js @@ -5,6 +5,8 @@ 'use strict'; angular.module('theHiveControllers') .controller('AuthenticationCtrl', function($rootScope, $scope, $state, $location, $uibModalStack, $stateParams, AuthenticationSrv, NotificationSrv, UtilsSrv, UrlParser, appConfig) { + $scope.version = appConfig.versions.TheHive; + $scope.params = { requireMfa: false }; diff --git a/frontend/app/views/login.html b/frontend/app/views/login.html index efb70a376b..d4dfdd4229 100644 --- a/frontend/app/views/login.html +++ b/frontend/app/views/login.html @@ -44,5 +44,7 @@

      - +
      + Version: {{version}} +
      From 5ea53a9f2c2b639d5af1872b5f14e5a533426e85 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 8 Nov 2020 12:31:25 +0100 Subject: [PATCH 149/237] #1625 Redesign the observable flag info in the list view --- frontend/app/index.html | 1 + .../common/observable-flags.component.js | 21 ++++++++ frontend/app/styles/main.css | 4 ++ .../common/observable-flags.component.html | 49 +++++++++++++++++++ .../observables/list/observables.html | 23 +-------- 5 files changed, 76 insertions(+), 22 deletions(-) create mode 100644 frontend/app/scripts/components/common/observable-flags.component.js create mode 100644 frontend/app/views/components/common/observable-flags.component.html diff --git a/frontend/app/index.html b/frontend/app/index.html index 2ed1e175d3..ecd05d8261 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -133,6 +133,7 @@ + diff --git a/frontend/app/scripts/components/common/observable-flags.component.js b/frontend/app/scripts/components/common/observable-flags.component.js new file mode 100644 index 0000000000..391331b2be --- /dev/null +++ b/frontend/app/scripts/components/common/observable-flags.component.js @@ -0,0 +1,21 @@ +(function() { + 'use strict'; + + angular.module('theHiveComponents') + .component('observableFlags', { + controller: function() { + this.filterBy = function(fieldName, newValue) { + this.onFilter({ + fieldName: fieldName, + value: newValue + }); + }; + }, + controllerAs: '$ctrl', + templateUrl: 'views/components/common/observable-flags.component.html', + bindings: { + observable: '<', + onFilter: '&' + } + }); +})(); diff --git a/frontend/app/styles/main.css b/frontend/app/styles/main.css index c2aa162bbd..050fbc4e9a 100644 --- a/frontend/app/styles/main.css +++ b/frontend/app/styles/main.css @@ -380,6 +380,10 @@ ul.observable-reports-summary li { background: #f5f5f5; } +.text-disabled { + color: #dfdfdf; +} + .text-yellow { color: rgb(231, 147, 0); } diff --git a/frontend/app/views/components/common/observable-flags.component.html b/frontend/app/views/components/common/observable-flags.component.html new file mode 100644 index 0000000000..699d749fd6 --- /dev/null +++ b/frontend/app/views/components/common/observable-flags.component.html @@ -0,0 +1,49 @@ +
      + +
      + +
      +
      + +
      + + +
      + +
      +
      + +
      + + +
      + +
      +
      + +
      + + +
      + +
      +
      + +
      + + + + + +
      diff --git a/frontend/app/views/partials/observables/list/observables.html b/frontend/app/views/partials/observables/list/observables.html index caaceb3baa..b13bbfa2d2 100644 --- a/frontend/app/views/partials/observables/list/observables.html +++ b/frontend/app/views/partials/observables/list/observables.html @@ -29,9 +29,6 @@

      - - - Type @@ -67,21 +64,7 @@

      - - - - - - - - - - + @@ -89,10 +72,6 @@

      - {{(artifact.data | fang) || (artifact.attachment.name | fang)}} From cfb9009e067d2586195151cf24faf1654fd95546 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 8 Nov 2020 12:54:13 +0100 Subject: [PATCH 150/237] #1625 Update the observable-label component to allo inline display --- .../scripts/components/common/observable-flags.component.js | 1 + frontend/app/styles/main.css | 4 ++++ .../views/components/alert/observable-list.component.html | 5 +---- .../views/components/common/observable-flags.component.html | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/app/scripts/components/common/observable-flags.component.js b/frontend/app/scripts/components/common/observable-flags.component.js index 391331b2be..9bb6c344ce 100644 --- a/frontend/app/scripts/components/common/observable-flags.component.js +++ b/frontend/app/scripts/components/common/observable-flags.component.js @@ -15,6 +15,7 @@ templateUrl: 'views/components/common/observable-flags.component.html', bindings: { observable: '<', + inline: '<', onFilter: '&' } }); diff --git a/frontend/app/styles/main.css b/frontend/app/styles/main.css index 050fbc4e9a..842300eac5 100644 --- a/frontend/app/styles/main.css +++ b/frontend/app/styles/main.css @@ -723,3 +723,7 @@ table tr td.task-actions span.action-button { background-color: #8a0000 !important; /*background: rgba(0,0,0,0.2) !important;*/ } + +.observable-flags.inline div { + display: inline; +} diff --git a/frontend/app/views/components/alert/observable-list.component.html b/frontend/app/views/components/alert/observable-list.component.html index 21abc48972..c9776b6ee3 100644 --- a/frontend/app/views/components/alert/observable-list.component.html +++ b/frontend/app/views/components/alert/observable-list.component.html @@ -33,10 +33,7 @@ - - - - + {{observable.dataType}} diff --git a/frontend/app/views/components/common/observable-flags.component.html b/frontend/app/views/components/common/observable-flags.component.html index 699d749fd6..704858f0ca 100644 --- a/frontend/app/views/components/common/observable-flags.component.html +++ b/frontend/app/views/components/common/observable-flags.component.html @@ -1,4 +1,4 @@ -
      +
      From 85908d4aa3d484f6b6a3fd1413c1bf0b1d788ee8 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 8 Nov 2020 13:10:27 +0100 Subject: [PATCH 151/237] #1290 Fix menu toggle icon and settings menu items --- frontend/app/styles/main.css | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frontend/app/styles/main.css b/frontend/app/styles/main.css index 842300eac5..be89ac44a3 100644 --- a/frontend/app/styles/main.css +++ b/frontend/app/styles/main.css @@ -727,3 +727,13 @@ table tr td.task-actions span.action-button { .observable-flags.inline div { display: inline; } + +@media (max-width: 991px) { + .main-header .navbar-right a { + color: #777; + } +} + +.icon-bar { + background-color: #FFF; +} From 8d5e14557ddcf6ff32c5f414cd873f2d4a4c523b Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 9 Nov 2020 09:15:03 +0100 Subject: [PATCH 152/237] #1625 Fix case link with ignoreSimilarity --- thehive/app/org/thp/thehive/services/CaseSrv.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index dfcc74af4c..710d0f2502 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -79,9 +79,10 @@ class CaseSrv @Inject() ( } _ <- createdTasks.toTry(t => shareSrv.shareTask(t, createdCase, organisation)) - caseTemplateCf = caseTemplate - .fold[Seq[RichCustomField]](Seq())(_.customFields) - .map(cf => InputCustomFieldValue(cf.name, cf.value, cf.order)) + caseTemplateCf = + caseTemplate + .fold[Seq[RichCustomField]](Seq())(_.customFields) + .map(cf => InputCustomFieldValue(cf.name, cf.value, cf.order)) cfs <- cleanCustomFields(caseTemplateCf, customFields).toTry { case InputCustomFieldValue(name, value, order) => createCustomField(createdCase, EntityIdOrName(name), value, order) } @@ -470,9 +471,11 @@ object CaseOps { traversal .as(originCaseLabel) .observables + .hasNot(_.ignoreSimilarity, true) .as(observableLabel) .data .observables + .hasNot(_.ignoreSimilarity, true) .shares .filter(_.organisation.current) .`case` From 92fec30dbd93f43e7b3fa78b9cc928cd3f67842c Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 9 Nov 2020 09:21:40 +0100 Subject: [PATCH 153/237] #1625 Display the ignoreSimilarity icon even for observable without that flag defined --- .../app/views/partials/observables/details/summary.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/app/views/partials/observables/details/summary.html b/frontend/app/views/partials/observables/details/summary.html index b92960cb92..ffaef9822b 100644 --- a/frontend/app/views/partials/observables/details/summary.html +++ b/frontend/app/views/partials/observables/details/summary.html @@ -81,13 +81,13 @@

      Ignored for similarity
      - - + +
      - +
      From 67f5ac7680231b91ef35e4f41ed05211203d3d8c Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 9 Nov 2020 09:41:11 +0100 Subject: [PATCH 154/237] #1625 Use observable-flags component in case links table --- .../components/common/observable-flags.component.js | 2 ++ .../components/common/observable-flags.component.html | 8 ++++---- frontend/app/views/partials/case/case.links.html | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/frontend/app/scripts/components/common/observable-flags.component.js b/frontend/app/scripts/components/common/observable-flags.component.js index 9bb6c344ce..4c02c4941b 100644 --- a/frontend/app/scripts/components/common/observable-flags.component.js +++ b/frontend/app/scripts/components/common/observable-flags.component.js @@ -16,6 +16,8 @@ bindings: { observable: '<', inline: '<', + hideSimilarity: '<', + hideSeen: '<', onFilter: '&' } }); diff --git a/frontend/app/views/components/common/observable-flags.component.html b/frontend/app/views/components/common/observable-flags.component.html index 704858f0ca..45d1c6a45a 100644 --- a/frontend/app/views/components/common/observable-flags.component.html +++ b/frontend/app/views/components/common/observable-flags.component.html @@ -20,19 +20,19 @@

      -
      +
      -
      +
      -
      -
      diff --git a/frontend/app/views/partials/case/case.links.html b/frontend/app/views/partials/case/case.links.html index e3bccb7aa3..ba0d6532fc 100644 --- a/frontend/app/views/partials/case/case.links.html +++ b/frontend/app/views/partials/case/case.links.html @@ -93,7 +93,8 @@
      - + + [{{observable.dataType}}]: {{observable.attachment.name}} From db7f789e4605717591209bdd80db8f5e86a267d5 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 9 Nov 2020 16:47:38 +0100 Subject: [PATCH 155/237] #1625 Add ignoreSimilarity to observable creation dialog --- .../controllers/case/ObservableCreationCtrl.js | 2 ++ .../app/views/partials/observables/creation/form.html | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/frontend/app/scripts/controllers/case/ObservableCreationCtrl.js b/frontend/app/scripts/controllers/case/ObservableCreationCtrl.js index 0aa20047b3..03a3548272 100644 --- a/frontend/app/scripts/controllers/case/ObservableCreationCtrl.js +++ b/frontend/app/scripts/controllers/case/ObservableCreationCtrl.js @@ -13,6 +13,7 @@ $scope.params = params || { ioc: false, sighted: false, + ignoreSimilarity: false, single: false, isUpload: true, isZip: false, @@ -74,6 +75,7 @@ dataType: params.dataType, ioc: params.ioc, sighted: params.sighted, + ignoreSimilarity: params.ignoreSimilarity, tlp: params.tlp, message: params.message, tags: _.unique(_.pluck(params.tags, 'text')) diff --git a/frontend/app/views/partials/observables/creation/form.html b/frontend/app/views/partials/observables/creation/form.html index aedf952384..43de4393ac 100644 --- a/frontend/app/views/partials/observables/creation/form.html +++ b/frontend/app/views/partials/observables/creation/form.html @@ -112,6 +112,17 @@
      +
      + +
      +

      + + + +

      +
      +
      +
      diff --git a/frontend/app/views/partials/alert/list/filters.html b/frontend/app/views/partials/alert/list/filters.html index d2c32b6cc1..12c948d197 100644 --- a/frontend/app/views/partials/alert/list/filters.html +++ b/frontend/app/views/partials/alert/list/filters.html @@ -12,7 +12,7 @@

      Filters

      diff --git a/frontend/app/views/partials/case/list/filters.html b/frontend/app/views/partials/case/list/filters.html index b7e40c6786..431c826ca5 100644 --- a/frontend/app/views/partials/case/list/filters.html +++ b/frontend/app/views/partials/case/list/filters.html @@ -11,7 +11,7 @@

      Filters

      diff --git a/frontend/app/views/partials/observables/list/filters.html b/frontend/app/views/partials/observables/list/filters.html index 1cb79dbb14..ffc4faaaa5 100644 --- a/frontend/app/views/partials/observables/list/filters.html +++ b/frontend/app/views/partials/observables/list/filters.html @@ -11,7 +11,7 @@

      Filters

      diff --git a/frontend/app/views/partials/search/list.html b/frontend/app/views/partials/search/list.html index df29b01e9c..8782940eae 100644 --- a/frontend/app/views/partials/search/list.html +++ b/frontend/app/views/partials/search/list.html @@ -51,7 +51,7 @@

      Search filters {{config[config.enti

      From 28c3b78ab6cf488bb254d7d4c2309514aff40c3d Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Tue, 10 Nov 2020 06:34:47 +0100 Subject: [PATCH 159/237] #1611 Add tag autocomplete in filter forms --- .../directives/dashboard/filter-editor.js | 11 +++- frontend/app/scripts/services/api/TagSrv.js | 53 +++++++++++++------ .../directives/dashboard/filter-editor.html | 21 ++++++++ 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/frontend/app/scripts/directives/dashboard/filter-editor.js b/frontend/app/scripts/directives/dashboard/filter-editor.js index def05a2327..9dbf55da66 100644 --- a/frontend/app/scripts/directives/dashboard/filter-editor.js +++ b/frontend/app/scripts/directives/dashboard/filter-editor.js @@ -1,6 +1,6 @@ (function() { 'use strict'; - angular.module('theHiveDirectives').directive('filterEditor', function($q, AuthenticationSrv, UserSrv, UtilsSrv) { + angular.module('theHiveDirectives').directive('filterEditor', function($q, AuthenticationSrv, UserSrv, TagSrv, UtilsSrv) { return { restrict: 'E', scope: { @@ -50,6 +50,11 @@ if(!field) { return; } + + if(field.name === 'tags') { + return field.name; + } + var type = field.type; if ((type === 'string' || type === 'number' || type === 'integer' || type === 'float' ) && field.values.length > 0) { @@ -64,7 +69,9 @@ var promise = null; - if(field.type === 'user') { + if(field.name === 'tags') { + return TagSrv.getTagsFor(scope.entity, query); + } else if(field.type === 'user') { promise = AuthenticationSrv.current() .then(function(user) { return UserSrv.autoComplete(user.organisation, query); diff --git a/frontend/app/scripts/services/api/TagSrv.js b/frontend/app/scripts/services/api/TagSrv.js index 9c1a95656e..d90300a3ca 100644 --- a/frontend/app/scripts/services/api/TagSrv.js +++ b/frontend/app/scripts/services/api/TagSrv.js @@ -3,27 +3,31 @@ angular.module('theHiveServices') .service('TagSrv', function(QuerySrv, $q) { + var self = this; + var getTags = function(objectType, term) { var defer = $q.defer(); - var operations = [ - { _name: 'listTag' }, - { _name: objectType }, - { - _name: 'filter', - _like: { - _field: 'text', - _value: '*' + term + '*' - } - }, - { - _name: 'text' - } + { _name: 'listTag' } ]; + if(objectType) { + operations.push({ _name: objectType }); + } + + operations.push({ + _name: 'filter', + _like: { + _field: 'text', + _value: '*' + term + '*' + } + }); + + operations.push({ _name: 'text' }); + // Get the list - QuerySrv.call('v0', operations) - .then(function(data) { + QuerySrv.call('v0', operations, {name: 'tags-auto-complete'}) + .then(function(data) { defer.resolve(_.map(_.unique(data), function(tag) { return {text: tag}; })); @@ -32,6 +36,21 @@ return defer.promise; }; + this.getTagsFor = function(entity, query) { + + switch(entity) { + case 'case': + return self.fromCases(query); + case 'case_artifact': + return self.fromObservables(query); + case 'alert': + return self.fromAlerts(query); + default: + return self.getTags(undefined, query); + } + + }; + this.fromCases = function(term) { return getTags('fromCase', term); }; @@ -40,5 +59,9 @@ return getTags('fromObservable', term); }; + this.fromAlerts = function(term) { + return getTags('fromAlert', term); + }; + }); })(); diff --git a/frontend/app/views/directives/dashboard/filter-editor.html b/frontend/app/views/directives/dashboard/filter-editor.html index cb2b6dcb61..8c18989202 100644 --- a/frontend/app/views/directives/dashboard/filter-editor.html +++ b/frontend/app/views/directives/dashboard/filter-editor.html @@ -21,6 +21,27 @@
      +
      +
      + + +
      +
      + + + +
      +
      +
      + + +
      (Closed at {{currentCase.endDate | showDate}} as {{$vm.CaseResolutionStatus[currentCase.resolutionStatus]}}) From d327736f0577674b196ec8529d82427cac800f68 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Tue, 10 Nov 2020 18:52:51 +0100 Subject: [PATCH 171/237] #1625 Fix typo --- frontend/app/views/partials/observables/observable.update.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/views/partials/observables/observable.update.html b/frontend/app/views/partials/observables/observable.update.html index 71e2ba7b67..948bf59ad4 100644 --- a/frontend/app/views/partials/observables/observable.update.html +++ b/frontend/app/views/partials/observables/observable.update.html @@ -42,7 +42,7 @@

      Update observable(s)

      - +

      From 4bfe9414b3cdbe07eaf1370c14cfcb3d1b2d9e1f Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Tue, 10 Nov 2020 23:03:20 +0100 Subject: [PATCH 172/237] #1637 Add a renderer to custom fields in list views --- frontend/app/index.html | 1 + .../app/scripts/filters/custom-field-value.js | 17 +++++++++++++++++ frontend/app/styles/label.css | 18 ------------------ .../common/custom-field-labels.component.html | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 frontend/app/scripts/filters/custom-field-value.js diff --git a/frontend/app/index.html b/frontend/app/index.html index b6a370d2b7..dc172a5f24 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -251,6 +251,7 @@ + diff --git a/frontend/app/scripts/filters/custom-field-value.js b/frontend/app/scripts/filters/custom-field-value.js new file mode 100644 index 0000000000..583749dabd --- /dev/null +++ b/frontend/app/scripts/filters/custom-field-value.js @@ -0,0 +1,17 @@ +(function() { + 'use strict'; + angular.module('theHiveFilters').filter('customFieldValue', function() { + return function(customField) { + if(!customField) { + return ''; + } + + switch(customField.type) { + case 'date': + return moment(customField.value).format('MM/DD/YY H:mm'); + default: + return customField.value; + } + }; + }); +})(); diff --git a/frontend/app/styles/label.css b/frontend/app/styles/label.css index f1522349a6..2aeadea342 100644 --- a/frontend/app/styles/label.css +++ b/frontend/app/styles/label.css @@ -1,13 +1,11 @@ .double-val-label { display: table; - /* margin: .2em .6em .3em; */ margin-top: 0; margin-left: 0; } .double-val-label>span { color: #ffffff; display: table-cell; - /* font-size: 0.9em; */ font-weight: 400; line-height: 1; padding: .3em .6em; @@ -25,10 +23,7 @@ .double-val-label>span:last-child { border-bottom-right-radius: 0.25em; border-top-right-radius: .25em; - /* border-left: 1px dashed #777; */ border-left: 1px dashed #3c8dbc; - /* background-color: #d2d6de; - color: #444; */ } .double-val-label>span.primary { background-color: #3c8dbc; @@ -38,16 +33,3 @@ background-color: #d2d6de; color: #444; } -/* -.double-val-label>span.success { - background-color: #5cb85c; -} -.double-val-label>span.info { - background-color: #5bc0de; -} -.double-val-label>span.warning { - background-color: #f0ad4e; -} -.double-val-label>span.danger { - background-color: #d9534f; -} */ diff --git a/frontend/app/views/components/common/custom-field-labels.component.html b/frontend/app/views/components/common/custom-field-labels.component.html index cf227347e4..b374c4e590 100644 --- a/frontend/app/views/components/common/custom-field-labels.component.html +++ b/frontend/app/views/components/common/custom-field-labels.component.html @@ -9,6 +9,6 @@ ng-if="cf.value !== undefined && cf.value !== null"> {{$cmp.fieldsCache[cf.name].name || cf.name}} - {{cf.value}} + {{cf | customFieldValue}}

      From b37a15afbad1fc5b03169c9da3e71e6bfcc1c790 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 11 Nov 2020 09:24:06 +0100 Subject: [PATCH 173/237] #1637 Add support to integer and float filter values --- frontend/app/scripts/services/common/ui/FilteringSrv.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/app/scripts/services/common/ui/FilteringSrv.js b/frontend/app/scripts/services/common/ui/FilteringSrv.js index d4cb37e0fe..fcc4ce0ec9 100644 --- a/frontend/app/scripts/services/common/ui/FilteringSrv.js +++ b/frontend/app/scripts/services/common/ui/FilteringSrv.js @@ -182,6 +182,13 @@ case 'boolean': filter.value = value; break; + case 'integer': + case 'float': + filter.value = { + operator: '=', + value: value + }; + break; case 'user': break; } From 117f1d6bd072631db0d07f95a5a76f9b7e1115fc Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 11 Nov 2020 10:55:01 +0100 Subject: [PATCH 174/237] #1637 Add a option to toggle displaying custom fields in alert and case lists --- frontend/app/scripts/controllers/alert/AlertListCtrl.js | 4 ++++ frontend/app/scripts/controllers/case/CaseListCtrl.js | 4 ++++ frontend/app/scripts/services/common/ui/FilteringSrv.js | 6 ++++++ frontend/app/views/partials/alert/list.html | 2 +- frontend/app/views/partials/alert/list/toolbar.html | 6 ++++++ frontend/app/views/partials/case/case.list.html | 2 +- frontend/app/views/partials/case/list/toolbar.html | 8 +++++++- 7 files changed, 29 insertions(+), 3 deletions(-) diff --git a/frontend/app/scripts/controllers/alert/AlertListCtrl.js b/frontend/app/scripts/controllers/alert/AlertListCtrl.js index 8c7d322cf2..13f80ce024 100755 --- a/frontend/app/scripts/controllers/alert/AlertListCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertListCtrl.js @@ -83,6 +83,10 @@ this.filtering.toggleFilters(); }; + this.toggleAdvanced = function () { + this.filtering.toggleAdvanced(); + }; + this.canMarkAsRead = AlertingSrv.canMarkAsRead; this.canMarkAsUnread = AlertingSrv.canMarkAsUnread; diff --git a/frontend/app/scripts/controllers/case/CaseListCtrl.js b/frontend/app/scripts/controllers/case/CaseListCtrl.js index 93e0e1a5bd..5d6f22f2ee 100644 --- a/frontend/app/scripts/controllers/case/CaseListCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseListCtrl.js @@ -168,6 +168,10 @@ this.filtering.toggleFilters(); }; + this.toggleAdvanced = function () { + this.filtering.toggleAdvanced(); + }; + this.filter = function () { self.filtering.filter().then(this.applyFilters); }; diff --git a/frontend/app/scripts/services/common/ui/FilteringSrv.js b/frontend/app/scripts/services/common/ui/FilteringSrv.js index fcc4ce0ec9..921c8edf8e 100644 --- a/frontend/app/scripts/services/common/ui/FilteringSrv.js +++ b/frontend/app/scripts/services/common/ui/FilteringSrv.js @@ -36,6 +36,7 @@ state: state, showFilters: self.defaults.showFilters || false, showStats: self.defaults.showStats || false, + showAdvanced: self.defaults.showAdvanced || true, pageSize: self.defaults.pageSize || 15, sort: self.defaults.sort || [], filters: self.defaultFilter || [] @@ -106,6 +107,11 @@ self.storeContext(); }; + this.toggleAdvanced = function() { + self.context.showAdvanced = !self.context.showAdvanced; + self.storeContext(); + }; + this.setPageSize = function(pageSize) { self.context.pageSize = pageSize; self.storeContext(); diff --git a/frontend/app/views/partials/alert/list.html b/frontend/app/views/partials/alert/list.html index 224dadefcd..6fbbf1fe8d 100644 --- a/frontend/app/views/partials/alert/list.html +++ b/frontend/app/views/partials/alert/list.html @@ -175,7 +175,7 @@

      List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList - + diff --git a/frontend/app/views/partials/alert/list/toolbar.html b/frontend/app/views/partials/alert/list/toolbar.html index 52342a720e..bc49d48021 100644 --- a/frontend/app/views/partials/alert/list/toolbar.html +++ b/frontend/app/views/partials/alert/list/toolbar.html @@ -102,6 +102,12 @@ Stats

      + +
      + +
      diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index c9c29928b2..e86b33b0fc 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -76,7 +76,7 @@

      List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount}

      - +
      diff --git a/frontend/app/views/partials/case/list/toolbar.html b/frontend/app/views/partials/case/list/toolbar.html index dbcffd29b2..8c6363308e 100644 --- a/frontend/app/views/partials/case/list/toolbar.html +++ b/frontend/app/views/partials/case/list/toolbar.html @@ -76,10 +76,16 @@
      -
      + +
      + +
      From 1acea7a8b8af71564412eeac8482521e13966204 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 11 Nov 2020 11:09:41 +0100 Subject: [PATCH 175/237] #1637 Fix dispplay of custom fields labels based on defined value --- .../components/common/custom-field-labels.component.js | 6 ++++++ .../components/common/custom-field-labels.component.html | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/app/scripts/components/common/custom-field-labels.component.js b/frontend/app/scripts/components/common/custom-field-labels.component.js index 3cc65868f5..68bc78a344 100644 --- a/frontend/app/scripts/components/common/custom-field-labels.component.js +++ b/frontend/app/scripts/components/common/custom-field-labels.component.js @@ -7,6 +7,7 @@ var self = this; this.fieldsCache = {}; + this.definedValues = 0; this.fieldClicked = function(fieldName, newValue) { this.onFieldClick({ name: fieldName, @@ -18,6 +19,11 @@ CustomFieldsSrv.all().then(function(fields) { self.fieldsCache = fields; }); + + self.definedValues = (_.filter(self.customFields, function(item) { + return item.value !== null && item.value !== undefined; + }) || []).length; + }; }, controllerAs: '$cmp', diff --git a/frontend/app/views/components/common/custom-field-labels.component.html b/frontend/app/views/components/common/custom-field-labels.component.html index b374c4e590..0845ab7043 100644 --- a/frontend/app/views/components/common/custom-field-labels.component.html +++ b/frontend/app/views/components/common/custom-field-labels.component.html @@ -1,7 +1,7 @@
      - None + None Date: Wed, 11 Nov 2020 19:22:30 +0100 Subject: [PATCH 176/237] #1641 Add chart representations statistics sections in case/alert/observable pages --- frontend/app/index.html | 2 + .../charts/donut-chart.component.js | 56 ++++++++++++++++++ .../components/list/stats-item.component.js | 28 +++++++++ .../controllers/alert/AlertStatsCtrl.js | 10 ++++ .../controllers/case/ObservablesStatsCtrl.js | 10 ++++ .../app/scripts/directives/charts/c3Chart.js | 6 +- frontend/app/styles/filters.css | 15 +++++ .../components/list/stats-item.component.html | 23 ++++++++ frontend/app/views/directives/charts/c3.html | 2 +- frontend/app/views/partials/alert/list.html | 8 +-- .../views/partials/alert/list/mini-stats.html | 49 ++++------------ .../app/views/partials/case/case.list.html | 6 +- .../views/partials/case/case.observables.html | 10 +--- .../views/partials/case/list/mini-stats.html | 57 +++---------------- .../partials/observables/list/mini-stats.html | 57 +++---------------- 15 files changed, 187 insertions(+), 152 deletions(-) create mode 100644 frontend/app/scripts/components/charts/donut-chart.component.js create mode 100644 frontend/app/scripts/components/list/stats-item.component.js create mode 100644 frontend/app/views/components/list/stats-item.component.html diff --git a/frontend/app/index.html b/frontend/app/index.html index dc172a5f24..5460192e47 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -133,11 +133,13 @@ + + diff --git a/frontend/app/scripts/components/charts/donut-chart.component.js b/frontend/app/scripts/components/charts/donut-chart.component.js new file mode 100644 index 0000000000..909d394faa --- /dev/null +++ b/frontend/app/scripts/components/charts/donut-chart.component.js @@ -0,0 +1,56 @@ +(function() { + 'use strict'; + + angular.module('theHiveComponents') + .component('donutChart', { + controller: function($scope) { + var self = this; + + this.$onInit = function() { + $scope.$watch('$cmp.data', function (data) { + self.updateChart(data); + }); + }; + + this.updateChart = function(rawData) { + this.error = false; + + var data = _.map(rawData, function(item) { + return [item.key, item.count]; + }); + + this.chart = { + data: { + columns: data, + names: self.labels || undefined, + type: 'donut', + onclick: function(d) { + if(self.onItemClicked) { + self.onItemClicked({ + value: d.id + }); + } + } + }, + donut: { + label: { + show: false + } + }, + legend: { + position: 'right' + } + }; + }; + }, + controllerAs: '$cmp', + template: '', + bindings: { + data: '<', + labels: '<', + title: '@', + field: '@', + onItemClicked: '&' + } + }); +})(); diff --git a/frontend/app/scripts/components/list/stats-item.component.js b/frontend/app/scripts/components/list/stats-item.component.js new file mode 100644 index 0000000000..1e7dedb8f9 --- /dev/null +++ b/frontend/app/scripts/components/list/stats-item.component.js @@ -0,0 +1,28 @@ +(function() { + 'use strict'; + angular.module('theHiveComponents') + .component('statsItem', { + controller: function() { + var self = this; + + this.onClick = function(value) { + self.onItemClicked({ + field: self.field, + value: self.values ? self.values[value] : value + }); + }; + }, + controllerAs: '$cmp', + templateUrl: 'views/components/list/stats-item.component.html', + replace: true, + bindings: { + field: '@', + title: '@', + data: '<', + mode: '<', + labels: '<', + values: '<', + onItemClicked: '&' + } + }); +})(); diff --git a/frontend/app/scripts/controllers/alert/AlertStatsCtrl.js b/frontend/app/scripts/controllers/alert/AlertStatsCtrl.js index 4883fb8ff1..a05d92b580 100644 --- a/frontend/app/scripts/controllers/alert/AlertStatsCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertStatsCtrl.js @@ -14,6 +14,16 @@ this.byStatus = {}; this.byTags = {}; + this.readAlerts = { + 'true': 'Read', + 'false': 'Unread' + }; + + this.readValues = { + 'true': true, + 'false': false + }; + self.$onInit = function() { // Get stats by tags diff --git a/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js b/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js index 59a22251af..e48194916b 100644 --- a/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js +++ b/frontend/app/scripts/controllers/case/ObservablesStatsCtrl.js @@ -12,6 +12,16 @@ this.byIoc = {}; this.byTags = {}; + this.iocLabels = { + 'true': 'IOC', + 'false': 'Not IOC' + }; + + this.iocValues = { + 'true': true, + 'false': false + }; + self.$onInit = function() { var caseId = $stateParams.caseId; diff --git a/frontend/app/scripts/directives/charts/c3Chart.js b/frontend/app/scripts/directives/charts/c3Chart.js index 92c296b7b2..467ef621c4 100644 --- a/frontend/app/scripts/directives/charts/c3Chart.js +++ b/frontend/app/scripts/directives/charts/c3Chart.js @@ -8,6 +8,8 @@ chart: '=', resizeOn: '@', error: '=', + height: ' +
      +

      {{$cmp.title}}

      +
      +
      + + +
      +
      +
      +
      + + + + + +
      {{$cmp.labels[item.key] || item.key}} + {{item.count}} +
      + +
      +
      diff --git a/frontend/app/views/directives/charts/c3.html b/frontend/app/views/directives/charts/c3.html index e66667cc10..109179d500 100644 --- a/frontend/app/views/directives/charts/c3.html +++ b/frontend/app/views/directives/charts/c3.html @@ -7,7 +7,7 @@

      -
      +
      CSV Image Save as diff --git a/frontend/app/views/partials/alert/list.html b/frontend/app/views/partials/alert/list.html index 6fbbf1fe8d..b87e43732c 100644 --- a/frontend/app/views/partials/alert/list.html +++ b/frontend/app/views/partials/alert/list.html @@ -7,12 +7,12 @@

      List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList
      -
      +
      -
      +
      -
      +
      List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList
      -
      +
      No records
      diff --git a/frontend/app/views/partials/alert/list/mini-stats.html b/frontend/app/views/partials/alert/list/mini-stats.html index 183fb1725f..6db30f8742 100644 --- a/frontend/app/views/partials/alert/list/mini-stats.html +++ b/frontend/app/views/partials/alert/list/mini-stats.html @@ -1,52 +1,23 @@
      -
      -

      Statistics

      -
      -
      -
      Alerts by Status
      -
      - - - - - -
      {{item.key === 'true' ? 'Read' : 'Unread'}} - {{item.count}} -
      -
      +
      + + +
      -
      -
      Top 5 Types
      -
      - - - - - -
      {{item.key}} - {{item.count}} -
      -
      +
      + +
      -
      -
      Top 5 tags
      -
      - - - - - -
      {{item.key}} - {{item.count}} -
      -
      +
      + +
      diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index e86b33b0fc..36e3c63db6 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -14,11 +14,11 @@

      List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount}
      -
      +
      -
      +
      -
      +
      -
      +
      -
      -
      +
      -
      - -
      -
      +
      diff --git a/frontend/app/views/partials/case/list/mini-stats.html b/frontend/app/views/partials/case/list/mini-stats.html index 7cc02b8da2..af698a4948 100644 --- a/frontend/app/views/partials/case/list/mini-stats.html +++ b/frontend/app/views/partials/case/list/mini-stats.html @@ -1,61 +1,22 @@
      -
      -

      Statistics

      -
      -
      -
      Cases by Status
      -
      - - - - - -
      {{item.key}} - {{item.count}} -
      -
      +
      + +
      -
      -
      Case by Resolution
      -
      - - - - - -
      {{item.key}} - {{item.count}} -
      -
      +
      + +
      -
      -
      Top 5 tags
      -
      - - - - - -
      {{item.key}} - {{item.count}} -
      -
      +
      + +
      - diff --git a/frontend/app/views/partials/observables/list/mini-stats.html b/frontend/app/views/partials/observables/list/mini-stats.html index ca1b0c66fa..4e2b789c52 100644 --- a/frontend/app/views/partials/observables/list/mini-stats.html +++ b/frontend/app/views/partials/observables/list/mini-stats.html @@ -1,61 +1,22 @@
      -
      -

      Statistics

      -
      -
      -
      Observables by type
      -
      - - - - - -
      {{item.key}} - {{item.count}} -
      -
      +
      + +
      -
      -
      Observables as IOC
      -
      - - - - - -
      {{(item.key === 'false') ? 'Not IOC' : 'IOC' }} - {{item.count}} -
      -
      +
      + +
      -
      -
      Top 10 tags
      -
      - - - - - -
      {{item.key}} - {{item.count}} -
      -
      +
      + +
      - From 8a8153d9801154d94cd568f038f732a377036480 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 11 Nov 2020 19:28:09 +0100 Subject: [PATCH 177/237] #1641 Add a no data message if no stats data are available --- frontend/app/views/components/list/stats-item.component.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/app/views/components/list/stats-item.component.html b/frontend/app/views/components/list/stats-item.component.html index d2882f129e..4fa1100fb5 100644 --- a/frontend/app/views/components/list/stats-item.component.html +++ b/frontend/app/views/components/list/stats-item.component.html @@ -9,6 +9,9 @@

      {{$cmp.title}}

      +
      + No Data +
      @@ -17,7 +20,7 @@

      {{$cmp.title}}

      {{$cmp.labels[item.key] || item.key}}
      -
      From a21c32ed9a4d75a0b44cc0bce073e3ce72d6239c Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 11 Nov 2020 19:38:20 +0100 Subject: [PATCH 178/237] #1641 Clean up --- frontend/app/styles/filters.css | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/frontend/app/styles/filters.css b/frontend/app/styles/filters.css index 58a8443f42..98bce17fd7 100644 --- a/frontend/app/styles/filters.css +++ b/frontend/app/styles/filters.css @@ -41,13 +41,8 @@ text-align: center; } -.stats-panel { - /* padding: 10px; - background-color: #f5f5f5; */ -} - .stats-panel .stats-item-wrapper { - padding: 5px; + padding: 5px; background-color: #f5f5f5; } @@ -55,19 +50,6 @@ margin-bottom: 0; } -/* -.filter-panel hr.filter-separator:after { - content: "OR"; - display: inline-block; - position: relative; - top: -0.8em; - font-size: 1.29em; - padding: 0 0.25em; - background: #f5f5f5; -} -*/ - - .similar-case-list .progress { height: 2px; opacity: 1; From c7eb6c4c9e8615d583cf7d2ee3fd1e33ec151c60 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 12 Nov 2020 06:01:29 +0100 Subject: [PATCH 179/237] #1633 Add tlp to observable flags component and use it in report observables import section --- .../common/observable-flags.component.js | 1 + .../app/scripts/directives/report-observables.js | 4 +++- .../alert/observable-list.component.html | 6 ++---- .../common/observable-flags.component.html | 16 ++++++++++++++++ .../app/views/directives/report-observables.html | 14 ++++++++++---- .../partials/observables/details/summary.html | 13 +++---------- .../partials/observables/list/observables.html | 2 +- 7 files changed, 36 insertions(+), 20 deletions(-) diff --git a/frontend/app/scripts/components/common/observable-flags.component.js b/frontend/app/scripts/components/common/observable-flags.component.js index 4c02c4941b..b8e90936cb 100644 --- a/frontend/app/scripts/components/common/observable-flags.component.js +++ b/frontend/app/scripts/components/common/observable-flags.component.js @@ -18,6 +18,7 @@ inline: '<', hideSimilarity: '<', hideSeen: '<', + hideTlp: '<', onFilter: '&' } }); diff --git a/frontend/app/scripts/directives/report-observables.js b/frontend/app/scripts/directives/report-observables.js index 4f7a73abd0..12c6cc9cf1 100644 --- a/frontend/app/scripts/directives/report-observables.js +++ b/frontend/app/scripts/directives/report-observables.js @@ -49,7 +49,8 @@ $scope.selectAll = function() { var type = $scope.pagination.filter; _.each(type === '' ? $scope.observables : $scope.groups[type], function(item) { - if(!item.id && !item.selected) { + //if(!item.id && !item.selected) { + if(!item.stats.observableId && !item.selected) { item.selected = true; $scope.selected++; } @@ -80,6 +81,7 @@ single: list.length === 1, ioc: false, sighted: false, + ignoreSimilarity: false, tlp: 2, message: message, tags: [{text: 'src:' + $scope.analyzer}] diff --git a/frontend/app/views/components/alert/observable-list.component.html b/frontend/app/views/components/alert/observable-list.component.html index c9776b6ee3..2f88854b66 100644 --- a/frontend/app/views/components/alert/observable-list.component.html +++ b/frontend/app/views/components/alert/observable-list.component.html @@ -22,8 +22,7 @@ - - + @@ -31,9 +30,8 @@ -
      Flags Type Data Date Added
      - + {{observable.dataType}} diff --git a/frontend/app/views/components/common/observable-flags.component.html b/frontend/app/views/components/common/observable-flags.component.html index 45d1c6a45a..5eca732365 100644 --- a/frontend/app/views/components/common/observable-flags.component.html +++ b/frontend/app/views/components/common/observable-flags.component.html @@ -1,4 +1,20 @@
      + +
      + +
      +
      diff --git a/frontend/app/views/directives/report-observables.html b/frontend/app/views/directives/report-observables.html index 53c6f4e103..7b3bf5754e 100644 --- a/frontend/app/views/directives/report-observables.html +++ b/frontend/app/views/directives/report-observables.html @@ -26,8 +26,9 @@ - - + + + @@ -37,9 +38,14 @@ - + +
      FlagsImported Type Data
      - + + + + + {{observable.dataType}} diff --git a/frontend/app/views/partials/observables/details/summary.html b/frontend/app/views/partials/observables/details/summary.html index ffaef9822b..741319e11a 100644 --- a/frontend/app/views/partials/observables/details/summary.html +++ b/frontend/app/views/partials/observables/details/summary.html @@ -127,22 +127,15 @@

      Links

      - - + - + -
      IOCTLPFlags CaseDate addedDate added
      - - - + diff --git a/frontend/app/views/partials/observables/list/observables.html b/frontend/app/views/partials/observables/list/observables.html index b13bbfa2d2..c649bea501 100644 --- a/frontend/app/views/partials/observables/list/observables.html +++ b/frontend/app/views/partials/observables/list/observables.html @@ -64,7 +64,7 @@

      		- + From 4b52d81625523d7940585d59079da4b046e5af61 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 12 Nov 2020 06:41:32 +0100 Subject: [PATCH 180/237] #1642 Improve alert observables tab --- .../alert/AlertObservableListCmp.js | 8 +++- .../alert/observable-list.component.html | 31 ++++++++------- .../components/alert/observables/filters.html | 38 +++++++++++++++++++ .../components/alert/observables/toolbar.html | 16 ++++++++ .../alert/similar-case-list.component.html | 4 +- .../common/observable-flags.component.html | 8 ++-- 6 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 frontend/app/views/components/alert/observables/filters.html create mode 100644 frontend/app/views/components/alert/observables/toolbar.html diff --git a/frontend/app/scripts/components/alert/AlertObservableListCmp.js b/frontend/app/scripts/components/alert/AlertObservableListCmp.js index 4c3106b705..8344ca5c71 100644 --- a/frontend/app/scripts/components/alert/AlertObservableListCmp.js +++ b/frontend/app/scripts/components/alert/AlertObservableListCmp.js @@ -10,7 +10,7 @@ this.filtering = new FilteringSrv('case_artifact', 'alert.dialog.observables', { version: 'v1', defaults: { - showFilters: true, + showFilters: false, showStats: false, pageSize: 15, sort: ['-startDate'] @@ -73,6 +73,12 @@ // this.filter = function () { // self.filtering.filter().then(this.applyFilters); // }; + // + + // Filtering methods + this.toggleFilters = function () { + this.filtering.toggleFilters(); + }; this.clearFilters = function () { this.filtering.clearFilters() diff --git a/frontend/app/views/components/alert/observable-list.component.html b/frontend/app/views/components/alert/observable-list.component.html index 2f88854b66..ef2b49c25a 100644 --- a/frontend/app/views/components/alert/observable-list.component.html +++ b/frontend/app/views/components/alert/observable-list.component.html @@ -1,4 +1,18 @@
      + +
      + +
      + +
      +
      + + +
      +
      +
      @@ -6,17 +20,6 @@
      - - @@ -31,10 +34,10 @@ diff --git a/frontend/app/views/components/alert/observables/filters.html b/frontend/app/views/components/alert/observables/filters.html new file mode 100644 index 0000000000..5b47d4771b --- /dev/null +++ b/frontend/app/views/components/alert/observables/filters.html @@ -0,0 +1,38 @@ +
      +
      +

      Filters

      +
      +
      +
      +
      + + + + +
      +
      +
      + +
      +
      +
      +
      + + Add a filter + + + Clear + + +
      +
      + + +
      +
      diff --git a/frontend/app/views/components/alert/observables/toolbar.html b/frontend/app/views/components/alert/observables/toolbar.html new file mode 100644 index 0000000000..e83ac7f7a4 --- /dev/null +++ b/frontend/app/views/components/alert/observables/toolbar.html @@ -0,0 +1,16 @@ +
      +
      + +
      +
      diff --git a/frontend/app/views/components/alert/similar-case-list.component.html b/frontend/app/views/components/alert/similar-case-list.component.html index be1bb10afb..25a19018ac 100644 --- a/frontend/app/views/components/alert/similar-case-list.component.html +++ b/frontend/app/views/components/alert/similar-case-list.component.html @@ -2,9 +2,9 @@
      -
      +
      -
      +
      -
      From 389901fe415cc20d0bc7979513c639bb409791e4 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 12 Nov 2020 07:38:38 +0100 Subject: [PATCH 181/237] #1634 Accept ZIP as input attachment for observable creation --- .../controllers/v0/ObservableCtrl.scala | 96 +++++++++++++++---- .../controllers/v1/ObservableCtrl.scala | 72 ++++++++++++-- 2 files changed, 144 insertions(+), 24 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 0cf30e6d08..972b86223d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -1,7 +1,13 @@ package org.thp.thehive.controllers.v0 +import java.io.FilterInputStream +import java.nio.file.Files + import javax.inject.{Inject, Named, Singleton} +import net.lingala.zip4j.ZipFile +import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ +import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ @@ -16,13 +22,17 @@ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TagOps._ import org.thp.thehive.services._ +import play.api.Configuration +import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.libs.json.{JsArray, JsObject, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} +import scala.collection.JavaConverters._ import scala.util.Success @Singleton class ObservableCtrl @Inject() ( + configuration: Configuration, override val entrypoint: Entrypoint, @Named("with-thehive-schema") override val db: Database, observableSrv: ObservableSrv, @@ -30,14 +40,21 @@ class ObservableCtrl @Inject() ( caseSrv: CaseSrv, errorHandler: ErrorHandler, @Named("v0") override val queryExecutor: QueryExecutor, - override val publicData: PublicObservable + override val publicData: PublicObservable, + temporaryFileCreator: DefaultTemporaryFileCreator ) extends ObservableRenderer with QueryCtrl { def create(caseId: String): Action[AnyContent] = entrypoint("create artifact") .extract("artifact", FieldsParser[InputObservable]) + .extract("isZip", FieldsParser.boolean.optional.on("isZip")) + .extract("zipPassword", FieldsParser.string.optional.on("zipPassword")) .auth { implicit request => val inputObservable: InputObservable = request.body("artifact") + val isZip: Option[Boolean] = request.body("isZip") + val zipPassword: Option[String] = request.body("zipPassword") + val inputAttachObs = if (isZip.contains(true)) getZipFiles(inputObservable, zipPassword) else Seq(inputObservable) + db .roTransaction { implicit graph => for { @@ -51,25 +68,24 @@ class ObservableCtrl @Inject() ( } .map { case (case0, observableType) => - val initialSuccessesAndFailures: (Seq[JsValue], Seq[JsValue]) = inputObservable - .attachment - .map { attachmentFile => - db - .tryTransaction { implicit graph => - observableSrv - .create(inputObservable.toObservable, observableType, attachmentFile, inputObservable.tags, Nil) - .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) + val initialSuccessesAndFailures: (Seq[JsValue], Seq[JsValue]) = + inputAttachObs.foldLeft[(Seq[JsValue], Seq[JsValue])](Nil -> Nil) { + case ((successes, failures), inputObservable) => + inputObservable.attachment.fold((successes, failures)) { attachment => + db + .tryTransaction { implicit graph => + observableSrv + .create(inputObservable.toObservable, observableType, attachment, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) + } + .fold( + e => + successes -> (failures :+ errorHandler.toErrorResult(e)._2 ++ Json + .obj("object" -> Json.obj("attachment" -> Json.obj("name" -> attachment.filename)))), + s => (successes :+ s) -> failures + ) } - .fold( - e => - Nil -> Seq( - errorHandler.toErrorResult(e)._2 ++ Json - .obj("object" -> Json.obj("attachment" -> Json.obj("name" -> attachmentFile.filename))) - ), - s => Seq(s) -> Nil - ) } - .getOrElse(Nil -> Nil) val (successes, failures) = inputObservable .data @@ -158,6 +174,50 @@ class ObservableCtrl @Inject() ( _ <- observableSrv.remove(observable) } yield Results.NoContent } + + // extract a file from the archive and make sure its size matches the header (to protect against zip bombs) + private def extractAndCheckSize(zipFile: ZipFile, header: FileHeader): Option[FFile] = { + val fileName = header.getFileName + if (fileName.contains('/')) None + else { + val file = temporaryFileCreator.create("zip") + + val input = zipFile.getInputStream(header) + val size = header.getUncompressedSize + val sizedInput: FilterInputStream = new FilterInputStream(input) { + var totalRead = 0 + + override def read(): Int = + if (totalRead < size) { + totalRead += 1 + super.read() + } else throw BadRequestError("Error extracting file: output size doesn't match header") + } + Files.delete(file) + val fileSize = Files.copy(sizedInput, file) + if (fileSize != size) { + file.toFile.delete() + throw InternalError("Error extracting file: output size doesn't match header") + } + input.close() + val contentType = Option(Files.probeContentType(file)).getOrElse("application/octet-stream") + Some(FFile(header.getFileName, file, contentType)) + } + } + + private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = + observable.attachment.toSeq.flatMap { attachment => + val zipFile = new ZipFile(attachment.filepath.toFile) + val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] + + if (zipFile.isEncrypted) + zipFile.setPassword(zipPassword.getOrElse(configuration.get[String]("datastore.attachment.password")).toCharArray) + + files + .filterNot(_.isDirectory) + .flatMap(extractAndCheckSize(zipFile, _)) + .map(ffile => observable.copy(attachment = Some(ffile))) + } } @Singleton diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index 910afb8f73..f383a7a025 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -1,7 +1,13 @@ package org.thp.thehive.controllers.v1 +import java.io.FilterInputStream +import java.nio.file.Files + import javax.inject.{Inject, Named, Singleton} +import net.lingala.zip4j.ZipFile +import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ +import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -15,8 +21,11 @@ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services._ -import play.api.Logger +import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.mvc.{Action, AnyContent, Results} +import play.api.{Configuration, Logger} + +import scala.collection.JavaConverters._ @Singleton class ObservableCtrl @Inject() ( @@ -26,7 +35,9 @@ class ObservableCtrl @Inject() ( observableSrv: ObservableSrv, observableTypeSrv: ObservableTypeSrv, caseSrv: CaseSrv, - organisationSrv: OrganisationSrv + organisationSrv: OrganisationSrv, + temporaryFileCreator: DefaultTemporaryFileCreator, + configuration: Configuration ) extends QueryableCtrl with ObservableRenderer { @@ -70,8 +81,13 @@ class ObservableCtrl @Inject() ( def create(caseId: String): Action[AnyContent] = entryPoint("create artifact") .extract("artifact", FieldsParser[InputObservable]) + .extract("isZip", FieldsParser.boolean.optional.on("isZip")) + .extract("zipPassword", FieldsParser.string.optional.on("zipPassword")) .authTransaction(db) { implicit request => implicit graph => + val isZip: Option[Boolean] = request.body("isZip") + val zipPassword: Option[String] = request.body("zipPassword") val inputObservable: InputObservable = request.body("artifact") + val inputAttachObs = if (isZip.contains(true)) getZipFiles(inputObservable, zipPassword) else Seq(inputObservable) for { case0 <- caseSrv @@ -83,12 +99,12 @@ class ObservableCtrl @Inject() ( inputObservable .data .toTry(d => observableSrv.create(inputObservable.toObservable, observableType, d, inputObservable.tags, Nil)) - observableWithAttachment <- - inputObservable - .attachment + observableWithAttachment <- inputAttachObs.toTry( + _.attachment .map(a => observableSrv.create(inputObservable.toObservable, observableType, a, inputObservable.tags, Nil)) .flip - createdObservables <- (observablesWithData ++ observableWithAttachment).toTry { richObservables => + ) + createdObservables <- (observablesWithData ++ observableWithAttachment.flatten).toTry { richObservables => caseSrv .addObservable(case0, richObservables) .map(_ => richObservables) @@ -149,4 +165,48 @@ class ObservableCtrl @Inject() ( _ <- observableSrv.remove(observable) } yield Results.NoContent } + + // extract a file from the archive and make sure its size matches the header (to protect against zip bombs) + private def extractAndCheckSize(zipFile: ZipFile, header: FileHeader): Option[FFile] = { + val fileName = header.getFileName + if (fileName.contains('/')) None + else { + val file = temporaryFileCreator.create("zip") + + val input = zipFile.getInputStream(header) + val size = header.getUncompressedSize + val sizedInput: FilterInputStream = new FilterInputStream(input) { + var totalRead = 0 + + override def read(): Int = + if (totalRead < size) { + totalRead += 1 + super.read() + } else throw BadRequestError("Error extracting file: output size doesn't match header") + } + Files.delete(file) + val fileSize = Files.copy(sizedInput, file) + if (fileSize != size) { + file.toFile.delete() + throw InternalError("Error extracting file: output size doesn't match header") + } + input.close() + val contentType = Option(Files.probeContentType(file)).getOrElse("application/octet-stream") + Some(FFile(header.getFileName, file, contentType)) + } + } + + private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = + observable.attachment.toSeq.flatMap { attachment => + val zipFile = new ZipFile(attachment.filepath.toFile) + val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] + + if (zipFile.isEncrypted) + zipFile.setPassword(zipPassword.getOrElse(configuration.get[String]("datastore.attachment.password")).toCharArray) + + files + .filterNot(_.isDirectory) + .flatMap(extractAndCheckSize(zipFile, _)) + .map(ffile => observable.copy(attachment = Some(ffile))) + } } From 8846067f1fe5b891397436c17749c6dbfb95a064 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 12 Nov 2020 08:29:25 +0100 Subject: [PATCH 182/237] #1643 Prevent creating a file observable if it already exist in the case --- .../controllers/v0/ObservableCtrl.scala | 5 ++++- .../org/thp/thehive/services/CaseSrv.scala | 21 ++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 972b86223d..9fadc80a6a 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -81,7 +81,10 @@ class ObservableCtrl @Inject() ( .fold( e => successes -> (failures :+ errorHandler.toErrorResult(e)._2 ++ Json - .obj("object" -> Json.obj("attachment" -> Json.obj("name" -> attachment.filename)))), + .obj( + "object" -> Json + .obj("data" -> s"file:${attachment.filename}", "attachment" -> Json.obj("name" -> attachment.filename)) + )), s => (successes :+ s) -> failures ) } diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 710d0f2502..dade050113 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -175,13 +175,20 @@ class CaseSrv @Inject() ( graph: Graph, authContext: AuthContext ): Try[Unit] = { - val alreadyExistInThatCase = observableSrv - .get(richObservable.observable) - .filteredSimilar - .visible - .`case` - .hasId(`case`._id) - .exists || get(`case`).observables.filter(_.hasId(richObservable.observable._id)).exists // FIXME + val alreadyExistInThatCase = richObservable + .dataOrAttachment + .fold( + _ => + observableSrv + .get(richObservable.observable) + .filteredSimilar + .visible + .`case` + .hasId(`case`._id) + .exists, + attachment => get(`case`).share.observables.attachments.has(_.attachmentId, attachment.attachmentId).exists + ) || get(`case`).observables.filter(_.hasId(richObservable.observable._id)).exists + if (alreadyExistInThatCase) Failure(CreateError("Observable already exists")) else From 553b37c18a574e17175c7c88d7de9fa16f5c72a5 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 12 Nov 2020 08:47:22 +0100 Subject: [PATCH 183/237] #1610 Fix Cortex job status check --- .../org/thp/thehive/connector/cortex/services/JobSrv.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala index 74b25092f9..c9196e6c74 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala @@ -13,7 +13,7 @@ import javax.inject.{Inject, Singleton} import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.Graph import org.thp.cortex.client.CortexClient -import org.thp.cortex.dto.v0.{InputArtifact, OutputArtifact, Attachment => CortexAttachment, OutputJob => CortexJob} +import org.thp.cortex.dto.v0.{InputArtifact, OutputArtifact, Attachment => CortexAttachment, OutputJob => CortexJob, JobStatus => CortexJobStatus} import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models.{Database, Entity} @@ -180,7 +180,7 @@ class JobSrv @Inject() ( } private def importAnalyzerTags(job: Job with Entity, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Unit] = - if (cortexJob.status == JobStatus.Success) + if (cortexJob.status == CortexJobStatus.Success) db.tryTransaction { implicit graph => val tags = cortexJob.report.fold[Seq[ReportTag]](Nil)(_.summary.map(_.toAnalyzerTag(job.workerName))) for { From fce7f4a2fd9ab6ef7e2273b3f10c6ddd56d4da24 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 12 Nov 2020 12:18:10 +0100 Subject: [PATCH 184/237] #1552 Fix order when a new custom field is added --- thehive/app/org/thp/thehive/services/CaseSrv.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index dade050113..d2e2e0c88f 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -249,7 +249,7 @@ class CaseSrv @Inject() ( )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { cf <- customFieldSrv.getOrFail(customFieldIdOrName) - ccf <- CustomFieldType.map(cf.`type`).setValue(CaseCustomField(), customFieldValue).map(_.order_=(order)) + ccf <- CustomFieldType.map(cf.`type`).setValue(CaseCustomField().order_=(order), customFieldValue) ccfe <- caseCustomFieldSrv.create(ccf, `case`, cf) } yield RichCustomField(cf, ccfe) From 3b27fe612f2dfcc5eac262c4c503ce274e42903d Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 12 Nov 2020 12:52:44 +0100 Subject: [PATCH 185/237] #1625 Use observable flags component in observables list --- .../app/views/partials/observables/list/observables.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/app/views/partials/observables/list/observables.html b/frontend/app/views/partials/observables/list/observables.html index c649bea501..7290b50631 100644 --- a/frontend/app/views/partials/observables/list/observables.html +++ b/frontend/app/views/partials/observables/list/observables.html @@ -24,11 +24,11 @@

      - + - {{observable.dataType}} + {{observable.dataType}}
      @@ -52,7 +55,7 @@
      		- {{observable._createdAt | shortDate}} + {{observable._createdAt | shortDate}}
      - + - + - +
      Flags Type @@ -59,12 +59,12 @@

      - + From f4a4a78099aba3f8adc59bb11e9f2cfac7814bf9 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 12 Nov 2020 12:54:32 +0100 Subject: [PATCH 186/237] #1633 Fix the imported flag in report observables list --- frontend/app/scripts/directives/report-observables.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/app/scripts/directives/report-observables.js b/frontend/app/scripts/directives/report-observables.js index 12c6cc9cf1..149f10926d 100644 --- a/frontend/app/scripts/directives/report-observables.js +++ b/frontend/app/scripts/directives/report-observables.js @@ -21,6 +21,10 @@ filter: '', data: scope.observables }; + + _.each(scope.observables, function(item) { + item.imported = !!item.stats.observableId; + }); }); }, controller: function($scope) { @@ -50,7 +54,7 @@ var type = $scope.pagination.filter; _.each(type === '' ? $scope.observables : $scope.groups[type], function(item) { //if(!item.id && !item.selected) { - if(!item.stats.observableId && !item.selected) { + if(!item.imported && !item.selected) { item.selected = true; $scope.selected++; } @@ -109,7 +113,8 @@ modal.result .then(function(/*response*/) { _.each(list, function(item) { - item.id = true; + //item.id = true; + item.imported = true; item.selected = false; }); }); From 625fadf8c673061a4d2d79b05eb206f5035c51be Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 12 Nov 2020 14:43:52 +0100 Subject: [PATCH 187/237] #1633 Refresh the job report on observable import from report --- .../case/CaseObservablesItemCtrl.js | 88 +++++++++++++------ .../scripts/directives/report-observables.js | 7 +- .../views/directives/report-observables.html | 4 +- .../observables/details/analysers.html | 2 + 4 files changed, 73 insertions(+), 28 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js b/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js index 605f994af8..f9793b094c 100644 --- a/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseObservablesItemCtrl.js @@ -125,33 +125,71 @@ $scope.similarArtifactsLimit = $scope.similarArtifactsLimit + 10; }; - $scope.showReport = function (jobId) { + $scope.refreshCurrentJob = function() { + $scope.loadReport($scope.currentJob); + }; + + $scope.loadReport = function(jobId) { $scope.report = {}; - CortexSrv.getJob(jobId, true).then(function(response) { - var job = response.data; - $scope.report = { - template: job.analyzerDefinition, - content: job.report, - status: job.status, - startDate: job.startDate, - endDate: job.endDate - }; - - $scope.currentJob = jobId; - - $timeout(function() { - var reportEl = angular.element('#analysis-report')[0]; - - // Scrolling hack using jQuery stuff - $('html,body').animate({ - scrollTop: $(reportEl).offset().top - }, 'fast'); - }, 500); - - }, function(/*err*/) { - NotificationSrv.error('An expected error occured while fetching the job report'); - }); + return CortexSrv.getJob(jobId, true) + .then(function(response) { + var job = response.data; + $scope.report = { + template: job.analyzerDefinition, + content: job.report, + status: job.status, + startDate: job.startDate, + endDate: job.endDate + }; + + $scope.currentJob = jobId; + }); + }; + + $scope.showReport = function (jobId) { + + $scope.loadReport(jobId) + .then(function(){ + $timeout(function() { + var reportEl = angular.element('#analysis-report')[0]; + + // Scrolling hack using jQuery stuff + $('html,body').animate({ + scrollTop: $(reportEl).offset().top + }, 'fast'); + }, 500); + }) + .catch(function(/*err*/) { + NotificationSrv.error('An expected error occured while fetching the job report'); + }); + + // $scope.report = {}; + + // CortexSrv.getJob(jobId, true).then(function(response) { + // var job = response.data; + // $scope.report = { + // template: job.analyzerDefinition, + // content: job.report, + // status: job.status, + // startDate: job.startDate, + // endDate: job.endDate + // }; + // + // $scope.currentJob = jobId; + // + // $timeout(function() { + // var reportEl = angular.element('#analysis-report')[0]; + // + // // Scrolling hack using jQuery stuff + // $('html,body').animate({ + // scrollTop: $(reportEl).offset().top + // }, 'fast'); + // }, 500); + // + // }, function(/*err*/) { + // NotificationSrv.error('An expected error occured while fetching the job report'); + // }); }; $scope.openArtifact = function (a) { diff --git a/frontend/app/scripts/directives/report-observables.js b/frontend/app/scripts/directives/report-observables.js index 149f10926d..5ef8020ac5 100644 --- a/frontend/app/scripts/directives/report-observables.js +++ b/frontend/app/scripts/directives/report-observables.js @@ -8,7 +8,8 @@ observables: '=', analyzer: '=', caseId: '=', - permissions: '=' + permissions: '=', + onRefresh: '&?' }, templateUrl: 'views/directives/report-observables.html', link: function(scope) { @@ -117,6 +118,10 @@ item.imported = true; item.selected = false; }); + + if($scope.onRefresh) { + $scope.onRefresh(); + } }); }); diff --git a/frontend/app/views/directives/report-observables.html b/frontend/app/views/directives/report-observables.html index 7b3bf5754e..a221402c9a 100644 --- a/frontend/app/views/directives/report-observables.html +++ b/frontend/app/views/directives/report-observables.html @@ -39,13 +39,13 @@ offset: (pagination.currentPage-1)*pagination.pageSize | limitTo: pagination.pageSize "> - + - + {{observable.dataType}} diff --git a/frontend/app/views/partials/observables/details/analysers.html b/frontend/app/views/partials/observables/details/analysers.html index 5a3b1c486e..a07a59b7ab 100644 --- a/frontend/app/views/partials/observables/details/analysers.html +++ b/frontend/app/views/partials/observables/details/analysers.html @@ -115,6 +115,8 @@

      observables="report.content.artifacts" analyzer="report.template" permissions="userPermissions" + on-refresh="refreshCurrentJob()" + > From a3b98d75faca4fa65fd52a0fbc76f9b54ed0d279 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 12 Nov 2020 15:54:48 +0100 Subject: [PATCH 188/237] #1644 Fix MISP export for hash observables --- .../misp/services/MispExportSrv.scala | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala index ee69a055e2..5b5d08d496 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala @@ -37,8 +37,22 @@ class MispExportSrv @Inject() ( observable.tags.map(t => MispTag(None, t.toString, Some(t.colour), None)) ++ tlpTags.get(observable.tlp) else tlpTags.get(observable.tlp).toSeq - connector - .attributeConverter(observable.`type`) + + observable + .data + .collect { + case data if observable.`type` == "hash" => data.data.length + } + .collect { + case 32 => "md5" + case 40 => "sha1" + case 56 => "sha224" + case 64 => "sha256" + case 96 => "sha384" + case 128 => "sha512" + } + .map("Payload delivery" -> _) + .orElse(connector.attributeConverter(observable.`type`)) .map { case (cat, tpe) => Attribute( From 98180ee46e91be45b9a0cbfdeecf9be165309f8c Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 12 Nov 2020 17:33:04 +0100 Subject: [PATCH 189/237] #1644 Fix type comparison --- .../org/thp/thehive/connector/misp/services/MispExportSrv.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala index 5b5d08d496..bf71fb3ed4 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala @@ -41,7 +41,7 @@ class MispExportSrv @Inject() ( observable .data .collect { - case data if observable.`type` == "hash" => data.data.length + case data if observable.`type`.name == "hash" => data.data.length } .collect { case 32 => "md5" From 1a48dbbefd6969914226a743fdea6e061594d20e Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 12 Nov 2020 17:55:17 +0100 Subject: [PATCH 190/237] #1645 Don't add admin users from TH3 to admin organisation --- .../main/scala/org/thp/thehive/migration/th3/Conversion.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index 2694c96c7b..af1545e564 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -263,8 +263,7 @@ trait Conversion { password <- (json \ "password").validateOpt[String] role <- (json \ "roles").validateOpt[Seq[String]].map(_.getOrElse(Nil)) organisationProfiles = - if (role.contains("admin")) - Map(Organisation.administration.name -> Profile.admin.name, mainOrganisation -> Profile.orgAdmin.name) + if (role.contains("admin")) Map(mainOrganisation -> Profile.orgAdmin.name) else if (role.contains("write")) Map(mainOrganisation -> Profile.analyst.name) else if (role.contains("read")) Map(mainOrganisation -> Profile.readonly.name) else Map(mainOrganisation -> Profile.readonly.name) From a5e74803c58a0bad0ee9685265a9f252bc9ec467 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 13 Nov 2020 12:07:58 +0100 Subject: [PATCH 191/237] Prepare release --- CHANGELOG.md | 87 +++++++++++++++++++++++++++++++++++++++++++ build.sbt | 2 +- frontend/bower.json | 2 +- frontend/package.json | 2 +- 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa69694a49..e6a42bac96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,92 @@ # Change Log +## [4.0.1](https://github.com/TheHive-Project/TheHive/milestone/60) (2020-11-13) + +**Implemented enhancements:** + +- [Enhancement] Remove gremlin-scala library [\#1501](https://github.com/TheHive-Project/TheHive/issues/1501) +- [Feature request] Improve case similarity details in alert preview pane [\#1579](https://github.com/TheHive-Project/TheHive/issues/1579) +- [Enhancement] Check tag autocompletion [\#1611](https://github.com/TheHive-Project/TheHive/issues/1611) +- [Feature] Add Cortex related notifiers in notification system [\#1619](https://github.com/TheHive-Project/TheHive/issues/1619) +- [Feature] Add properties related to share [\#1621](https://github.com/TheHive-Project/TheHive/issues/1621) +- [Feature Request] Update user settings view to give access to API key [\#1623](https://github.com/TheHive-Project/TheHive/issues/1623) +- [Feature Request] Permit to disable similarity (case and alert) for some observable [\#1625](https://github.com/TheHive-Project/TheHive/issues/1625) +- [Enhancement] Add link to report template archive [\#1627](https://github.com/TheHive-Project/TheHive/issues/1627) +- [Enahancement] Display TheHive version in the login page [\#1629](https://github.com/TheHive-Project/TheHive/issues/1629) +- [Feature Request] Display custom fields in alert and case list [\#1637](https://github.com/TheHive-Project/TheHive/issues/1637) +- [Feature Request] Revamp the statistics section in lists [\#1641](https://github.com/TheHive-Project/TheHive/issues/1641) +- [Enhancement] Improve the filter observables panel [\#1642](https://github.com/TheHive-Project/TheHive/issues/1642) +- [Enhancement] Refine the migration of users with admin role [\#1645](https://github.com/TheHive-Project/TheHive/issues/1645) + +**Closed issues:** + +- [Bug] default MISP connector import line has a typo [\#1595](https://github.com/TheHive-Project/TheHive/issues/1595) + +**Fixed bugs:** + +- [Bug] Mobile-responsive Hamburger not visible [\#1290](https://github.com/TheHive-Project/TheHive/issues/1290) +- [Bug] Unable to start TheHive after migration [\#1450](https://github.com/TheHive-Project/TheHive/issues/1450) +- [Bug] Expired session should show a dialog or login page on pageload [\#1456](https://github.com/TheHive-Project/TheHive/issues/1456) +- [Bug] TheHive 4 - Application.conf file [\#1461](https://github.com/TheHive-Project/TheHive/issues/1461) +- [Bug] Improve migration [\#1469](https://github.com/TheHive-Project/TheHive/issues/1469) +- [Bug] Merge Alert in similar Case button does not work [\#1470](https://github.com/TheHive-Project/TheHive/issues/1470) +- [Bug] Missing Case number in Alert Preview / Similar Cases tab [\#1471](https://github.com/TheHive-Project/TheHive/issues/1471) +- [Bug] Dashboard shared/private [\#1474](https://github.com/TheHive-Project/TheHive/issues/1474) +- [Bug]Migration tool date/number/duration params don't work [\#1478](https://github.com/TheHive-Project/TheHive/issues/1478) +- [Bug] AuditSrv: undefined on non-case page(s), thehive4-4.0.0-1, Ubuntu [\#1479](https://github.com/TheHive-Project/TheHive/issues/1479) +- [Bug] MISP->THEHIVE4 'ExportOnly' and 'Exceptions' ignored in application.conf file [\#1482](https://github.com/TheHive-Project/TheHive/issues/1482) +- [Bug] Unable to enumerate tasks via API [\#1483](https://github.com/TheHive-Project/TheHive/issues/1483) +- [Bug] Case close notification displays "#undefined" instead of case number [\#1488](https://github.com/TheHive-Project/TheHive/issues/1488) +- [Bug] Task under "Waiting tasks" and "My tasks" do not display the case number [\#1489](https://github.com/TheHive-Project/TheHive/issues/1489) +- [Bug] Live Stream log in main page is not limited to 10 entries [\#1490](https://github.com/TheHive-Project/TheHive/issues/1490) +- [Bug] Several API Endpoints could never get called due to the routing structure [\#1492](https://github.com/TheHive-Project/TheHive/issues/1492) +- [Bug] Missing link to linked cases from observable details view [\#1494](https://github.com/TheHive-Project/TheHive/issues/1494) +- [Bug] TheHive V4 API Errors "Operation Not Permitted" and "Date format" [\#1496](https://github.com/TheHive-Project/TheHive/issues/1496) +- [Bug] V4 Merge observable tags with existing observables during importing alerts into case [\#1499](https://github.com/TheHive-Project/TheHive/issues/1499) +- [Bug] Multiline dashboard doesn't work [\#1503](https://github.com/TheHive-Project/TheHive/issues/1503) +- [Bug] Tags of observables in Alerts are not created when promoted [\#1510](https://github.com/TheHive-Project/TheHive/issues/1510) +- [Bug] Alert creation fails if alert contains similar observables [\#1514](https://github.com/TheHive-Project/TheHive/issues/1514) +- [Bug] "Undefined" in notification message when a case is closed [\#1515](https://github.com/TheHive-Project/TheHive/issues/1515) +- [Bug] The creation of multiline observable is not possible [\#1517](https://github.com/TheHive-Project/TheHive/issues/1517) +- [Bug] Entrypoint: Waiting for cassandra with --no-config [\#1519](https://github.com/TheHive-Project/TheHive/issues/1519) +- [Bug] Suppress Reduntant AuthenticationFailed Error+Warn [\#1523](https://github.com/TheHive-Project/TheHive/issues/1523) +- [Bug] API v0: "startDate" sort criteria not implemented [\#1540](https://github.com/TheHive-Project/TheHive/issues/1540) +- [Bug] Fix case search in case merge dialog [\#1541](https://github.com/TheHive-Project/TheHive/issues/1541) +- [Bug] Soft-Deleted cases show up as "(Closed at as )" in the case list. [\#1543](https://github.com/TheHive-Project/TheHive/issues/1543) +- [Bug] Related cases show only one observable [\#1544](https://github.com/TheHive-Project/TheHive/issues/1544) +- [Bug] An user can create a task even if it doesn't the permission [\#1545](https://github.com/TheHive-Project/TheHive/issues/1545) +- [Bug] Wrong stats url on user and audit [\#1546](https://github.com/TheHive-Project/TheHive/issues/1546) +- [Bug] Add DATETIME information to each task log [\#1547](https://github.com/TheHive-Project/TheHive/issues/1547) +- [Bug] Custom configuration is not correctly read in docker image [\#1548](https://github.com/TheHive-Project/TheHive/issues/1548) +- [Bug] Typo in MFA onboarding [\#1549](https://github.com/TheHive-Project/TheHive/issues/1549) +- [Bug] New custom fields doesn't appear in search criteria [\#1550](https://github.com/TheHive-Project/TheHive/issues/1550) +- [Bug] Custom Field Order ignored [\#1552](https://github.com/TheHive-Project/TheHive/issues/1552) +- [Bug] Additional Fields are discarded during merge [\#1553](https://github.com/TheHive-Project/TheHive/issues/1553) +- [Bug] Unable to list alerts in case's related alerts section [\#1554](https://github.com/TheHive-Project/TheHive/issues/1554) +- [Bug] Deleting the first case breaks the the audit flow until the next restart [\#1556](https://github.com/TheHive-Project/TheHive/issues/1556) +- [Bug] Issues surrounding Alerts merging [\#1557](https://github.com/TheHive-Project/TheHive/issues/1557) +- [Bug] Uncaught exception with duplicate mail type observables when added to case [\#1561](https://github.com/TheHive-Project/TheHive/issues/1561) +- [Bug] Case Tasks get deleted if not started [\#1565](https://github.com/TheHive-Project/TheHive/issues/1565) +- [Bug] Can't export Case tags to MISP event [\#1566](https://github.com/TheHive-Project/TheHive/issues/1566) +- [Bug]The link to similar observable in observable details page doesn't work [\#1567](https://github.com/TheHive-Project/TheHive/issues/1567) +- [Bug] TheHive4 'follow/unfollow' API doesn't return alert objects like TheHive3 does [\#1571](https://github.com/TheHive-Project/TheHive/issues/1571) +- [Bug] Alert Custom Field with integer value [\#1588](https://github.com/TheHive-Project/TheHive/issues/1588) +- [Bug] Tag filter is broken [\#1590](https://github.com/TheHive-Project/TheHive/issues/1590) +- [Bug] Admin user does not have the right to list users of other organisations [\#1592](https://github.com/TheHive-Project/TheHive/issues/1592) +- [Bug] Add missing query operations [\#1599](https://github.com/TheHive-Project/TheHive/issues/1599) +- [Bug] Fix configuration sample [\#1600](https://github.com/TheHive-Project/TheHive/issues/1600) +- [Bug] Analyzer tags are removes if Cortex job fails [\#1610](https://github.com/TheHive-Project/TheHive/issues/1610) +- [Bug] deleted Tasks displayed in MyTasks [\#1612](https://github.com/TheHive-Project/TheHive/issues/1612) +- [Bug] the "_in" query operator doesn't work [\#1617](https://github.com/TheHive-Project/TheHive/issues/1617) +- [Bug] Sort filter field dropdowns [\#1630](https://github.com/TheHive-Project/TheHive/issues/1630) +- [Bug] Alert imported multiple times [\#1631](https://github.com/TheHive-Project/TheHive/issues/1631) +- [Bug] Import observables from analyzer report is broken [\#1633](https://github.com/TheHive-Project/TheHive/issues/1633) +- [Bug] Import observable from a zip archive doesn't work [\#1634](https://github.com/TheHive-Project/TheHive/issues/1634) +- [Bug] Case handling duration attributes are not working in time based dashboard widgets [\#1635](https://github.com/TheHive-Project/TheHive/issues/1635) +- [Bug] Fix custom field in filter forms [\#1636](https://github.com/TheHive-Project/TheHive/issues/1636) +- [Bug] It is possible to add an identical file observable several times in a case [\#1643](https://github.com/TheHive-Project/TheHive/issues/1643) +- [Bug] Hash observables are not correctly export to MISP [\#1644](https://github.com/TheHive-Project/TheHive/issues/1644) + ## [4.0.0](https://github.com/TheHive-Project/TheHive/milestone/59) (2020-07-24) **Implemented enhancements:** diff --git a/build.sbt b/build.sbt index cd699f5508..c93640d396 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.0.1-1-SNAPSHOT" +val thehiveVersion = "4.0.1-1" val scala212 = "2.12.12" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/frontend/bower.json b/frontend/bower.json index dc76d44f6e..4a58c3aed4 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.1-1-SNAPSHOT", + "version": "4.0.1-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index eaa2a3099b..e13f63a44a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.1-1-SNAPSHOT", + "version": "4.0.1-1", "license": "AGPL-3.0", "repository": { "type": "git", From 8116303609e530927fd0f9a17554ae2101b7717d Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 08:51:47 +0100 Subject: [PATCH 192/237] #1648 Create link with case when merging alert --- thehive/app/org/thp/thehive/services/AlertSrv.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index b9877b41e2..f4927bde2d 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -276,17 +276,16 @@ class AlertSrv @Inject() ( def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = auditSrv .mergeAudits { + // No audit for markAsRead and observables + // Audits for customFields, description and tags val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" - for { _ <- markAsRead(alert._id) _ <- importObservables(alert, `case`) _ <- importCustomFields(alert, `case`) - _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) - // No audit for markAsRead and observables - // Audits for customFields, description and tags - c <- caseSrv.getOrFail(`case`._id) + _ <- alertCaseSrv.create(AlertCase(), alert, `case`) + c <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") details <- Success( Json.obj( "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), From 3e82fdd7255c33697235d8adefacbf68baaa45bc Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 09:12:33 +0100 Subject: [PATCH 193/237] #1652 Return the observable on update --- .../org/thp/thehive/controllers/v0/ObservableCtrl.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 9fadc80a6a..f7dbdb0221 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -133,7 +133,13 @@ class ObservableCtrl @Inject() ( _.get(EntityIdOrName(observableId)).can(Permissions.manageObservable), propertyUpdaters ) - .map(_ => Results.NoContent) + .flatMap { + case (observables, _) => + observables + .richObservable + .getOrFail("Observable") + .map(richObservable => Results.Ok(richObservable.toJson)) + } } def findSimilar(observableId: String): Action[AnyContent] = From 01ed14575dd5fe95f0b8f1e21d95b1eee343be52 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 09:12:50 +0100 Subject: [PATCH 194/237] #1652 Return the log on update --- thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala index 583a43105f..4285118d2d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala @@ -55,7 +55,13 @@ class LogCtrl @Inject() ( .can(Permissions.manageTask), propertyUpdaters ) - .map(_ => Results.NoContent) + .flatMap { + case (logs, _) => + logs + .richLog + .getOrFail("Log") + .map(richLog => Results.Ok(richLog.toJson)) + } } def delete(logId: String): Action[AnyContent] = From 1e0a04dac2e81c4ba3167a9254038842f05bbfe6 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 09:13:06 +0100 Subject: [PATCH 195/237] #1652 Return the case template on update --- .../org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala index 6973c978b2..5a1d824314 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala @@ -68,7 +68,13 @@ class CaseTemplateCtrl @Inject() ( .can(Permissions.manageCaseTemplate), propertyUpdaters ) - .map(_ => Results.NoContent) + .flatMap { + case (caseTemplates, _) => + caseTemplates + .richCaseTemplate + .getOrFail("CaseTemplate") + .map(richCaseTemplate => Results.Ok(richCaseTemplate.toJson)) + } } def delete(caseTemplateNameOrId: String): Action[AnyContent] = From 5f1bec9313b266bc5f3090385d6343ff7e741462 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Tue, 17 Nov 2020 09:24:07 +0100 Subject: [PATCH 196/237] #1657 Fix the case attachment section to ignore logs from cancelled tasks --- frontend/app/scripts/controllers/case/CaseDetailsCtrl.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js b/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js index 699af3d81f..16b87faf41 100644 --- a/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js @@ -24,6 +24,7 @@ operations: [ { '_name': 'getCase', 'idOrName': $scope.caseId }, { '_name': 'tasks' }, + { '_name': 'filter', '_ne':{'_field': 'status', '_value': 'Cancel'}}, { '_name': 'logs' }, ] }); From 8f4ea83af0064d1fa56956ed2359e690ba1b0c12 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 09:40:43 +0100 Subject: [PATCH 197/237] #1652 Fix tests --- .../org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala index a2bba30c28..31c50cec90 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala @@ -131,7 +131,8 @@ class CaseTemplateCtrlTest extends PlaySpecification with TestAppBuilder { ) val result = app[CaseTemplateCtrl].update("spam")(request) - status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") + status(result) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") + contentAsJson(result).as[OutputCaseTemplate].displayName must beEqualTo("patched") val updatedOutput = app[Database].roTransaction { implicit graph => app[CaseTemplateSrv].get(EntityName("spam")).richCaseTemplate.head From 6bc19c35a20d025213ac31356b54f76f2b981c88 Mon Sep 17 00:00:00 2001 From: garanews Date: Tue, 17 Nov 2020 14:13:50 +0100 Subject: [PATCH 198/237] update docker compose integration guide --- docker/README.md | 54 +++++++++++++++++++++------------- docker/cortex/application.conf | 4 +-- docker/docker-compose.yml | 14 ++++++--- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/docker/README.md b/docker/README.md index 3aadbdb298..9211c3ab3e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,14 +1,14 @@ ## Example of docker-compose (not for production) With this docker-compose.yml you will be able to run the following images: -- The Hive 4 +- The Hive 4.0.1-1 - Cassandra 3.11 - Cortex 3.1.0-1 - Elasticsearch 7.9.3 - Kibana 7.9.3 -- MISP 2.4.133 +- MISP 2.4.134 - Mysql 8.0.22 - Redis 6.0.9 -- Shuffle 0.7.1 +- Shuffle 0.7.6 ## Some Hint @@ -17,47 +17,46 @@ In docker-compose version is set 3.8, to run this version you need at least Dock ``` Compose file format Docker Engine release 3.8 19.03.0+ -3.7 18.06.0+ -3.6 18.02.0+ -3.5 17.12.0+ -3.4 17.09.0+ +3.7 18.06.0+ +3.6 18.02.0+ +3.5 17.12.0+ +3.4 17.09.0+ ``` If for some reason you have a previous version of Docker Engine or a previous version of Docker Compose and can't upgrade those, you can use 3.7 or 3.6 in docker-compose.yml ### Mapping volumes -If you take a look of docker-compose.yml you will see you need some local folder that needs to be mapped, so before do docker-compose up, ensure folders (and config files) exist: -- ./elasticsearch/data:/usr/share/elasticsearch/data -- ./elasticsearch/logs:/usr/share/elasticsearch/logs +If you take a look of docker-compose.yml you will see you need some local folder that needs to be mapped, so before do docker-compose up, ensure at least folders with config files exist: - ./cortex/application.conf:/etc/cortex/application.conf - ./thehive/application.conf:/etc/thehive/application.conf -- ./data:/data -- ./mysql:/var/lib/mysql Structure would look like: ``` ├── docker-compose.yml -├── elasticsearch -│ └── data -│ └── logs +├── elasticsearch_data +|── elasticsearch_logs ├── cortex │ └── application.conf -└── thehive - └── application.conf -└── data -└── mysql +|── thehive +| └── application.conf +|── data +|── mysql ``` +If you run docker-compose with sudo, ensure you have created elasticsearch_data and elasticsearch_logs folders with non root user, otherwise elasticsearch container will not start. ### ElasticSearch ElasticSearch container likes big mmap count (https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) so from shell you can change with ```sysctl -w vm.max_map_count=262144``` -Due you would run all on same system and maybe you have a limited amount of RAM, better to set some size, for ElasticSearch, in docker-compose.yml I added those: +To set this value permanently, update the vm.max_map_count setting in /etc/sysctl.conf. To verify after rebooting, run sysctl vm.max_map_count + +If you would run all containers on the same system - and maybe you have a limited amount of RAM - better to set some limit, for ElasticSearch, in docker-compose.yml I added those: ```- bootstrap.memory_lock=true``` ```- "ES_JAVA_OPTS=-Xms256m -Xmx256m"``` Adjust depending on your needs and your env. Without these settings in my environment ElasticSearch was using 1.5GB + ### Cassandra Like for ElasticSearch maybe you would run all on same system and maybe you don't have a limited amount of RAM, better to set some size, here for Cassandra, in docker-compose.yml I added those: @@ -68,7 +67,7 @@ Adjust depending on your needs and your env. Without these settings in my enviro ### Cortex-Analyzers - In order to use Analyzers in docker version, it is set the online json url instead absolute path of analyzers in the application.conf of Cortex: - https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json + https://download.thehive-project.org/analyzers.json - In order to use Analyzers in docker version it is set the application.conf thejob: ``` job { runner = [docker] @@ -142,3 +141,16 @@ curl -XPUT -uuser@thehive.local:user@thehive.local -H 'Content-type: application ``` - Now are able to play automation with The Hive, Cortex-Analyzers, MISP thanks to SHUFFLE! + +### Result +In conclusion, after execute ```sudo docker-compose up``` you will have the following services running: + + +| Service | Address | User | Password | +|----------|:-------------:|:------:|------:| +| The Hive | http://localhost:9000 | admin@thehive.local | secret +| Cortex | http://localhost:9001 | | +| Elasticsearch | http://localhost:9200 | | +| Kibana | http://localhost:5601 | | +| MISP | https://localhost:443 | admin@admin.test | admin +| Shuffle | http://localhost:3001 | | \ No newline at end of file diff --git a/docker/cortex/application.conf b/docker/cortex/application.conf index 0fe0c01b63..6236c81902 100644 --- a/docker/cortex/application.conf +++ b/docker/cortex/application.conf @@ -179,7 +179,7 @@ analyzer { # - directory where analyzers are installed # - json file containing the list of analyzer descriptions urls = [ - "https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json" + "https://download.thehive-project.org/analyzers.json" #"/absolute/path/of/analyzers" ] @@ -199,7 +199,7 @@ analyzer { responder { # responder location (same format as analyzer.urls) urls = [ - "https://dl.bintray.com/thehive-project/cortexneurons/responders.json" + "https://download.thehive-project.org/responders.json" #"/absolute/path/of/responders" ] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ebd066e945..4040565e44 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,8 +22,8 @@ services: soft: 65536 hard: 65536 volumes: - - ./elasticsearch/data:/usr/share/elasticsearch/data - - ./elasticsearch/logs:/usr/share/elasticsearch/logs + - ./elasticsearch_data:/usr/share/elasticsearch/data + - ./elasticsearch_logs:/usr/share/elasticsearch/logs kibana: image: 'docker.elastic.co/kibana/kibana:7.9.3' container_name: kibana @@ -98,12 +98,18 @@ services: - "80:80" - "443:443" environment: - - "HOSTNAME=http://misp" + - "HOSTNAME=https://localhost" - "REDIS_FQDN=redis" - "INIT=true" # Initialze MISP, things includes, attempting to import SQL and the Files DIR - "CRON_USER_ID=1" # The MISP user ID to run cron jobs as - "DISIPV6=true" # Disable IPV6 in nginx - + misp-modules: + image: coolacid/misp-docker:modules-latest + environment: + - "REDIS_BACKEND=redis" + depends_on: + - redis + - db #READY FOR AUTOMATION ? frontend: From a53ce20b82c9c515b0e3a95f3627e9c8653fc97a Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 14:46:52 +0100 Subject: [PATCH 199/237] #1649 Fix attachment filters --- ScalliGraph | 2 +- .../app/org/thp/thehive/controllers/v0/LogCtrl.scala | 6 +++++- .../thp/thehive/controllers/v0/ObservableCtrl.scala | 3 ++- .../org/thp/thehive/controllers/v0/UserCtrl.scala | 1 - .../org/thp/thehive/controllers/v1/Properties.scala | 12 ++++++++++-- .../org/thp/thehive/services/th3/Aggregation.scala | 2 +- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 0dc00d560b..8566855f29 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 0dc00d560b7c48f5f7a781cd4c9861a64f56cd23 +Subproject commit 8566855f2909b5aef31f77be47aadfddf64c3f46 diff --git a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala index 4285118d2d..ca1bc469a4 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala @@ -96,6 +96,10 @@ class PublicLog @Inject() (logSrv: LogSrv, organisationSrv: OrganisationSrv) ext .property("deleted", UMapping.boolean)(_.field.updatable) .property("startDate", UMapping.date)(_.rename("date").readonly) .property("status", UMapping.string)(_.select(_.constant("Ok")).readonly) - .property("attachment", UMapping.string)(_.select(_.attachments.value(_.attachmentId)).readonly) + .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) + .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index f7dbdb0221..629711357d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -307,8 +307,9 @@ class PublicObservable @Inject() ( .property("dataType", UMapping.string)(_.select(_.observableType.value(_.name)).readonly) .property("data", UMapping.string.optional)(_.select(_.data.value(_.data)).readonly) .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) - .property("attachment.hashes", UMapping.hash)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala index bf4393a499..2f46fa2bb9 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala @@ -117,7 +117,6 @@ class UserCtrl @Inject() ( .update(userSrv.get(EntityIdOrName(userId)), propertyUpdaters) // Authorisation is managed in public properties .flatMap { case (user, _) => user.richUser.getOrFail("User") } } yield Results.Ok(user.toJson) - } def setPassword(userId: String): Action[AnyContent] = diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index fa40aae20b..4f1122e467 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -435,7 +435,11 @@ class Properties @Inject() ( .property("message", UMapping.string)(_.field.updatable) .property("deleted", UMapping.boolean)(_.field.updatable) .property("date", UMapping.date)(_.field.readonly) - .property("attachment", UMapping.string)(_.select(_.attachments.value(_.attachmentId)).readonly) + .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) + .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build lazy val user: PublicProperties = @@ -480,6 +484,10 @@ class Properties @Inject() ( .property("tlp", UMapping.int)(_.field.updatable) .property("dataType", UMapping.string)(_.select(_.observableType.value(_.name)).readonly) .property("data", UMapping.string.optional)(_.select(_.data.value(_.data)).readonly) - // TODO add attachment ? + .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) + .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build } diff --git a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala index 358d5a744b..90032f84dc 100644 --- a/thehive/app/org/thp/thehive/services/th3/Aggregation.scala +++ b/thehive/app/org/thp/thehive/services/th3/Aggregation.scala @@ -177,7 +177,7 @@ case class AggAvg(aggName: Option[String], fieldName: String) extends Aggregatio traversal: Traversal.Unk, authContext: AuthContext ): Traversal.Domain[Output[_]] = { - val fieldPath = if (fieldName.startsWith("computed")) FPathElem(fieldName) else FPath(fieldName) + val fieldPath = FPath(fieldName) val property = publicProperties .get[Traversal.UnkD, Traversal.UnkDU](fieldPath, traversalType) .getOrElse(throw BadRequestError(s"Property $fieldName for type $traversalType not found")) From 9862b11037e6e0b810d8efd4532900e5db9b35e0 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 14:54:44 +0100 Subject: [PATCH 200/237] #1661 Fix typo in the migrate parameter "es-index" --- .../src/main/scala/org/thp/thehive/migration/Migrate.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala index b29a7f57bd..2e1b770131 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala @@ -56,7 +56,7 @@ object Migrate extends App with MigrationOps { opt[String]('i', "es-index") .valueName("") .text("TheHive3 ElasticSearch index name") - .action((i, c) => addConfig(c, "intput.search.index", i)), + .action((i, c) => addConfig(c, "input.search.index", i)), opt[String]('a', "es-keepalive") .valueName("") .text("TheHive3 ElasticSearch keepalive") From 8d7aa1ff8fe3527e7bc7213ccad079ee22ca1bd3 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 18:02:35 +0100 Subject: [PATCH 201/237] #1655 Add permission to access TheHiveFS --- ScalliGraph | 2 +- .../app/org/thp/thehive/controllers/dav/Router.scala | 7 ++++--- thehive/app/org/thp/thehive/models/Permissions.scala | 6 ++++-- thehive/app/org/thp/thehive/models/Role.scala | 3 ++- .../thp/thehive/models/TheHiveSchemaDefinition.scala | 12 +++++++++++- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 8566855f29..96c0bc6494 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 8566855f2909b5aef31f77be47aadfddf64c3f46 +Subproject commit 96c0bc6494d29146c69b62da62f685b2c7b247a5 diff --git a/thehive/app/org/thp/thehive/controllers/dav/Router.scala b/thehive/app/org/thp/thehive/controllers/dav/Router.scala index dbe2c6930b..b746ba3ab7 100644 --- a/thehive/app/org/thp/thehive/controllers/dav/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/dav/Router.scala @@ -6,6 +6,7 @@ import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database +import org.thp.thehive.models.Permissions import org.thp.thehive.services.AttachmentSrv import play.api.Logger import play.api.http.{HttpEntity, Status, Writeable} @@ -65,7 +66,7 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s def dav(path: String): Action[AnyContent] = entrypoint("dav") .extract("xml", FieldsParser.xml.on("xml")) - .authRoTransaction(db) { implicit request => implicit graph => + .authPermittedRoTransaction(db, Permissions.accessTheHiveFS) { implicit request => implicit graph => val pathElements = path.split('/').toList.filterNot(_.isEmpty) val baseUrl = if (request.uri.endsWith("/")) request.uri @@ -102,7 +103,7 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s def downloadFile(id: String): Action[AnyContent] = entrypoint("download attachment") - .authRoTransaction(db) { request => implicit graph => + .authPermittedRoTransaction(db, Permissions.accessTheHiveFS) { request => implicit graph => attachmentSrv.getOrFail(EntityIdOrName(id)).map { attachment => val range = request.headers.get("Range") range match { @@ -129,7 +130,7 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s def head(path: String): Action[AnyContent] = entrypoint("head") - .authRoTransaction(db) { implicit request => implicit graph => + .authPermittedRoTransaction(db, Permissions.accessTheHiveFS) { implicit request => implicit graph => val pathElements = path.split('/').toList vfs .get(pathElements) diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index bf10a22dde..14b45cf5fc 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -19,7 +19,8 @@ object Permissions extends Perms { lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") lazy val manageAnalyse: PermissionDesc = PermissionDesc("manageAnalyse", "Run Cortex analyzer", "organisation") lazy val managePage: PermissionDesc = PermissionDesc("managePage", "Manage pages", "organisation") - lazy val manageObservableTemplate: PermissionDesc = PermissionDesc("manageObservableTemplate", "Manage observable types ", "admin") + lazy val manageObservableTemplate: PermissionDesc = PermissionDesc("manageObservableTemplate", "Manage observable types", "admin") + lazy val accessTheHiveFS: PermissionDesc = PermissionDesc("accessTheHiveFS", "Access to TheHiveFS", "organisation") lazy val list: Set[PermissionDesc] = Set( @@ -39,7 +40,8 @@ object Permissions extends Perms { manageShare, manageAnalyse, managePage, - manageObservableTemplate + manageObservableTemplate, + accessTheHiveFS ) // These permissions are available only if the user is in admin organisation, they are removed for other organisations diff --git a/thehive/app/org/thp/thehive/models/Role.scala b/thehive/app/org/thp/thehive/models/Role.scala index 51a2bc80cd..996b9709bd 100644 --- a/thehive/app/org/thp/thehive/models/Role.scala +++ b/thehive/app/org/thp/thehive/models/Role.scala @@ -26,7 +26,8 @@ object Profile { Permissions.manageAction, Permissions.manageShare, Permissions.manageAnalyse, - Permissions.managePage + Permissions.managePage, + Permissions.accessTheHiveFS ) ) val readonly: Profile = Profile("read-only", Set.empty) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 62683434d6..eeab7f15fd 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -3,7 +3,9 @@ package org.thp.thehive.models import java.lang.reflect.Modifier import javax.inject.{Inject, Singleton} +import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.Graph +import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality import org.janusgraph.core.schema.ConsistencyModifier import org.janusgraph.graphdb.types.TypeDefinitionCategory import org.reflections.Reflections @@ -68,12 +70,20 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .noop // .addIndex("Tag", IndexType.unique, "namespace", "predicate", "value") .noop // .addIndex("Audit", IndexType.basic, "requestId", "mainAction") .rebuildIndexes - // release 4.0.0 + //=====[release 4.0.0]===== .updateGraph("Remove cases with a Deleted status", "Case") { traversal => traversal.unsafeHas("status", "Deleted").remove() Success(()) } .addProperty[Option[Boolean]]("Observable", "ignoreSimilarity") + //=====[release 4.0.1]===== + .updateGraph("Add accessTheHiveFS permission to analyst and org-admin profiles", "Profile") { traversal => + traversal + .unsafeHas("name", P.within("org-admin", "analyst")) + .onRaw(_.property(Cardinality.set: Cardinality, "permissions", "accessTheHiveFS", Nil: _*)) // Nil is for disambiguate the overloaded methods + .iterate() + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() From f1c65f2f24287e50a9d8bffd96e76c65754fb64d Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 17 Nov 2020 18:27:00 +0100 Subject: [PATCH 202/237] #1655 Fix tests --- .../test/org/thp/thehive/controllers/v1/UserCtrlTest.scala | 3 ++- thehive/test/org/thp/thehive/services/CaseSrvTest.scala | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala index e7ac8f762c..8a5773b794 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala @@ -110,7 +110,8 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { Permissions.manageObservable, Permissions.manageAlert, Permissions.manageAction, - Permissions.manageConfig + Permissions.manageConfig, + Permissions.accessTheHiveFS ), organisation = "cert" ) diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index de36479d95..258c2c38cc 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -62,7 +62,8 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { Permissions.manageAction, Permissions.manageAnalyse, Permissions.manageShare, - Permissions.managePage + Permissions.managePage, + Permissions.accessTheHiveFS ) ) richCase.tags.map(_.toString) must contain(exactly("testNamespace:testPredicate=\"t1\"", "testNamespace:testPredicate=\"t3\"")) @@ -102,7 +103,8 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { Permissions.manageAction, Permissions.manageAnalyse, Permissions.manageShare, - Permissions.managePage + Permissions.managePage, + Permissions.accessTheHiveFS ) ) richCase.tags.map(_.toString) must contain(exactly("testNamespace:testPredicate=\"t2\"", "testNamespace:testPredicate=\"t1\"")) From 4f0ac9c445e067a615499164480e935ea780bb72 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 18 Nov 2020 07:33:05 +0100 Subject: [PATCH 203/237] #1653 Fix display of long custom field values in case and alert lists --- frontend/app/styles/label.css | 23 ++++++++++++++----- .../common/custom-field-labels.component.html | 6 ++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/frontend/app/styles/label.css b/frontend/app/styles/label.css index 2aeadea342..c4afc10d54 100644 --- a/frontend/app/styles/label.css +++ b/frontend/app/styles/label.css @@ -1,9 +1,9 @@ -.double-val-label { +.kv-label { display: table; margin-top: 0; margin-left: 0; } -.double-val-label>span { +.kv-label>span { color: #ffffff; display: table-cell; font-weight: 400; @@ -13,23 +13,34 @@ vertical-align: baseline; white-space: nowrap; } -.double-val-label>span:first-child { +.kv-label span.kv-label-key { border-bottom-left-radius: 0.25em; border-top-left-radius: .25em; border-left-color: #3c8dbc; border-left-width: 3px; border-left-style: solid; } -.double-val-label>span:last-child { +.kv-label span.kv-label-val { border-bottom-right-radius: 0.25em; border-top-right-radius: .25em; border-left: 1px dashed #3c8dbc; + + overflow: hidden; + text-overflow: ellipsis; + max-width: 200px; } -.double-val-label>span.primary { +.kv-label>span.primary { background-color: #3c8dbc; color: #fff; } -.double-val-label>span.default { +.kv-label>span.default { background-color: #d2d6de; color: #444; } + + +.kv-label .tooltip-inner{ + word-wrap: break-word; + text-align: left; + white-space: pre-line; +} diff --git a/frontend/app/views/components/common/custom-field-labels.component.html b/frontend/app/views/components/common/custom-field-labels.component.html index 0845ab7043..48e8c6f16e 100644 --- a/frontend/app/views/components/common/custom-field-labels.component.html +++ b/frontend/app/views/components/common/custom-field-labels.component.html @@ -3,12 +3,12 @@ None - - {{$cmp.fieldsCache[cf.name].name || cf.name}} - {{cf | customFieldValue}} + {{$cmp.fieldsCache[cf.name].name || cf.name}} + {{cf | customFieldValue}} From c65a183b8447fd012d9f08c04d9546d813a9e783 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 18 Nov 2020 07:44:27 +0100 Subject: [PATCH 204/237] #1649 Fix type return by describe API for hashes --- thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala | 2 +- thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala index 16a8ceaaf5..65599ccb62 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala @@ -215,7 +215,7 @@ class DescribeCtrl @Inject() ( prop.mapping.domainTypeClass match { case c if c == classOf[Boolean] || c == classOf[JBoolean] => Seq(PropertyDescription(prop.propertyName, "boolean")) case c if c == classOf[Date] => Seq(PropertyDescription(prop.propertyName, "date")) - case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "hash")) + case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "string")) case c if classOf[Number].isAssignableFrom(c) => Seq(PropertyDescription(prop.propertyName, "number")) case c if c == classOf[String] => Seq(PropertyDescription(prop.propertyName, "string")) case _ => diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index ac393112f8..c5a2322a85 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -207,7 +207,7 @@ class DescribeCtrl @Inject() ( prop.mapping.domainTypeClass match { case c if c == classOf[Boolean] || c == classOf[JBoolean] => Seq(PropertyDescription(prop.propertyName, "boolean")) case c if c == classOf[Date] => Seq(PropertyDescription(prop.propertyName, "date")) - case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "hash")) + case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "string")) case c if classOf[Number].isAssignableFrom(c) => Seq(PropertyDescription(prop.propertyName, "number")) case c if c == classOf[String] => Seq(PropertyDescription(prop.propertyName, "string")) case _ => From ccc05022dcd65f889158fb0a07349fa5a5b55f5a Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 18 Nov 2020 08:21:32 +0100 Subject: [PATCH 205/237] #1659 #1660 Fix filters on child/parent in v0/_search --- .../controllers/v0/TheHiveQueryExecutor.scala | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala index c783dc61ea..b55bbac9f1 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala @@ -149,12 +149,14 @@ class ParentQueryInputFilter(parentFilter: InputQuery[Traversal.Unk, Traversal.U authContext: AuthContext ): Traversal.Unk = { def filter[F, T: ru.TypeTag](t: Traversal.V[F] => Traversal.V[T]): Traversal.Unk = - parentFilter( - db, - publicProperties, - ru.typeOf[Traversal.V[T]], - t(traversal.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], - authContext + traversal.filter(parent => + parentFilter( + db, + publicProperties, + ru.typeOf[Traversal.V[T]], + t(parent.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], + authContext + ) ) RichType @@ -189,12 +191,14 @@ class ChildQueryInputFilter(childType: String, childFilter: InputQuery[Traversal authContext: AuthContext ): Traversal.Unk = { def filter[F, T: ru.TypeTag](t: Traversal.V[F] => Traversal.V[T]): Traversal.Unk = - childFilter( - db, - publicProperties, - ru.typeOf[Traversal.V[T]], - t(traversal.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], - authContext + traversal.filter(child => + childFilter( + db, + publicProperties, + ru.typeOf[Traversal.V[T]], + t(child.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], + authContext + ) ) RichType From b1783ac1b2093130a0e7910ee27bea57099ccf2d Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 18 Nov 2020 09:19:22 +0100 Subject: [PATCH 206/237] #1656 Filter initial flow with flow.maxAge --- .../org/thp/thehive/services/FlowActor.scala | 28 +++++++++++++------ thehive/conf/reference.conf | 2 ++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/FlowActor.scala b/thehive/app/org/thp/thehive/services/FlowActor.scala index 44b6da1d26..5c3411498a 100644 --- a/thehive/app/org/thp/thehive/services/FlowActor.scala +++ b/thehive/app/org/thp/thehive/services/FlowActor.scala @@ -1,13 +1,17 @@ package org.thp.thehive.services +import java.util.Date + import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.cluster.singleton.{ClusterSingletonManager, ClusterSingletonManagerSettings, ClusterSingletonProxy, ClusterSingletonProxySettings} import com.google.inject.name.Names import com.google.inject.{Injector, Key => GuiceKey} import javax.inject.{Inject, Provider, Singleton} -import org.apache.tinkerpop.gremlin.process.traversal.Order +import org.apache.tinkerpop.gremlin.process.traversal.{Order, P} import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.EventSrv +import org.thp.scalligraph.services.config.ApplicationConfig.finiteDurationFormat +import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{EntityId, EntityIdOrName} import org.thp.thehive.GuiceAkkaExtension @@ -15,6 +19,8 @@ import org.thp.thehive.services.AuditOps._ import org.thp.thehive.services.CaseOps._ import play.api.cache.SyncCacheApi +import scala.concurrent.duration.FiniteDuration + object FlowActor { case class FlowId(organisation: EntityIdOrName, caseId: Option[EntityIdOrName]) { override def toString: String = s"$organisation;${caseId.getOrElse("-")}" @@ -25,20 +31,26 @@ object FlowActor { class FlowActor extends Actor { import FlowActor._ - lazy val injector: Injector = GuiceAkkaExtension(context.system).injector - lazy val cache: SyncCacheApi = injector.getInstance(classOf[SyncCacheApi]) - lazy val auditSrv: AuditSrv = injector.getInstance(classOf[AuditSrv]) - lazy val caseSrv: CaseSrv = injector.getInstance(classOf[CaseSrv]) - lazy val db: Database = injector.getInstance(GuiceKey.get(classOf[Database], Names.named("with-thehive-schema"))) - lazy val eventSrv: EventSrv = injector.getInstance(classOf[EventSrv]) + lazy val injector: Injector = GuiceAkkaExtension(context.system).injector + lazy val cache: SyncCacheApi = injector.getInstance(classOf[SyncCacheApi]) + lazy val auditSrv: AuditSrv = injector.getInstance(classOf[AuditSrv]) + lazy val caseSrv: CaseSrv = injector.getInstance(classOf[CaseSrv]) + lazy val db: Database = injector.getInstance(GuiceKey.get(classOf[Database], Names.named("with-thehive-schema"))) + lazy val appConfig: ApplicationConfig = injector.getInstance(classOf[ApplicationConfig]) + lazy val maxAgeConfig: ConfigItem[FiniteDuration, FiniteDuration] = + appConfig.item[FiniteDuration]("flow.maxAge", "Max age of audit logs shown in initial flow") + def fromDate: Date = new Date(System.currentTimeMillis() - maxAgeConfig.get.toMillis) + lazy val eventSrv: EventSrv = injector.getInstance(classOf[EventSrv]) override def preStart(): Unit = eventSrv.subscribe(StreamTopic(), self) override def receive: Receive = { case flowId @ FlowId(organisation, caseId) => val auditIds = cache.getOrElseUpdate(flowId.toString) { db.roTransaction { implicit graph => caseId - .fold(auditSrv.startTraversal.has(_.mainAction, true).visible(organisation))(caseSrv.get(_).audits(organisation)) + .fold(auditSrv.startTraversal.has(_.mainAction, true).has(_._createdAt, P.gt(fromDate)).visible(organisation))( + caseSrv.get(_).audits(organisation) + ) .sort(_.by("_createdAt", Order.desc)) .range(0, 10) ._id diff --git a/thehive/conf/reference.conf b/thehive/conf/reference.conf index 1bce841547..f9f5bf54d3 100644 --- a/thehive/conf/reference.conf +++ b/thehive/conf/reference.conf @@ -9,6 +9,8 @@ storage { localfs.directory: /data/thehive } +flow.maxAge: 1 day + auth { providers: [ {name: session} From 09afaed783a7d8ab4674c77d6d742ecc055172bb Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 18 Nov 2020 11:00:20 +0100 Subject: [PATCH 207/237] #1655 Add accessTheHiveFS permission --- frontend/app/scripts/services/api/ProfileSrv.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/app/scripts/services/api/ProfileSrv.js b/frontend/app/scripts/services/api/ProfileSrv.js index e0c1934a94..40ebe5af40 100644 --- a/frontend/app/scripts/services/api/ProfileSrv.js +++ b/frontend/app/scripts/services/api/ProfileSrv.js @@ -42,7 +42,8 @@ 'manageObservable', 'manageTask', 'manageAction', - 'manageAnalyse' + 'manageAnalyse', + 'accessTheHiveFS' ], labels: { manageUser: 'Manage users', @@ -53,7 +54,8 @@ manageObservable: 'Manage observables', manageTask: 'Manage tasks', manageAction: 'Run Cortex responders', - manageAnalyse: 'Run Cortex analyzer' + manageAnalyse: 'Run Cortex analyzer', + accessTheHiveFS: 'Access to TheHiveFS service' } } }; From 82a4be41ca4ac9a9c9dce7b36d75210d9771badd Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 18 Nov 2020 12:14:31 +0100 Subject: [PATCH 208/237] #1653 Redesign the filter preview component and take into account long values --- frontend/app/styles/filters.css | 8 +++++ frontend/app/styles/label.css | 15 ++++++-- frontend/app/styles/main.css | 5 +++ .../search/filters-preview.component.html | 34 +++++++++---------- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/frontend/app/styles/filters.css b/frontend/app/styles/filters.css index 98bce17fd7..691502fd94 100644 --- a/frontend/app/styles/filters.css +++ b/frontend/app/styles/filters.css @@ -27,6 +27,14 @@ padding-left: 10px; } +.active-filters .filter-value { + max-width: 200px; +} + +.active-filters .kv-label .default { + background-color: #eee; +} + .filter-panel { padding: 10px; background-color: #f5f5f5; diff --git a/frontend/app/styles/label.css b/frontend/app/styles/label.css index c4afc10d54..c37c35e2c6 100644 --- a/frontend/app/styles/label.css +++ b/frontend/app/styles/label.css @@ -14,8 +14,8 @@ white-space: nowrap; } .kv-label span.kv-label-key { - border-bottom-left-radius: 0.25em; - border-top-left-radius: .25em; + /* border-bottom-left-radius: 0.25em; + border-top-left-radius: .25em; */ border-left-color: #3c8dbc; border-left-width: 3px; border-left-style: solid; @@ -29,6 +29,17 @@ text-overflow: ellipsis; max-width: 200px; } + +.kv-label.kv-label-addon span.kv-label-val { + border-bottom-right-radius: 0; + border-top-right-radius: 0; +} + +.kv-label.kv-label-addon span:last-child { + border-bottom-right-radius: 0.25em; + border-top-right-radius: .25em; +} + .kv-label>span.primary { background-color: #3c8dbc; color: #fff; diff --git a/frontend/app/styles/main.css b/frontend/app/styles/main.css index cfb024f4db..3b328dba34 100644 --- a/frontend/app/styles/main.css +++ b/frontend/app/styles/main.css @@ -761,3 +761,8 @@ table.tbody-stripped>tbody+tbody { table.tbody-stripped > tbody > tr > td { } + +.ellipsable { + overflow: hidden; + text-overflow: ellipsis; +} diff --git a/frontend/app/views/components/search/filters-preview.component.html b/frontend/app/views/components/search/filters-preview.component.html index 7d28eeb9fd..778b3b648d 100644 --- a/frontend/app/views/components/search/filters-preview.component.html +++ b/frontend/app/views/components/search/filters-preview.component.html @@ -1,18 +1,18 @@ -
      -
        -
      • {{$ctrl.filters.length}} - filter(s) applied: -
        • -
      • - - {{filter.field || '???'}}: - {{filter.value | filterValue}} - - - -
        • -
      • - Clear filters -
        • -
      +
      + {{$ctrl.filters.length}} filter(s) applied: + + + {{filter.field || '???'}} + {{filter.value | filterValue}} + + + + + + + + Clear filters + +
      From f0daab7dcc497f5e0a3676ddf76c6b53c79ffa31 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 18 Nov 2020 15:46:06 +0100 Subject: [PATCH 209/237] #1662 Fix customField aggregations --- .../org/thp/thehive/controllers/v0/CaseCtrl.scala | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index c48612d9af..f8fcd8aeed 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -288,10 +288,17 @@ class PublicCase @Inject() ( } yield Json.obj("impactStatus" -> impactStatus) }) .property("customFields", UMapping.jsonNative)(_.subSelect { - case (FPathElem(_, FPathElem(name, _)), caseSteps) => - caseSteps - .customFields(EntityIdOrName(name)) - .jsonValue + case (FPathElem(_, FPathElem(name, _)), caseTraversal) => + db + .roTransaction(implicit graph => customFieldSrv.get(EntityIdOrName(name)).value(_.`type`).getOrFail("CustomField")) + .map { + case CustomFieldType.boolean => caseTraversal.customFields(EntityIdOrName(name)).value(_.booleanValue).domainMap(v => JsBoolean(v)) + case CustomFieldType.date => caseTraversal.customFields(EntityIdOrName(name)).value(_.dateValue).domainMap(v => JsNumber(v.getTime)) + case CustomFieldType.float => caseTraversal.customFields(EntityIdOrName(name)).value(_.floatValue).domainMap(v => JsNumber(v)) + case CustomFieldType.integer => caseTraversal.customFields(EntityIdOrName(name)).value(_.integerValue).domainMap(v => JsNumber(v)) + case CustomFieldType.string => caseTraversal.customFields(EntityIdOrName(name)).value(_.stringValue).domainMap(v => JsString(v)) + } + .getOrElse(caseTraversal.constant2(null)) case (_, caseSteps) => caseSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) } .filter { From 515c75ea7a45a1aceaa640c3cac4d732847efcac Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 18 Nov 2020 16:26:32 +0100 Subject: [PATCH 210/237] #1649 Fix the query used to list attachments --- frontend/app/scripts/controllers/case/CaseDetailsCtrl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js b/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js index 16b87faf41..030302eaae 100644 --- a/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js @@ -17,7 +17,7 @@ version: 'v1', loadAll: false, filter: { - '_contains': 'attachment' + '_contains': 'attachment.id' }, extraData: ['taskId'], pageSize: 100, From 6a25635f7e5200438cc894edc275147fa6947639 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 18 Nov 2020 17:15:42 +0100 Subject: [PATCH 211/237] #1666 Fix the responder error message display --- frontend/app/scripts/directives/responder-actions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/scripts/directives/responder-actions.js b/frontend/app/scripts/directives/responder-actions.js index 3dc14a528f..d8fbce6bb2 100644 --- a/frontend/app/scripts/directives/responder-actions.js +++ b/frontend/app/scripts/directives/responder-actions.js @@ -11,7 +11,7 @@ templateUrl: 'views/directives/responder-actions.html', controller: function($scope, $uibModal) { - $scope.$watch('actions', function(list) { + $scope.$watchCollection('actions.values', function(list) { if(!list) { return; } From b889dd9c90d5e286f1886533e65848c9936787dd Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 18 Nov 2020 18:27:55 +0100 Subject: [PATCH 212/237] #1598 Prevent failure on traversal when a data is corrupted --- ScalliGraph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScalliGraph b/ScalliGraph index 96c0bc6494..77cfc096a3 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 96c0bc6494d29146c69b62da62f685b2c7b247a5 +Subproject commit 77cfc096a366626887ab441299b538ad6f562868 From ec7739d6625b3fba0f7908cdf3569cc3d92d2a88 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 18 Nov 2020 18:49:51 +0100 Subject: [PATCH 213/237] #1653 Allow updating the layout of case and alert custom field panel --- .../services/common/ui/AppLayoutSrv.js | 14 ++++++++++- .../views/partials/alert/custom.fields.html | 4 ++-- .../views/partials/alert/event.dialog.html | 17 ++++++++++++++ .../partials/case/details/custom.fields.html | 23 ++++++++++++++++--- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/frontend/app/scripts/services/common/ui/AppLayoutSrv.js b/frontend/app/scripts/services/common/ui/AppLayoutSrv.js index 123efd80e1..e5711dc80b 100644 --- a/frontend/app/scripts/services/common/ui/AppLayoutSrv.js +++ b/frontend/app/scripts/services/common/ui/AppLayoutSrv.js @@ -8,7 +8,9 @@ this.init = function() { this.layout = localStorageService.get(key) || { - showFlow: true + showFlow: true, + caseCustomFieldColumns: 3, + alertCustomFieldColumns: 2 }; this.saveLayout(); @@ -33,6 +35,16 @@ this.saveLayout(); }; + this.caseCustomFields = function(columns) { + this.layout.caseCustomFieldColumns = columns; + this.saveLayout(); + }; + + this.alertCustomFields = function(columns) { + this.layout.alertCustomFieldColumns = columns; + this.saveLayout(); + }; + this.detachFlow = function(/*root*/) { this.showFlow(false); $window.open($state.href('live'), 'TheHiveLive', 'width=500,height=700,menubar=no,status=no,toolbar=no,location=no,scrollbars=yes'); diff --git a/frontend/app/views/partials/alert/custom.fields.html b/frontend/app/views/partials/alert/custom.fields.html index 2d3800ed50..5bdda66925 100644 --- a/frontend/app/views/partials/alert/custom.fields.html +++ b/frontend/app/views/partials/alert/custom.fields.html @@ -5,8 +5,8 @@
      -
      -
      +
      +

      Additional fields + + + + Layout + + +

      diff --git a/frontend/app/views/partials/case/details/custom.fields.html b/frontend/app/views/partials/case/details/custom.fields.html index d42f4985c7..27047213df 100644 --- a/frontend/app/views/partials/case/details/custom.fields.html +++ b/frontend/app/views/partials/case/details/custom.fields.html @@ -3,7 +3,7 @@

      Additional information - + Add
      • @@ -11,6 +11,23 @@

      + + + + Layout + + +

      @@ -19,8 +36,8 @@

      -
      -
      +
      +
      Date: Wed, 18 Nov 2020 21:24:47 +0100 Subject: [PATCH 214/237] #1669 Normalize the fields used to update objects --- frontend/app/index.html | 2 + frontend/app/styles/custom-fields.css | 10 ++++ frontend/app/styles/main.css | 7 ++- frontend/app/styles/updatable.css | 38 +++++++++++++++ .../common/custom-field-input.component.html | 36 +++++++------- .../views/directives/updatable-boolean.html | 27 +++++------ .../app/views/directives/updatable-date.html | 48 +++++++++---------- .../views/directives/updatable-select.html | 27 +++++------ .../directives/updatable-simple-text.html | 29 +++++------ .../app/views/directives/updatable-tags.html | 43 ++++++++++++++++- 10 files changed, 180 insertions(+), 87 deletions(-) create mode 100644 frontend/app/styles/custom-fields.css create mode 100644 frontend/app/styles/updatable.css diff --git a/frontend/app/index.html b/frontend/app/index.html index 5460192e47..800b08a1ae 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -43,12 +43,14 @@ + + diff --git a/frontend/app/styles/custom-fields.css b/frontend/app/styles/custom-fields.css new file mode 100644 index 0000000000..39513cef4c --- /dev/null +++ b/frontend/app/styles/custom-fields.css @@ -0,0 +1,10 @@ +/* .custom-field-input dd { + overflow: hidden; +} + + +div.custom-field-input-wrapper { + max-width: 100%; + text-overflow: ellipsis; +} + */ diff --git a/frontend/app/styles/main.css b/frontend/app/styles/main.css index 3b328dba34..0425ebbcd9 100644 --- a/frontend/app/styles/main.css +++ b/frontend/app/styles/main.css @@ -160,7 +160,7 @@ pre.clearpre { /***************************/ .flexwrap { - display: flex; + display: flex !important; flex-wrap: wrap; justify-content: flex-start; align-items: flex-start; @@ -340,6 +340,11 @@ ul.observable-reports-summary li { width: 200px !important; } +.case-details dd, +.case-custom-fields dd { + margin-left: 200px !important; +} + .case-custom-fields dt { background-color: #f9f9f9; padding-left: 5px; diff --git a/frontend/app/styles/updatable.css b/frontend/app/styles/updatable.css new file mode 100644 index 0000000000..4ef2fabbbc --- /dev/null +++ b/frontend/app/styles/updatable.css @@ -0,0 +1,38 @@ +.updatable-input { + /* border: 1px solid green; */ +} + +.updatable-input .updatable-value { + vertical-align: top; + white-space: pre-wrap +} + +.updatable-input .updatable-input-value-wrapper { + position: relative; + display: inline-block; + border-bottom: 1px solid #fff; +} + +.updatable-input .updatable-input-value-wrapper:hover{ + border-bottom: 1px solid #337ab7; + cursor: pointer; +} + +.updatable-input .updatable-input-value-wrapper .updatable-input-icon { + display: none; + float: right; + margin-left: 10px; + color: #337ab7; +} + +.updatable-input .updatable-input-value-wrapper .updatable-input-icon.lg { + line-height: 18px; +} + +.updatable-input .updatable-input-value-wrapper:hover .updatable-input-icon { + display: block; +} + +.updatable-input.updatable-input-tags { + +} diff --git a/frontend/app/views/components/common/custom-field-input.component.html b/frontend/app/views/components/common/custom-field-input.component.html index cde52043e6..6d62361643 100644 --- a/frontend/app/views/components/common/custom-field-input.component.html +++ b/frontend/app/views/components/common/custom-field-input.component.html @@ -1,4 +1,4 @@ -
      +
      {{$ctrl.field.name}}
      - +
      + - + - + - + - + - Not Editable + Not Editable +
      {{$ctrl.value | shortDate}} diff --git a/frontend/app/views/directives/updatable-boolean.html b/frontend/app/views/directives/updatable-boolean.html index 0c9c7e6e16..f2eb94ab1b 100644 --- a/frontend/app/views/directives/updatable-boolean.html +++ b/frontend/app/views/directives/updatable-boolean.html @@ -1,15 +1,14 @@ - - {{value ? trueText || 'True' : falseText || 'False'}} - Not Specified - - - - - -       - - -
      +
      + + {{value ? trueText || 'True' : falseText || 'False'}} + Not Specified + + + + + + +
      @@ -19,7 +18,7 @@ - +
      - +
      diff --git a/frontend/app/views/directives/updatable-date.html b/frontend/app/views/directives/updatable-date.html index 3e4a1e2964..6dc9c05410 100644 --- a/frontend/app/views/directives/updatable-date.html +++ b/frontend/app/views/directives/updatable-date.html @@ -1,29 +1,27 @@ - - {{value | shortDate}} - Not Specified - - - - - -       - - +
      + + {{value | shortDate}} + Not Specified -
      -
      - - + + + + + + +
      + + - Now - - + Now + + + + -
      - - +
      + +
      diff --git a/frontend/app/views/directives/updatable-select.html b/frontend/app/views/directives/updatable-select.html index 4fc1d7a403..e901a03d06 100644 --- a/frontend/app/views/directives/updatable-select.html +++ b/frontend/app/views/directives/updatable-select.html @@ -1,15 +1,14 @@ - - {{value}} - Not Specified - - - - - -       - - -
      +
      + + {{value}} + Not Specified + + + + + + +
      @@ -19,7 +18,7 @@ - +
      - +
      diff --git a/frontend/app/views/directives/updatable-simple-text.html b/frontend/app/views/directives/updatable-simple-text.html index ddabf16bbc..21cd440100 100644 --- a/frontend/app/views/directives/updatable-simple-text.html +++ b/frontend/app/views/directives/updatable-simple-text.html @@ -1,17 +1,18 @@ - - {{value}} - Not Specified - - - - - -       - - -
      +
      + + {{value}} + Not Specified + + + + + + +
      + + - +
      - +
      diff --git a/frontend/app/views/directives/updatable-tags.html b/frontend/app/views/directives/updatable-tags.html index 46ceceb402..d388b77f09 100644 --- a/frontend/app/views/directives/updatable-tags.html +++ b/frontend/app/views/directives/updatable-tags.html @@ -1,16 +1,54 @@ +
      + + Not Specified + + {{tag.text}} + + + + + + + + + + +
      + + + +
      +
      + + +
      +       +
      + + +       -       @@ -28,3 +66,4 @@
      +--> From bcdef7de39c0fcaa33d1fb9b43b98154df013804 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 19 Nov 2020 05:51:55 +0100 Subject: [PATCH 215/237] #1669 Update the user updatable component --- .../app/views/directives/updatable-user.html | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/frontend/app/views/directives/updatable-user.html b/frontend/app/views/directives/updatable-user.html index c2844dddcd..7581a6b787 100644 --- a/frontend/app/views/directives/updatable-user.html +++ b/frontend/app/views/directives/updatable-user.html @@ -1,26 +1,25 @@ - - - - - - - -       +
      + + - - - - - + + + - - + +
      + + + +
      -       +
      From 0fd7464d9ab601f89bcc95c627d1838a773eac53 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 19 Nov 2020 06:32:09 +0100 Subject: [PATCH 216/237] #1669 Clean up --- frontend/app/styles/updatable.css | 5 ++ .../app/views/directives/updatable-tags.html | 41 --------------- .../app/views/directives/updatable-text.html | 51 +++++++++---------- 3 files changed, 30 insertions(+), 67 deletions(-) diff --git a/frontend/app/styles/updatable.css b/frontend/app/styles/updatable.css index 4ef2fabbbc..397ce021f8 100644 --- a/frontend/app/styles/updatable.css +++ b/frontend/app/styles/updatable.css @@ -36,3 +36,8 @@ .updatable-input.updatable-input-tags { } + +.updatable-input.updatable-input-text .updatable-input-markdown .updatable-input-icon { + float:left; + margin-left: 0; +} diff --git a/frontend/app/views/directives/updatable-tags.html b/frontend/app/views/directives/updatable-tags.html index d388b77f09..feaf4e1a7a 100644 --- a/frontend/app/views/directives/updatable-tags.html +++ b/frontend/app/views/directives/updatable-tags.html @@ -8,15 +8,6 @@ - - -       @@ -35,35 +26,3 @@
      - - diff --git a/frontend/app/views/directives/updatable-text.html b/frontend/app/views/directives/updatable-text.html index 05b865e1fd..557019bb56 100644 --- a/frontend/app/views/directives/updatable-text.html +++ b/frontend/app/views/directives/updatable-text.html @@ -1,37 +1,36 @@ - - - - - - +
      + + + Not Specified + + + -
      - - - Not specified - - - - - + + + + + + +
      -       - -
      +
      -
      - - - - - - +
      +
      + + +
      Markdown Reference
      - +
      From e9f1e5bd8f3da352ba3dda11fe499348e6b740e4 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 19 Nov 2020 06:44:35 +0100 Subject: [PATCH 217/237] #1669 Support `clearable` option in all updatable fields and use it in custom field panels --- frontend/app/scripts/directives/updatableBoolean.js | 3 ++- .../app/scripts/directives/updatableDataDropdown.js | 3 ++- frontend/app/scripts/directives/updatableDate.js | 9 ++++++++- frontend/app/scripts/directives/updatableSelect.js | 3 ++- .../app/scripts/directives/updatableSimpleText.js | 3 ++- frontend/app/scripts/directives/updatableTags.js | 3 ++- frontend/app/scripts/directives/updatableText.js | 3 ++- frontend/app/scripts/directives/updatableUser.js | 3 ++- frontend/app/scripts/services/common/UtilsSrv.js | 3 +++ .../common/custom-field-input.component.html | 12 ++++++------ frontend/app/views/directives/updatable-boolean.html | 3 +++ frontend/app/views/directives/updatable-date.html | 3 +++ frontend/app/views/directives/updatable-select.html | 3 +++ .../app/views/directives/updatable-simple-text.html | 3 +++ frontend/app/views/directives/updatable-tags.html | 3 +++ frontend/app/views/directives/updatable-text.html | 3 +++ frontend/app/views/directives/updatable-user.html | 5 ++++- 17 files changed, 53 insertions(+), 15 deletions(-) diff --git a/frontend/app/scripts/directives/updatableBoolean.js b/frontend/app/scripts/directives/updatableBoolean.js index e6e231e814..5d84fc826f 100644 --- a/frontend/app/scripts/directives/updatableBoolean.js +++ b/frontend/app/scripts/directives/updatableBoolean.js @@ -12,7 +12,8 @@ 'active': '=?', 'placeholder': '@', 'trueText': '@?', - 'falseText': '@?' + 'falseText': '@?', + 'clearable': ' + value="$ctrl.value" clearable="true">
      + input-type="text" on-update="$ctrl.updateField(newValue)" value="$ctrl.value" clearable="true"> - + + value="$ctrl.value" clearable="true"> + value="$ctrl.value" clearable="true"> + value="$ctrl.value" clearable="true"> Not Editable
      diff --git a/frontend/app/views/directives/updatable-boolean.html b/frontend/app/views/directives/updatable-boolean.html index f2eb94ab1b..4a383096de 100644 --- a/frontend/app/views/directives/updatable-boolean.html +++ b/frontend/app/views/directives/updatable-boolean.html @@ -18,6 +18,9 @@ +
      diff --git a/frontend/app/views/directives/updatable-date.html b/frontend/app/views/directives/updatable-date.html index 6dc9c05410..0479e71ec3 100644 --- a/frontend/app/views/directives/updatable-date.html +++ b/frontend/app/views/directives/updatable-date.html @@ -21,6 +21,9 @@ +
      diff --git a/frontend/app/views/directives/updatable-select.html b/frontend/app/views/directives/updatable-select.html index e901a03d06..d7ade46e19 100644 --- a/frontend/app/views/directives/updatable-select.html +++ b/frontend/app/views/directives/updatable-select.html @@ -18,6 +18,9 @@ +
      diff --git a/frontend/app/views/directives/updatable-simple-text.html b/frontend/app/views/directives/updatable-simple-text.html index 21cd440100..9a6bbb2b14 100644 --- a/frontend/app/views/directives/updatable-simple-text.html +++ b/frontend/app/views/directives/updatable-simple-text.html @@ -20,6 +20,9 @@ +

      diff --git a/frontend/app/views/directives/updatable-tags.html b/frontend/app/views/directives/updatable-tags.html index feaf4e1a7a..e835f01ed3 100644 --- a/frontend/app/views/directives/updatable-tags.html +++ b/frontend/app/views/directives/updatable-tags.html @@ -23,6 +23,9 @@ +
      diff --git a/frontend/app/views/directives/updatable-text.html b/frontend/app/views/directives/updatable-text.html index 557019bb56..ef2ec25a26 100644 --- a/frontend/app/views/directives/updatable-text.html +++ b/frontend/app/views/directives/updatable-text.html @@ -28,6 +28,9 @@ +
      Markdown Reference diff --git a/frontend/app/views/directives/updatable-user.html b/frontend/app/views/directives/updatable-user.html index 7581a6b787..a64a013626 100644 --- a/frontend/app/views/directives/updatable-user.html +++ b/frontend/app/views/directives/updatable-user.html @@ -4,7 +4,7 @@ - +
      @@ -15,6 +15,9 @@ +
      • {{user.organisation}}/{{user.name}} From 37d08cb46fc760fd58b1573909b4f983895d45b8 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 19 Nov 2020 06:57:25 +0100 Subject: [PATCH 218/237] #1669 Remove the pointer cursor from markdown updatable fields --- frontend/app/styles/updatable.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/app/styles/updatable.css b/frontend/app/styles/updatable.css index 397ce021f8..7363d2637c 100644 --- a/frontend/app/styles/updatable.css +++ b/frontend/app/styles/updatable.css @@ -37,6 +37,9 @@ } +.updatable-input.updatable-input-text .updatable-input-markdown { + cursor: auto; +} .updatable-input.updatable-input-text .updatable-input-markdown .updatable-input-icon { float:left; margin-left: 0; From 9506fe3d2a38ea8b513df5a0eaef041bcaa87e58 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 19 Nov 2020 07:19:12 +0100 Subject: [PATCH 219/237] #1669 Refactor severity and tlp components --- frontend/app/views/directives/tlp.html | 39 ++++++++----------- .../app/views/partials/case/case.details.html | 2 +- .../partials/observables/details/summary.html | 2 +- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/frontend/app/views/directives/tlp.html b/frontend/app/views/directives/tlp.html index 2b4b76e5ab..a72abb610d 100644 --- a/frontend/app/views/directives/tlp.html +++ b/frontend/app/views/directives/tlp.html @@ -1,23 +1,18 @@ - -
        - - WHITE - GREEN - AMBER - RED -
        - - - {{namespace || 'TLP'}}:WHITE - {{namespace || 'TLP'}}:GREEN - {{namespace || 'TLP'}}:AMBER - {{namespace || 'TLP'}}:RED - - - - - - - - +
        + WHITE + GREEN + AMBER + RED +
        + + {{namespace || 'TLP'}}:WHITE + {{namespace || 'TLP'}}:GREEN + {{namespace || 'TLP'}}:AMBER + {{namespace || 'TLP'}}:RED + + + + + + diff --git a/frontend/app/views/partials/case/case.details.html b/frontend/app/views/partials/case/case.details.html index 8a9b2cc1ca..79ead25542 100644 --- a/frontend/app/views/partials/case/case.details.html +++ b/frontend/app/views/partials/case/case.details.html @@ -16,7 +16,7 @@

        Summary

        Severity
        - +
        diff --git a/frontend/app/views/partials/observables/details/summary.html b/frontend/app/views/partials/observables/details/summary.html index 741319e11a..360ea6f016 100644 --- a/frontend/app/views/partials/observables/details/summary.html +++ b/frontend/app/views/partials/observables/details/summary.html @@ -25,7 +25,7 @@

        TLP
        - +
        From 8d0dd4193839c91e859f5f4dd0e7478b15f5266e Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 19 Nov 2020 09:05:06 +0100 Subject: [PATCH 220/237] #1652 Clone traversal to fix updates --- thehive/app/org/thp/thehive/services/LogSrv.scala | 8 +++----- thehive/app/org/thp/thehive/services/ObservableSrv.scala | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 7f8f68b25d..9e0aa4f5b5 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -68,11 +68,9 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv, taskSr )(implicit graph: Graph, authContext: AuthContext): Try[(Traversal.V[Log], JsObject)] = auditSrv.mergeAudits(super.update(traversal, propertyUpdaters)) { case (logSteps, updatedFields) => - for { - task <- logSteps.clone().task.getOrFail("Task") - log <- logSteps.getOrFail("Log") - _ <- auditSrv.log.update(log, task, updatedFields) - } yield () + logSteps.clone().project(_.by.by(_.task)).getOrFail("Log").flatMap { + case (log, task) => auditSrv.log.update(log, task, updatedFields) + } } } diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 2a156ec43f..dc47f08504 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -183,10 +183,10 @@ class ObservableSrv @Inject() ( )(implicit graph: Graph, authContext: AuthContext): Try[(Traversal.V[Observable], JsObject)] = auditSrv.mergeAudits(super.update(traversal, propertyUpdaters)) { case (observableSteps, updatedFields) => - for { - observable <- observableSteps.getOrFail("Observable") - _ <- auditSrv.observable.update(observable, updatedFields) - } yield () + observableSteps + .clone() + .getOrFail("Observable") + .flatMap(observable => auditSrv.observable.update(observable, updatedFields)) } } From f7c062073afcb0040afdf8abd78ed708fa1aa937 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 19 Nov 2020 09:57:29 +0100 Subject: [PATCH 221/237] #1669 Refactor updatable fields to fix the edit icon --- frontend/app/styles/custom-fields.css | 11 ++--------- frontend/app/styles/updatable.css | 20 ++++++++------------ 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/frontend/app/styles/custom-fields.css b/frontend/app/styles/custom-fields.css index 39513cef4c..06437a4ed3 100644 --- a/frontend/app/styles/custom-fields.css +++ b/frontend/app/styles/custom-fields.css @@ -1,10 +1,3 @@ -/* .custom-field-input dd { - overflow: hidden; +.custom-field-input dd { + overflow-x: auto; } - - -div.custom-field-input-wrapper { - max-width: 100%; - text-overflow: ellipsis; -} - */ diff --git a/frontend/app/styles/updatable.css b/frontend/app/styles/updatable.css index 7363d2637c..13a11cdb75 100644 --- a/frontend/app/styles/updatable.css +++ b/frontend/app/styles/updatable.css @@ -1,16 +1,13 @@ -.updatable-input { - /* border: 1px solid green; */ -} - .updatable-input .updatable-value { vertical-align: top; - white-space: pre-wrap + white-space: pre-wrap; } .updatable-input .updatable-input-value-wrapper { position: relative; display: inline-block; border-bottom: 1px solid #fff; + padding-right: 20px; } .updatable-input .updatable-input-value-wrapper:hover{ @@ -30,17 +27,16 @@ } .updatable-input .updatable-input-value-wrapper:hover .updatable-input-icon { - display: block; -} - -.updatable-input.updatable-input-tags { - + display: inline-block; + position: absolute; + top: 0; + right: 0; } .updatable-input.updatable-input-text .updatable-input-markdown { cursor: auto; } .updatable-input.updatable-input-text .updatable-input-markdown .updatable-input-icon { - float:left; - margin-left: 0; + /* float:left; + margin-left: 0; */ } From 1aa821a3da13b9c596dc0d6b73667df14fa572ba Mon Sep 17 00:00:00 2001 From: "Kaan S. Karadag" Date: Thu, 19 Nov 2020 13:04:46 +0200 Subject: [PATCH 222/237] Wrong data volume mapping for thehive (#1582) thehive application.conf points to /opt/data for file storage but docker-compose file mounts to /data. This commit fixes the mapping. --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4040565e44..b7b475d4d5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -67,7 +67,7 @@ services: - '0.0.0.0:9000:9000' volumes: - ./thehive/application.conf:/etc/thehive/application.conf - - ./data:/data + - ./data:/opt/data command: '--no-config --no-config-secret' redis: From 2387ed9a41850546acf98164415499e804859899 Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 19 Nov 2020 14:18:39 +0100 Subject: [PATCH 223/237] fix localfs parameter --- docker/thehive/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/thehive/application.conf b/docker/thehive/application.conf index 793d661be0..b6ed0da698 100644 --- a/docker/thehive/application.conf +++ b/docker/thehive/application.conf @@ -20,7 +20,7 @@ db { storage { provider: localfs - localfs.directory: /opt/data + localfs.location: /opt/data } play.modules.enabled += org.thp.thehive.connector.cortex.CortexModule From eca5a23fae410a5d4dbfd7970dceca24c8c896d6 Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 19 Nov 2020 14:31:16 +0100 Subject: [PATCH 224/237] added container name for mysql and misp modules --- docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b7b475d4d5..1bb7b6b63e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -77,6 +77,7 @@ services: db: image: mysql:latest + container_name: mysql restart: unless-stopped command: --default-authentication-plugin=mysql_native_password restart: always @@ -105,6 +106,7 @@ services: - "DISIPV6=true" # Disable IPV6 in nginx misp-modules: image: coolacid/misp-docker:modules-latest + container_name: misp-modules environment: - "REDIS_BACKEND=redis" depends_on: From d26c4dc95f7a791187880b684a14d8408440c0dd Mon Sep 17 00:00:00 2001 From: garanews Date: Thu, 19 Nov 2020 14:50:56 +0100 Subject: [PATCH 225/237] added pictures about docker images --- docker/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 9211c3ab3e..2bb97b37c2 100644 --- a/docker/README.md +++ b/docker/README.md @@ -153,4 +153,9 @@ In conclusion, after execute ```sudo docker-compose up``` you will have the foll | Elasticsearch | http://localhost:9200 | | | Kibana | http://localhost:5601 | | | MISP | https://localhost:443 | admin@admin.test | admin -| Shuffle | http://localhost:3001 | | \ No newline at end of file +| Shuffle | http://localhost:3001 | | + + + +![image](https://user-images.githubusercontent.com/16938405/99674126-e8c99f80-2a75-11eb-9a8b-1603cf67d665.png) +![image](https://user-images.githubusercontent.com/16938405/99674544-7c02d500-2a76-11eb-92a5-3fbb5c3c5cc5.png) From 9888ed8bbcb5b506140b82fcb4ba14c1b1371fe2 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 19 Nov 2020 15:17:34 +0100 Subject: [PATCH 226/237] #1673 Fix alert creation when contains empty file observable --- thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index a3187ae1f3..855cc83b39 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -316,6 +316,10 @@ class AlertCtrl @Inject() ( attachmentSrv .create(filename, contentType, data) .flatMap(attachment => observableSrv.create(observable.toObservable, attachmentType, attachment, observable.tags, Nil)) + case Array(filename, contentType) => + attachmentSrv + .create(filename, contentType, Array.emptyByteArray) + .flatMap(attachment => observableSrv.create(observable.toObservable, attachmentType, attachment, observable.tags, Nil)) case data => Failure(InvalidFormatAttributeError("artifacts.data", "filename;contentType;base64value", Set.empty, FString(data.mkString(";")))) } From 87a10ff2cda9a254c9a28ad7ec63d881dcd53378 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 19 Nov 2020 16:25:51 +0100 Subject: [PATCH 227/237] #1672 Refine alert modal and case related alerts panel --- frontend/app/scripts/app.js | 6 +-- .../app/scripts/controllers/SearchCtrl.js | 16 +++++-- .../controllers/alert/AlertEventCtrl.js | 6 +-- .../controllers/case/CaseAlertsCtrl.js | 11 ++++- .../app/scripts/services/api/AlertingSrv.js | 15 +++---- frontend/app/scripts/services/api/CaseSrv.js | 27 +++++++++--- .../views/partials/alert/event.dialog.html | 4 +- .../app/views/partials/case/case.alerts.html | 42 +++++++++---------- 8 files changed, 74 insertions(+), 53 deletions(-) diff --git a/frontend/app/scripts/app.js b/frontend/app/scripts/app.js index 019121422f..800a72397d 100644 --- a/frontend/app/scripts/app.js +++ b/frontend/app/scripts/app.js @@ -411,11 +411,7 @@ angular.module('thehive', [ controller: 'CaseAlertsCtrl', resolve: { alerts: function($stateParams, CaseSrv) { - return CaseSrv.alerts({range: 'all'}, { - query: { - case: $stateParams.caseId - } - }).$promise; + return CaseSrv.alerts($stateParams.caseId); } }, guard: { diff --git a/frontend/app/scripts/controllers/SearchCtrl.js b/frontend/app/scripts/controllers/SearchCtrl.js index 0ba6db0c72..6eb5522550 100644 --- a/frontend/app/scripts/controllers/SearchCtrl.js +++ b/frontend/app/scripts/controllers/SearchCtrl.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveControllers') - .controller('SearchCtrl', function($scope, $q, $stateParams, $uibModal, PSearchSrv, CaseTemplateSrv, CaseTaskSrv, NotificationSrv, EntitySrv, UserSrv, QueryBuilderSrv, GlobalSearchSrv, metadata) { + .controller('SearchCtrl', function($scope, $q, $stateParams, $uibModal, PSearchSrv, AlertingSrv, CaseTemplateSrv, CaseTaskSrv, NotificationSrv, EntitySrv, UserSrv, QueryBuilderSrv, GlobalSearchSrv, metadata) { $scope.metadata = metadata; $scope.toolbar = [ // {name: 'all', label: 'All', icon: 'glyphicon glyphicon-search'}, @@ -42,14 +42,24 @@ controllerAs: 'dialog', size: 'max', resolve: { - event: event, + event: function() { + return AlertingSrv.get(event.id); + }, templates: function() { return CaseTemplateSrv.list(); }, readonly: true } - }).result.then(function(/*response*/) { + }) + .result + .then(function(/*response*/) { $scope.searchResults.update(); + }) + .catch(function(err) { + if(err && !_.isString(err)) { + NotificationSrv.error('AlertPreview', err.data, err.status); + } + }); }; diff --git a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js index c36e492711..c1b8d09075 100644 --- a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js @@ -39,11 +39,6 @@ AlertingSrv.get(eventId).then(function(data) { self.event = data; self.loading = false; - - self.dataTypes = _.countBy(self.event.artifacts, function(attr) { - return attr.dataType; - }); - }, function(response) { self.loading = false; NotificationSrv.error('AlertEventCtrl', response.data, response.status); @@ -185,6 +180,7 @@ self.copyId = function(id) { clipboard.copyText(id); + NotificationSrv.log('Alert ID has been copied to clipboard', 'success'); }; this.$onInit = function() { diff --git a/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js b/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js index f410a7e27f..b695e32698 100644 --- a/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveControllers').controller('CaseAlertsCtrl', - function($scope, $state, $stateParams, $uibModal, $timeout, CaseTabsSrv, VersionSrv, alerts) { + function($scope, $state, $stateParams, $uibModal, $timeout, CaseTabsSrv, VersionSrv, NotificationSrv, alerts) { $scope.caseId = $stateParams.caseId; $scope.alerts = alerts; $scope.alertStats = []; @@ -84,9 +84,16 @@ templates: function() { //return CaseTemplateSrv.list(); return []; - }, + }, readonly: true } + }) + .result + .catch(function(err) { + if(err && !_.isString(err)) { + NotificationSrv.error('AlertPreview', err.data, err.status); + } + }); }; diff --git a/frontend/app/scripts/services/api/AlertingSrv.js b/frontend/app/scripts/services/api/AlertingSrv.js index 954644aec3..0d20782547 100644 --- a/frontend/app/scripts/services/api/AlertingSrv.js +++ b/frontend/app/scripts/services/api/AlertingSrv.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveServices') - .factory('AlertingSrv', function($q, $http, $rootScope, StatSrv, StreamSrv, PSearchSrv, PaginatedQuerySrv, QuerySrv) { + .factory('AlertingSrv', function($q, $http, $rootScope, StatSrv, StreamSrv, PSearchSrv, PaginatedQuerySrv) { var baseUrl = './api/alert'; @@ -144,15 +144,10 @@ }, get: function(alertId) { - return QuerySrv.call('v1', [{ - '_name': 'getAlert', - 'idOrName': alertId - } - ], { - name: 'get-alert-' + alertId - }).then(function(response) { - return response[0]; - }); + return $http.get('./api/v1/alert/' + alertId) + .then(function(response) { + return response.data; + }); }, create: function(alertId, data) { diff --git a/frontend/app/scripts/services/api/CaseSrv.js b/frontend/app/scripts/services/api/CaseSrv.js index e5e391ac98..1597dd9f9a 100644 --- a/frontend/app/scripts/services/api/CaseSrv.js +++ b/frontend/app/scripts/services/api/CaseSrv.js @@ -32,11 +32,11 @@ url: './api/case/_search', isArray: true }, - alerts: { - method: 'POST', - url: './api/alert/_search', - isArray: true - } + // alerts: { + // method: 'POST', + // url: './api/alert/_search', + // isArray: true + // } }); this.get = resource.get; @@ -48,6 +48,23 @@ this.merge = resource.merge; this.query = resource.query; + this.alerts = function(id) { + var defer = $q.defer(); + + QuerySrv.call('v1', [{ + '_name': 'getCase', + 'idOrName': id + }, {'_name': 'alerts'}], { + name:'get-case-alerts' + id + }).then(function(response) { + defer.resolve(response); + }).catch(function(err){ + defer.reject(err); + }); + + return defer.promise; + }; + this.getById = function(id, withStats) { var defer = $q.defer(); diff --git a/frontend/app/views/partials/alert/event.dialog.html b/frontend/app/views/partials/alert/event.dialog.html index e43d493888..465646c9cf 100644 --- a/frontend/app/views/partials/alert/event.dialog.html +++ b/frontend/app/views/partials/alert/event.dialog.html @@ -1,6 +1,6 @@

        - Alert Preview {{dialog.event.read ? 'Ignored' : 'New'}} + Alert Preview {{dialog.event.caseId ? 'Imported' : 'New'}}

        @@ -20,7 +20,7 @@

        ID: - {{dialog.event._id}} + {{dialog.event._id}} Date: diff --git a/frontend/app/views/partials/case/case.alerts.html b/frontend/app/views/partials/case/case.alerts.html index 6ad492473f..590976743b 100644 --- a/frontend/app/views/partials/case/case.alerts.html +++ b/frontend/app/views/partials/case/case.alerts.html @@ -29,7 +29,7 @@

        - +
        - - - - + + - - + + + + + + + + +
        @@ -40,7 +40,7 @@ + Type @@ -48,14 +48,6 @@ - - Status - - - - - Title @@ -94,8 +86,8 @@
        {{::event.sourceRef}} @@ -109,18 +101,10 @@ {{::event.type}} - {{::event.status}} -
        {{::event.title}}
        -
        - - None - {{tag}} -
        		{{::event.source}} @@ -129,13 +113,29 @@ {{::event.artifacts.length || 0}}{{event.date | showDate}}{{event.date | shortDate}}
        +
        + + None + {{tag}} +
        +
        + +

      From 14281beb99c1e1a03626953e12a4668e80207426 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 08:50:53 +0100 Subject: [PATCH 228/237] Update sbt and play --- project/build.properties | 2 +- project/plugins.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project/build.properties b/project/build.properties index 08e4d79332..947bdd3020 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.4.1 +sbt.version=1.4.3 diff --git a/project/plugins.sbt b/project/plugins.sbt index 0eacab22d7..171541c183 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.3") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.5") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.3.0") addSbtPlugin("org.thehive-project" % "sbt-github-changelog" % "0.3.0") From c89cdf24ba722aa19871b11cb6c0ea2defa5c947 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 08:53:11 +0100 Subject: [PATCH 229/237] #1676 Fix creation of duplicated audit during migration --- ScalliGraph | 2 +- .../org/thp/thehive/migration/Input.scala | 2 + .../thp/thehive/migration/MigrationOps.scala | 10 ++--- .../org/thp/thehive/migration/th3/Input.scala | 38 +++++++++++++++++-- .../thp/thehive/migration/th4/Output.scala | 25 +++++++----- 5 files changed, 58 insertions(+), 19 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 77cfc096a3..f6a4d2165c 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 77cfc096a366626887ab441299b538ad6f562868 +Subproject commit f6a4d2165c26826c5b28db1a513ade15dfb060f2 diff --git a/migration/src/main/scala/org/thp/thehive/migration/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/Input.scala index ddca2631d0..e6037cceeb 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Input.scala @@ -137,9 +137,11 @@ trait Input { def listAction(filter: Filter): Source[Try[(String, InputAction)], NotUsed] def countAction(filter: Filter): Future[Long] def listAction(entityId: String): Source[Try[(String, InputAction)], NotUsed] + def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed] def countAction(entityId: String): Future[Long] def listAudit(filter: Filter): Source[Try[(String, InputAudit)], NotUsed] def countAudit(filter: Filter): Future[Long] def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed] + def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed] def countAudit(entityId: String, filter: Filter): Future[Long] } diff --git a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala index aa12e5990f..dd4e70fb06 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala @@ -7,7 +7,7 @@ import org.thp.scalligraph.{EntityId, NotFoundError, RichOptionTry} import org.thp.thehive.migration.dto.{InputAlert, InputAudit, InputCase, InputCaseTemplate} import play.api.Logger -import scala.collection.{immutable, mutable} +import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} @@ -251,10 +251,10 @@ trait MigrationOps { output.createJobObservable ) caseEntitiesIds = caseTaskIds ++ caseTaskLogIds ++ caseObservableIds ++ jobIds ++ jobObservableIds :+ caseId - actionSource = Source(caseEntitiesIds.to[immutable.Iterable]).flatMapConcat(id => input.listAction(id.inputId)) + actionSource = input.listActions(caseEntitiesIds.map(_.inputId).distinct) actionIds <- migrateWithParent("Action", caseEntitiesIds, actionSource, output.createAction) caseEntitiesAuditIds = caseEntitiesIds ++ actionIds - auditSource = Source(caseEntitiesAuditIds.to[immutable.Iterable]).flatMapConcat(id => input.listAudit(id.inputId, filter)) + auditSource = input.listAudits(caseEntitiesAuditIds.map(_.inputId).distinct, filter) _ <- migrateAudit(caseEntitiesAuditIds, auditSource, output.createAudit) } yield Some(caseId) } @@ -282,10 +282,10 @@ trait MigrationOps { output.createAlertObservable ) alertEntitiesIds = alertId +: alertObservableIds - actionSource = Source(alertEntitiesIds.to[immutable.Iterable]).flatMapConcat(id => input.listAction(id.inputId)) + actionSource = input.listActions(alertEntitiesIds.map(_.inputId).distinct) actionIds <- migrateWithParent("Action", alertEntitiesIds, actionSource, output.createAction) alertEntitiesAuditIds = alertEntitiesIds ++ actionIds - auditSource = Source(alertEntitiesAuditIds.to[immutable.Iterable]).flatMapConcat(id => input.listAudit(id.inputId, filter)) + auditSource = input.listAudits(alertEntitiesAuditIds.map(_.inputId).distinct, filter) _ <- migrateAudit(alertEntitiesAuditIds, auditSource, output.createAudit) } yield () } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala index 050e15593d..c6ce3faca7 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala @@ -639,7 +639,16 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(termQuery("relations", "action")))._2 override def listAction(entityId: String): Source[Try[(String, InputAction)], NotUsed] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(Seq(termQuery("relations", "action"), idsQuery(entityId)), Nil, Nil))) + dbFind(Some("all"), Nil)(indexName => + search(indexName).query(bool(Seq(termQuery("relations", "action"), termQuery("objectId", entityId)), Nil, Nil)) + ) + ._1 + .read[(String, InputAction)] + + override def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed] = + dbFind(Some("all"), Nil)(indexName => + search(indexName).query(bool(Seq(termQuery("relations", "action"), termsQuery("objectId", entityIds)), Nil, Nil)) + ) ._1 .read[(String, InputAction)] @@ -679,11 +688,34 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe override def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed] = dbFind(Some("all"), Nil)(indexName => - search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), Nil, Nil)) + search(indexName).query( + bool( + auditFilter(filter) ++ auditIncludeFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), + Nil, + auditExcludeFilter(filter) + ) + ) + )._1.read[(String, InputAudit)] + + override def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed] = + dbFind(Some("all"), Nil)(indexName => + search(indexName).query( + bool( + auditFilter(filter) ++ auditIncludeFilter(filter) :+ termQuery("relations", "audit") :+ termsQuery("objectId", entityIds), + Nil, + auditExcludeFilter(filter) + ) + ) )._1.read[(String, InputAudit)] def countAudit(entityId: String, filter: Filter): Future[Long] = dbFind(Some("0-0"), Nil)(indexName => - search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), Nil, Nil)) + search(indexName).query( + bool( + auditFilter(filter) ++ auditIncludeFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), + Nil, + auditExcludeFilter(filter) + ) + ) )._2 } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 68ed3a2167..fe388fb90a 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -32,6 +32,7 @@ import play.api.{Configuration, Environment, Logger} import scala.collection.JavaConverters._ import scala.concurrent.ExecutionContext import scala.util.{Failure, Success, Try} +import org.thp.thehive.controllers.v1.Conversion._ object Output { @@ -220,15 +221,15 @@ class Output @Inject() ( alerts.nonEmpty ) logger.info(s"""Already migrated: - | ${profiles.size} profiles\n - | ${organisations.size} organisations\n - | ${users.size} users\n - | ${impactStatuses.size} impactStatuses\n - | ${resolutionStatuses.size} resolutionStatuses\n - | ${observableTypes.size} observableTypes\n - | ${customFields.size} customFields\n - | ${caseTemplates.size} caseTemplates\n - | ${caseNumbers.size} caseNumbers\n + | ${profiles.size} profiles + | ${organisations.size} organisations + | ${users.size} users + | ${impactStatuses.size} impactStatuses + | ${resolutionStatuses.size} resolutionStatuses + | ${observableTypes.size} observableTypes + | ${customFields.size} customFields + | ${caseTemplates.size} caseTemplates + | ${caseNumbers.size} caseNumbers | ${alerts.size} alerts""".stripMargin) } @@ -579,7 +580,9 @@ class Output @Inject() ( for { task <- taskSrv.getOrFail(taskId) _ = logger.debug(s"Create log in task ${task.title}") - log <- logSrv.create(inputLog.log, task) + log <- logSrv.createEntity(inputLog.log) + _ <- logSrv.taskLogSrv.create(TaskLog(), task, log) + _ <- auditSrv.log.create(log, task, RichLog(log, Nil).toJson) _ = updateMetaData(log, inputLog.metaData) _ <- inputLog.attachments.toTry { inputAttachment => attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { attachment => @@ -717,6 +720,7 @@ class Output @Inject() ( case "Log" => logSrv.getOrFail(entityId) case "Alert" => alertSrv.getOrFail(entityId) case "Job" => jobSrv.getOrFail(entityId) + case "Action" => actionSrv.getOrFail(entityId) case _ => Failure(BadRequestError(s"objectType $entityType is not recognised")) } @@ -744,6 +748,7 @@ class Output @Inject() ( case "Alert" => "Alert" case "Log" | "Task" | "Observable" | "Case" | "Job" => "Case" case "User" => "User" + case "Action" => "Action" // FIXME case other => logger.error(s"Unknown object type: $other") other From 1b572829d3ed0ed5ca0fe48aa17e4fa29165509d Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 15:04:25 +0100 Subject: [PATCH 230/237] #1625 Fix similarity when several observables match --- thehive/app/org/thp/thehive/services/AlertSrv.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index f4927bde2d..49c91f328d 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -417,10 +417,9 @@ object AlertOps { ) .by( _.selectValues - .unfold .project( - _.by(_.groupCount(_.byValue(_.ioc))) - .by(_.groupCount(_.by(_.typeName))) + _.by(_.unfold.groupCount(_.byValue(_.ioc))) + .by(_.unfold.groupCount(_.by(_.typeName))) ) ) ) From 2b609faa392ff641a494b82437149afd5fe9db2b Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 08:56:02 +0100 Subject: [PATCH 231/237] #1675 Add group in task --- thehive/app/org/thp/thehive/controllers/v1/Properties.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 4f1122e467..a41f6a537d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -413,6 +413,7 @@ class Properties @Inject() ( .property("endDate", UMapping.date.optional)(_.field.updatable) .property("order", UMapping.int)(_.field.updatable) .property("dueDate", UMapping.date.optional)(_.field.updatable) + .property("group", UMapping.string)(_.field.updatable) .property("assignee", UMapping.string.optional)(_.select(_.assignee.value(_.login)).custom { case (_, value, vertex, _, graph, authContext) => taskSrv From 38b6f933b0bab400ad912d0823b18f005919e81b Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 09:14:37 +0100 Subject: [PATCH 232/237] #1675 Fix observable _type --- thehive/app/org/thp/thehive/controllers/v1/Conversion.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 5c569fb0d0..ac556fca70 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -369,7 +369,7 @@ object Conversion { case (richObservable, extraData) => richObservable .into[OutputObservable] - .withFieldConst(_._type, "case_artifact") + .withFieldConst(_._type, "Observable") .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.dataType, _.`type`.name) .withFieldComputed(_.startDate, _.observable._createdAt) From 0dfaa2a32592e87c31cd9eaeee2b244952c84cae Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 09:18:35 +0100 Subject: [PATCH 233/237] #1675 Add "case" in logs extraData --- .../thp/thehive/controllers/v1/LogCtrl.scala | 4 +- .../thehive/controllers/v1/LogRenderer.scala | 59 +++++++++++-------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala index 0a4a08dfe3..fcd7e2be74 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala @@ -41,9 +41,9 @@ class LogCtrl @Inject() ( override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Log], IteratorOutput]( "page", FieldsParser[OutputParam], - (range, logSteps, _) => + (range, logSteps, authContext) => logSteps.richPage(range.from, range.to, range.extraData.contains("total"))( - _.richLogWithCustomRenderer(logStatsRenderer(range.extraData - "total")) + _.richLogWithCustomRenderer(logStatsRenderer(range.extraData - "total")(authContext)) ) ) override val outputQuery: Query = Query.output[RichLog, Traversal.V[Log]](_.richLog) diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala index a06de4a596..f28671eeda 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala @@ -4,16 +4,23 @@ import java.lang.{Long => JLong} import java.util.{List => JList, Map => JMap} import org.apache.tinkerpop.gremlin.structure.Vertex +import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models.Log +import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.LogOps._ import org.thp.thehive.services.TaskOps._ import play.api.libs.json._ trait LogRenderer { + def caseParent(implicit + authContext: AuthContext + ): Traversal.V[Log] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = + _.`case`.richCase.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) + def taskParent: Traversal.V[Log] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = _.task.richTask.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) @@ -23,33 +30,35 @@ trait LogRenderer { def actionCount: Traversal.V[Log] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = _.in("ActionContext").count.domainMap(JsNumber(_)) - def logStatsRenderer(extraData: Set[String]): Traversal.V[Log] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { - traversal => - def addData[G]( - name: String - )(f: Traversal.V[Log] => Traversal[JsValue, G, Converter[JsValue, G]]): Traversal[JsObject, JMap[String, Any], Converter[ - JsObject, - JMap[String, Any] - ]] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { t => - val dataTraversal = f(traversal.start) - t.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.by(dataTraversal.raw)) { jmap => - t.converter(jmap) + (name -> dataTraversal.converter(jmap.get(name).asInstanceOf[G])) - } + def logStatsRenderer(extraData: Set[String])(implicit + authContext: AuthContext + ): Traversal.V[Log] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { traversal => + def addData[G]( + name: String + )(f: Traversal.V[Log] => Traversal[JsValue, G, Converter[JsValue, G]]): Traversal[JsObject, JMap[String, Any], Converter[ + JsObject, + JMap[String, Any] + ]] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { t => + val dataTraversal = f(traversal.start) + t.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.by(dataTraversal.raw)) { jmap => + t.converter(jmap) + (name -> dataTraversal.converter(jmap.get(name).asInstanceOf[G])) } + } - if (extraData.isEmpty) traversal.constant2[JsObject, JMap[String, Any]](JsObject.empty) - else { - val dataName = extraData.toSeq - dataName.foldLeft[Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]]]( - traversal.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.project(dataName.head, dataName.tail: _*))(_ => - JsObject.empty - ) - ) { - case (f, "task") => addData("task")(taskParent)(f) - case (f, "taskId") => addData("taskId")(taskParentId)(f) - case (f, "actionCount") => addData("actionCount")(actionCount)(f) - case (f, _) => f - } + if (extraData.isEmpty) traversal.constant2[JsObject, JMap[String, Any]](JsObject.empty) + else { + val dataName = extraData.toSeq + dataName.foldLeft[Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]]]( + traversal.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.project(dataName.head, dataName.tail: _*))(_ => + JsObject.empty + ) + ) { + case (f, "case") => addData("case")(caseParent)(f) + case (f, "task") => addData("task")(taskParent)(f) + case (f, "taskId") => addData("taskId")(taskParentId)(f) + case (f, "actionCount") => addData("actionCount")(actionCount)(f) + case (f, _) => f } + } } } From bf9645a3a94645ee8946dbfb979e18f68db09373 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 09:52:08 +0100 Subject: [PATCH 234/237] #1675 Fix cortex entities in describe v1 --- .../org/thp/thehive/controllers/v1/DescribeCtrl.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index c5a2322a85..33e3fae52e 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -18,6 +18,7 @@ import play.api.cache.SyncCacheApi import play.api.inject.Injector import play.api.libs.json._ import play.api.mvc.{Action, AnyContent, Results} +import org.thp.thehive.controllers.v0.{QueryCtrl => QueryCtrlV0} import scala.concurrent.duration.Duration import scala.util.{Failure, Success, Try} @@ -72,14 +73,15 @@ class DescribeCtrl @Inject() ( def describeCortexEntity( name: String, className: String, - packageName: String = "org.thp.thehive.connector.cortex.controllers.v1" + packageName: String = "org.thp.thehive.connector.cortex.controllers.v0" ): Option[EntityDescription] = Try( EntityDescription( name, injector .instanceOf(getClass.getClassLoader.loadClass(s"$packageName.$className")) - .asInstanceOf[QueryableCtrl] + .asInstanceOf[QueryCtrlV0] + .publicData .publicProperties .list .flatMap(propertyToJson(name, _)) @@ -103,8 +105,8 @@ class DescribeCtrl @Inject() ( EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))) // EntityDescription("dashboard", dashboardCtrl.publicProperties.list.flatMap(propertyToJson("dashboard", _))), // EntityDescription("page", pageCtrl.publicProperties.list.flatMap(propertyToJson("page", _))) - ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ - describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") + ) ++ describeCortexEntity("case_artifact_job", "JobCtrl") ++ + describeCortexEntity("action", "ActionCtrl") } implicit val propertyDescriptionWrites: Writes[PropertyDescription] = From fceb2e03a6c27259d0f4394ec99649562f7d175b Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 10:14:12 +0100 Subject: [PATCH 235/237] #1675 Include case in extraData/task of log --- .../app/org/thp/thehive/controllers/v1/LogRenderer.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala index f28671eeda..6b160c1635 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala @@ -21,8 +21,13 @@ trait LogRenderer { ): Traversal.V[Log] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = _.`case`.richCase.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) - def taskParent: Traversal.V[Log] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = - _.task.richTask.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) + def taskParent(implicit + authContext: AuthContext + ): Traversal.V[Log] => Traversal[JsValue, JMap[String, Any], Converter[JsValue, JMap[String, Any]]] = + _.task.project(_.by(_.richTask.fold).by(_.`case`.richCase.fold)).domainMap { + case (task, case0) => + task.headOption.fold[JsValue](JsNull)(_.toJson.as[JsObject] + ("case" -> case0.headOption.fold[JsValue](JsNull)(_.toJson))) + } def taskParentId: Traversal.V[Log] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = _.task.fold.domainMap(_.headOption.fold[JsValue](JsNull)(c => JsString(c._id.toString))) From 487c070d435bee3bb1bf988da9115d654b140c14 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 13:46:56 +0100 Subject: [PATCH 236/237] #1675 Fix alert status property --- .../thehive/controllers/v0/AlertCtrl.scala | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 855cc83b39..d0ba0e18fe 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -10,7 +10,7 @@ import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} +import org.thp.scalligraph.traversal.{Converter, IdentityConverter, IteratorOutput, Traversal} import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityId, EntityIdOrName, EntityName, InvalidFormatAttributeError, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase} @@ -411,17 +411,22 @@ class PublicAlert @Inject() ( .property("read", UMapping.boolean)(_.field.updatable) .property("follow", UMapping.boolean)(_.field.updatable) .property("status", UMapping.string)( - _.select( - _.project( + _.select { alerts => + val readAndCase = alerts.project( _.byValue(_.read) .by(_.`case`.limit(1).count) - ).domainMap { - case (false, caseCount) if caseCount == 0L => "New" - case (false, _) => "Updated" - case (true, caseCount) if caseCount == 0L => "Ignored" - case (true, _) => "Imported" - } - ).readonly + ) + readAndCase.graphMap[String, String, IdentityConverter[String]]( + jmap => + readAndCase.converter.apply(jmap) match { + case (false, caseCount) if caseCount == 0L => "New" + case (false, _) => "Updated" + case (true, caseCount) if caseCount == 0L => "Ignored" + case (true, _) => "Imported" + }, + Converter.identity[String] + ) + }.readonly ) .property("summary", UMapping.string.optional)(_.field.updatable) .property("user", UMapping.string)(_.field.updatable) From 22cbf236bf89996ec5970f7086b32170000c1cb1 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 20 Nov 2020 15:19:03 +0100 Subject: [PATCH 237/237] Prepare release --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- build.sbt | 2 +- frontend/bower.json | 2 +- frontend/package.json | 2 +- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a42bac96..ac6646e61d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Change Log +## [4.0.2](https://github.com/TheHive-Project/TheHive/milestone/64) (2020-11-20) + +**Implemented enhancements:** + +- [Feature Request] Add a dedicated permission to give access to TheHiveFS [\#1655](https://github.com/TheHive-Project/TheHive/issues/1655) +- [Feature Request] Normalize editable input fields [\#1669](https://github.com/TheHive-Project/TheHive/issues/1669) + +**Fixed bugs:** + +- [Bug] Unable to list Cases [\#1598](https://github.com/TheHive-Project/TheHive/issues/1598) +- [Bug] Alert to case merge is broken in v4.0.1 [\#1648](https://github.com/TheHive-Project/TheHive/issues/1648) +- [Bug] Attachment.* filters are broken under observable search in v4.0.1 [\#1649](https://github.com/TheHive-Project/TheHive/issues/1649) +- [Bug] Result of observable update API v0 is empty [\#1652](https://github.com/TheHive-Project/TheHive/issues/1652) +- [Bug] Display issue of custom fields [\#1653](https://github.com/TheHive-Project/TheHive/issues/1653) +- [Bug] Persistent AuditSrv:undefined error on 4.0.1 [\#1656](https://github.com/TheHive-Project/TheHive/issues/1656) +- [Bug] Issues with case attachments section [\#1657](https://github.com/TheHive-Project/TheHive/issues/1657) +- [Bug] API method broken: /api/case/artifact/_search in 4.0.1 [\#1659](https://github.com/TheHive-Project/TheHive/issues/1659) +- [Bug] API method broken: /api/case/task/log/_search in 4.0.1 [\#1660](https://github.com/TheHive-Project/TheHive/issues/1660) +- [Bug] Unable to define ES index on migration [\#1661](https://github.com/TheHive-Project/TheHive/issues/1661) +- [Bug] Dashboard max aggregation on custom-integer field does not work [\#1662](https://github.com/TheHive-Project/TheHive/issues/1662) +- [Bug] Missing the fix for errorMessage [\#1666](https://github.com/TheHive-Project/TheHive/issues/1666) +- [Bug] Fix alert details dialog [\#1672](https://github.com/TheHive-Project/TheHive/issues/1672) +- [Bug] error 500 with adding an empty file in Observables of an Alert [\#1673](https://github.com/TheHive-Project/TheHive/issues/1673) +- [Bug] Fix migration of audit logs [\#1676](https://github.com/TheHive-Project/TheHive/issues/1676) + ## [4.0.1](https://github.com/TheHive-Project/TheHive/milestone/60) (2020-11-13) **Implemented enhancements:** @@ -24,6 +49,7 @@ **Fixed bugs:** +- [Bug] MISP->THEHIVE4 'ExportOnly' and 'Exceptions' ignored in application.conf file [\#1482](https://github.com/TheHive-Project/TheHive/issues/1482) - [Bug] Mobile-responsive Hamburger not visible [\#1290](https://github.com/TheHive-Project/TheHive/issues/1290) - [Bug] Unable to start TheHive after migration [\#1450](https://github.com/TheHive-Project/TheHive/issues/1450) - [Bug] Expired session should show a dialog or login page on pageload [\#1456](https://github.com/TheHive-Project/TheHive/issues/1456) @@ -34,7 +60,6 @@ - [Bug] Dashboard shared/private [\#1474](https://github.com/TheHive-Project/TheHive/issues/1474) - [Bug]Migration tool date/number/duration params don't work [\#1478](https://github.com/TheHive-Project/TheHive/issues/1478) - [Bug] AuditSrv: undefined on non-case page(s), thehive4-4.0.0-1, Ubuntu [\#1479](https://github.com/TheHive-Project/TheHive/issues/1479) -- [Bug] MISP->THEHIVE4 'ExportOnly' and 'Exceptions' ignored in application.conf file [\#1482](https://github.com/TheHive-Project/TheHive/issues/1482) - [Bug] Unable to enumerate tasks via API [\#1483](https://github.com/TheHive-Project/TheHive/issues/1483) - [Bug] Case close notification displays "#undefined" instead of case number [\#1488](https://github.com/TheHive-Project/TheHive/issues/1488) - [Bug] Task under "Waiting tasks" and "My tasks" do not display the case number [\#1489](https://github.com/TheHive-Project/TheHive/issues/1489) diff --git a/build.sbt b/build.sbt index c93640d396..e4fa911b8b 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.0.1-1" +val thehiveVersion = "4.0.2-1" val scala212 = "2.12.12" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/frontend/bower.json b/frontend/bower.json index 4a58c3aed4..26f486b888 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.1-1", + "version": "4.0.2-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index e13f63a44a..d1bca2f2ce 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.1-1", + "version": "4.0.2-1", "license": "AGPL-3.0", "repository": { "type": "git",