diff --git a/.drone.yml b/.drone.yml index fce5ea1d8d..8ab55edf43 100644 --- a/.drone.yml +++ b/.drone.yml @@ -30,20 +30,6 @@ steps: commands: - sbt -Duser.home=$PWD test:compile - # Save external libraries in cache - - name: save-cache - image: drillster/drone-volume-cache - settings: - rebuild: true - backend: "filesystem" - mount: - - .sbt - - .ivy2 - - .cache - - frontend/node_modules - - frontend/bower_components - volumes: [{name: cache, path: /cache}] - # Build packages - name: build-packages image: thehiveproject/drone-scala-node @@ -71,6 +57,20 @@ steps: when: event: [tag] + # Save external libraries in cache + - name: save-cache + image: drillster/drone-volume-cache + settings: + rebuild: true + backend: "filesystem" + mount: + - .sbt + - .ivy2 + - .cache + - frontend/node_modules + - frontend/bower_components + volumes: [{name: cache, path: /cache}] + # Send packages using scp - name: send packages image: appleboy/drone-scp diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d6aad35ff..44b9983ee8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Change Log +## [4.1.6](https://github.com/TheHive-Project/TheHive/milestone/75) (2021-06-14) + +**Implemented enhancements:** + +- [Feature Request] Add API to repair database [\#2081](https://github.com/TheHive-Project/TheHive/issues/2081) + +**Fixed bugs:** + +- [Bug] Editing case template tasks incorrectly removes tasks [\#1926](https://github.com/TheHive-Project/TheHive/issues/1926) +- [Bug] When creating cases from alerts via API, the same case number gets assigned to multiple distinct cases [\#1970](https://github.com/TheHive-Project/TheHive/issues/1970) +- [Bug] src:MISP-ORG missing on MISP alerts [\#2058](https://github.com/TheHive-Project/TheHive/issues/2058) +- [Bug] Analyzer reports dissapear in 4.1.5 (observable already exists error) [\#2059](https://github.com/TheHive-Project/TheHive/issues/2059) +- [Bug] CaseNumber Conflict [\#2061](https://github.com/TheHive-Project/TheHive/issues/2061) +- [Bug] Alert tags glitch after previewing alert [\#2062](https://github.com/TheHive-Project/TheHive/issues/2062) +- [Bug] Case Template content mixed across organisations [\#2068](https://github.com/TheHive-Project/TheHive/issues/2068) +- [Bug] /api/v1//key returns 401/403 if user hasn't key [\#2069](https://github.com/TheHive-Project/TheHive/issues/2069) +- [Bug] When API call returns failure, actual response depends on authentication methods [\#2070](https://github.com/TheHive-Project/TheHive/issues/2070) +- [Bug] Deleting observables doesn't produce audit log [\#2076](https://github.com/TheHive-Project/TheHive/issues/2076) + ## [4.1.5](https://github.com/TheHive-Project/TheHive/milestone/74) (2021-06-03) **Implemented enhancements:** diff --git a/ScalliGraph b/ScalliGraph index 7269094914..f1a647d4dd 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 72690949146e846a51a405c9272699ec8a7eb86f +Subproject commit f1a647d4dd61f538d0d390f30d357e37d6a3aebc diff --git a/build.sbt b/build.sbt index 22eaccbf18..8db30d29ea 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.1.5-1" +val thehiveVersion = "4.1.6-1" val scala212 = "2.12.13" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala index 554f94221e..3f5955c062 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala @@ -1,7 +1,7 @@ package org.thp.thehive.connector.cortex.controllers.v0 import com.google.inject.name.Named -import org.thp.scalligraph.EntityIdOrName +import org.thp.scalligraph.{EntityIdOrName, NotFoundError} import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser} import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ @@ -37,6 +37,9 @@ class AnalyzerTemplateCtrl @Inject() ( analyzerTemplateSrv .getOrFail(EntityIdOrName(id)) .map(report => Results.Ok(report.content)) + .recover { + case _: NotFoundError => Results.NotFound(s"AnalyzerTemplate $id not found") + } } def importTemplates: Action[AnyContent] = diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala index 322b750e4a..34d5df72bf 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala @@ -227,11 +227,11 @@ class JobSrv @Inject() ( Future .fromTry { db.tryTransaction { implicit graph => - for { - origObs <- get(job).observable.getOrFail("Observable") - obs <- observableSrv.create(artifact.toObservable(job._id, origObs.organisationIds), artifact.data.get) - _ <- addObservable(job, obs.observable) - } yield () + get(job).observable.getOrFail("Observable").map { origObs => + observableSrv + .create(artifact.toObservable(job._id, origObs.organisationIds), artifact.data.get) + .foreach(obs => addObservable(job, obs.observable)) + } } } } @@ -336,7 +336,7 @@ object JobOps { _.reportObservables .project( _.by(_.richObservable) - .by(_.similar.filter(_.`case`.observables.out[ObservableJob].where(P.eq[String](thisJob.name)))._id.fold) + .by(_.similar.where(_.`case`.observables.out[ObservableJob].v[Job].isStep(P.eq[String](thisJob.name)))._id.fold) ) .fold ) diff --git a/frontend/app/scripts/components/organisation/OrgCaseTemplateListCmp.js b/frontend/app/scripts/components/organisation/OrgCaseTemplateListCmp.js index d0cc9b90be..23cd4fd79d 100644 --- a/frontend/app/scripts/components/organisation/OrgCaseTemplateListCmp.js +++ b/frontend/app/scripts/components/organisation/OrgCaseTemplateListCmp.js @@ -1,9 +1,9 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveComponents') .component('orgCaseTemplateList', { - controller: function($uibModal, $scope, $q, CaseTemplateSrv, PaginatedQuerySrv, FilteringSrv, UserSrv, NotificationSrv, ModalUtilsSrv) { + controller: function ($uibModal, $scope, $q, CaseTemplateSrv, PaginatedQuerySrv, FilteringSrv, UserSrv, NotificationSrv, ModalUtilsSrv) { var self = this; self.task = ''; @@ -13,7 +13,7 @@ self.getUserInfo = UserSrv.getCache; - this.$onInit = function() { + this.$onInit = function () { self.filtering = new FilteringSrv('caseTemplate', 'caseTemplate.list', { version: 'v1', defaults: { @@ -26,7 +26,7 @@ }); self.filtering.initContext(self.organisation.name) - .then(function() { + .then(function () { self.load(); $scope.$watch('$vm.list.pageSize', function (newValue) { @@ -35,7 +35,7 @@ }); }; - this.load = function() { + this.load = function () { self.list = new PaginatedQuerySrv({ name: 'organisation-case-templates', @@ -46,15 +46,15 @@ pageSize: self.filtering.context.pageSize, filter: this.filtering.buildQuery(), operations: [{ - '_name': 'getOrganisation', - 'idOrName': self.organisation.name - }, - { - '_name': 'caseTemplates' - } + '_name': 'getOrganisation', + 'idOrName': self.organisation.name + }, + { + '_name': 'caseTemplates' + } ], - onFailure: function(err) { - if(err && err.status === 400) { + onFailure: function (err) { + if (err && err.status === 400) { self.filtering.resetContext(); self.load(); } @@ -90,28 +90,28 @@ this.search(); }; - this.filterBy = function(field, value) { + this.filterBy = function (field, value) { self.filtering.clearFilters() - .then(function(){ + .then(function () { self.addFilterValue(field, value); }); }; - this.sortBy = function(sort) { + this.sortBy = function (sort) { self.list.sort = sort; self.list.update(); self.filtering.setSort(sort); }; - this.sortByField = function(field) { + this.sortByField = function (field) { var context = this.filtering.context; var currentSort = Array.isArray(context.sort) ? context.sort[0] : context.sort; var sort = null; - if(currentSort.substr(1) !== field) { + if (currentSort.substr(1) !== field) { sort = ['+' + field]; } else { - sort = [(currentSort === '+' + field) ? '-'+field : '+'+field]; + sort = [(currentSort === '+' + field) ? '-' + field : '+' + field]; } self.list.sort = sort; @@ -119,7 +119,7 @@ self.filtering.setSort(sort); }; - this.newTemplate = function() { + this.newTemplate = function () { self.showTemplate({ name: '', titlePrefix: '', @@ -133,12 +133,12 @@ }); }; - this.showTemplate = function(template) { + this.showTemplate = function (template) { var promise = template._id ? CaseTemplateSrv.get(template._id) : $q.resolve(template); promise - .then(function(response) { + .then(function (response) { var modalInstance = $uibModal.open({ animation: true, keyboard: false, @@ -148,10 +148,16 @@ controllerAs: '$vm', size: 'max', resolve: { - template: function() { - return response; + template: function () { + var tmpl = angular.copy(response); + + if (tmpl.tasks && tmpl.tasks.length > 0) { + tmpl.tasks = _.sortBy(tmpl.tasks, 'order'); + } + + return tmpl; }, - fields: function() { + fields: function () { return self.fields; } } @@ -159,32 +165,32 @@ return modalInstance.result; }) - .then(function() { + .then(function () { self.load(); }) - .catch(function(err) { - if(err && !_.isString(err)) { + .catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Case Template Admin', err.data, err.status); } }) } - self.createTemplate = function(template) { + self.createTemplate = function (template) { return CaseTemplateSrv.create(template).then( - function(/*response*/) { + function (/*response*/) { self.load(); $scope.$emit('templates:refresh'); NotificationSrv.log('The template [' + template.name + '] has been successfully created', 'success'); }, - function(response) { + function (response) { NotificationSrv.error('TemplateCtrl', response.data, response.status); } ); }; - self.importTemplate = function() { + self.importTemplate = function () { var modalInstance = $uibModal.open({ animation: true, templateUrl: 'views/components/org/case-template/import.html', @@ -194,10 +200,10 @@ }); modalInstance.result - .then(function(template) { + .then(function (template) { return self.createTemplate(template); }) - .catch(function(err) { + .catch(function (err) { if (err && err.status) { NotificationSrv.error('TemplateCtrl', err.data, err.status); } @@ -221,7 +227,7 @@ self.exportTemplate = function (template) { CaseTemplateSrv.get(template._id) - .then(function(response) { + .then(function (response) { var fileName = 'Case-Template__' + response.name.replace(/\s/gi, '_') + '.json'; // Create a blob of the data diff --git a/frontend/app/scripts/controllers/alert/AlertListCtrl.js b/frontend/app/scripts/controllers/alert/AlertListCtrl.js index 9f89ba4b1a..39480f529b 100755 --- a/frontend/app/scripts/controllers/alert/AlertListCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertListCtrl.js @@ -192,7 +192,9 @@ controllerAs: 'dialog', size: 'max', resolve: { - event: event, + event: function () { + return angular.copy(event); + }, templates: function () { return CaseTemplateSrv.list(); }, diff --git a/frontend/app/views/components/org/case-template/tasks.html b/frontend/app/views/components/org/case-template/tasks.html index 4b37095d32..0f77fd5540 100644 --- a/frontend/app/views/components/org/case-template/tasks.html +++ b/frontend/app/views/components/org/case-template/tasks.html @@ -11,7 +11,8 @@ No tasks have been specified. Add a task
-
+
@@ -22,14 +23,16 @@ [{{t.group || 'default'}}] {{t.title}} - (Assigned to ) + (Assigned to + + )  Edit - -  Delete + +  Delete
diff --git a/frontend/app/views/directives/flow/observable.html b/frontend/app/views/directives/flow/observable.html index 5b1b6e5889..330a2d6153 100644 --- a/frontend/app/views/directives/flow/observable.html +++ b/frontend/app/views/directives/flow/observable.html @@ -1,16 +1,23 @@
- {{base.object.dataType}}: + {{base.object.dataType || base.details.dataType}}: + + + +
- {{summary.case_artifact.Creation}} other observables have also been {{base.operation === 'Creation' ? 'added' : 'updated'}} - See all + {{summary.case_artifact.Creation}} other observables have also been {{base.operation === 'Creation' ? + 'added' : 'updated'}} + See + all
@@ -39,6 +46,10 @@
+
+ {{k}}: {{v.name}} +
+
{{k}}: {{v}} diff --git a/frontend/app/views/partials/alert/event.dialog.html b/frontend/app/views/partials/alert/event.dialog.html index 27512e2e07..9ab978f49f 100644 --- a/frontend/app/views/partials/alert/event.dialog.html +++ b/frontend/app/views/partials/alert/event.dialog.html @@ -54,12 +54,12 @@

Tags
-
+
-
- +
+
@@ -68,11 +68,11 @@

Description

-
+
-
+
diff --git a/frontend/bower.json b/frontend/bower.json index 3725157055..31b21b6795 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.4-1", + "version": "4.1.6-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index 652c7464c0..1f6fa7f507 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.4-1", + "version": "4.1.6-1", "license": "AGPL-3.0", "repository": { "type": "git", diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala index 32246c5e54..2411bb7b91 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala @@ -72,7 +72,7 @@ class MispImportSrv @Inject() ( read = false, follow = true, organisationId = organisationId, - tags = event.tags.map(_.name), + tags = s"src:${event.orgc}" +: event.tags.map(_.name), caseId = EntityId.empty ) } diff --git a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala index b610f6c63e..8321f93d88 100644 --- a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala +++ b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala @@ -92,7 +92,7 @@ class MispImportSrvTest(implicit ec: ExecutionContext) extends PlaySpecification pap = 2, read = false, follow = true, - tags = Seq("TH-test", "TH-test-2"), + tags = Seq("src:ORGNAME", "TH-test", "TH-test-2"), organisationId = alert.organisationId, caseId = EntityId.empty ) diff --git a/thehive/app/org/thp/thehive/TheHiveModule.scala b/thehive/app/org/thp/thehive/TheHiveModule.scala index e9922041eb..6667d840b2 100644 --- a/thehive/app/org/thp/thehive/TheHiveModule.scala +++ b/thehive/app/org/thp/thehive/TheHiveModule.scala @@ -1,6 +1,7 @@ package org.thp.thehive import akka.actor.ActorRef +import akka.actor.typed.{ActorRef => TypedActorRef} import com.google.inject.AbstractModule import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder} import org.thp.scalligraph.SingleInstance @@ -106,6 +107,7 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps] integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps] bind[ActorRef].annotatedWithName("integrity-check-actor").toProvider[IntegrityCheckActorProvider] + bind[TypedActorRef[CaseNumberActor.Request]].annotatedWithName("case-number-actor").toProvider[CaseNumberActorProvider] bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider] diff --git a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala index 21637a1986..f91a146d3e 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala @@ -6,7 +6,7 @@ import akka.util.Timeout import ch.qos.logback.classic.{Level, LoggerContext} import org.slf4j.LoggerFactory import org.thp.scalligraph.controllers.Entrypoint -import org.thp.scalligraph.models.Database +import org.thp.scalligraph.models._ import org.thp.scalligraph.services.GenIntegrityCheckOps import org.thp.thehive.models.Permissions import org.thp.thehive.services.{CheckState, CheckStats, GetCheckStats, GlobalCheckRequest} @@ -18,7 +18,7 @@ import javax.inject.{Inject, Named, Singleton} import scala.collection.immutable import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} -import scala.util.Success +import scala.util.{Failure, Success} @Singleton class AdminCtrl @Inject() ( @@ -26,6 +26,7 @@ class AdminCtrl @Inject() ( @Named("integrity-check-actor") integrityCheckActor: ActorRef, integrityCheckOps: immutable.Set[GenIntegrityCheckOps], db: Database, + schemas: immutable.Set[UpdatableSchema], implicit val ec: ExecutionContext ) { @@ -113,6 +114,7 @@ class AdminCtrl @Inject() ( "Job", "Attachment" ) + def indexStatus: Action[AnyContent] = entrypoint("Get index status") .authPermittedRoTransaction(db, Permissions.managePlatform) { _ => graph => @@ -122,7 +124,7 @@ class AdminCtrl @Inject() ( catch { case error: Throwable => logger.error("Index fetch error", error) - 0 + 0L } Json.obj("name" -> label, "count" -> count) @@ -137,4 +139,82 @@ class AdminCtrl @Inject() ( Future(db.reindexData(label)) Success(Results.NoContent) } + + private val rangeRegex = "(\\d+)-(\\d+)".r + private def getOperations(schemaName: String, select: Option[String], filter: Option[String]): Seq[(Operation, Int)] = { + val ranges = select.fold(Seq(0 until Int.MaxValue))(_.split(',').toSeq.map { + case rangeRegex(from, to) => from.toInt to to.toInt + case number => number.toInt to number.toInt + }) + + val filters = filter.fold(Seq("all"))(_.split(',')) + + schemas + .filter(_.name == schemaName) + .toSeq + .flatMap { schema => + schema + .operations + .operations + .zipWithIndex + .filter { + case (_, i) => ranges.exists(_.contains(i)) + } + .filter { + case (_: AddVertexModel, _) => + filters.contains("AddVertexModel") || filters + .contains("schema") || (filters.contains("all") && !filters.contains("!schema") && !filters.contains("!AddVertexModel")) + case (_: AddEdgeModel, _) => + filter.contains("AddEdgeModel") || filter + .contains("schema") || (filters.contains("all") && !filter.contains("!schema") && !filter.contains("!AddEdgeModel")) + case (_: AddProperty, _) => + filter.contains("AddProperty") || filter + .contains("schema") || (filters.contains("all") && !filter.contains("!schema") && !filter.contains("!AddProperty")) + case (_: RemoveProperty, _) => + filter.contains("RemoveProperty") || filter + .contains("schema") || (filters.contains("all") && !filter.contains("!schema") && !filter.contains("!RemoveProperty")) + case (_: UpdateGraph, _) => + filter.contains("UpdateGraph") || filter + .contains("data") || (filters.contains("all") && !filter.contains("!data") && !filter.contains("!UpdateGraph")) + case (_: AddIndex, _) => + filter.contains("AddIndex") || filter + .contains("index") || (filters.contains("all") && !filter.contains("!index") && !filter.contains("!AddIndex")) + case (RebuildIndexes, _) => + filter.contains("RebuildIndexes") || filter + .contains("index") || (filters.contains("all") && !filter.contains("!index") && !filter.contains("!RebuildIndexes")) + case (NoOperation, _) => false + case (_: RemoveIndex, _) => + filter.contains("RemoveIndex") || filter + .contains("index") || (filters.contains("all") && !filter.contains("!index") && !filter.contains("!RemoveIndex")) + case (_: DBOperation[_], _) => + filter.contains("DBOperation") || filter + .contains("data") || (filters.contains("all") && !filter.contains("!data") && !filter.contains("!DBOperation")) + } + } + } + + def schemaRepair(schemaName: String, select: Option[String], filter: Option[String]): Action[AnyContent] = + entrypoint("Repair schema") + .authPermitted(Permissions.managePlatform) { _ => + val result = getOperations(schemaName, select, filter) + .map { + case (operation, index) => + logger.info(s"Repair schema: $index=${operation.info}") + operation.execute(db, logger.info(_)) match { + case _: Success[_] => s"${operation.info}: Success" + case Failure(error) => s"${operation.info}: Failure $error" + } + } + + Success(Results.Ok(Json.toJson(result))) + } + + def schemaInfo(schemaName: String, select: Option[String], filter: Option[String]): Action[AnyContent] = + entrypoint("Schema info") + .authPermitted(Permissions.managePlatform) { _ => + val output = getOperations(schemaName, select, filter).map { + case (o, i) => s"$i=${o.info}" + } + Success(Results.Ok(Json.toJson(output))) + } } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index e5b3ca9fa9..622167041a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -43,11 +43,13 @@ class Router @Inject() ( case GET(p"/status") => statusCtrl.get // GET /health controllers.StatusCtrl.health - case GET(p"/admin/check/stats") => adminCtrl.checkStats - case GET(p"/admin/check/$name/trigger") => adminCtrl.triggerCheck(name) - case GET(p"/admin/index/status") => adminCtrl.indexStatus - case GET(p"/admin/index/$name/reindex") => adminCtrl.reindex(name) - case GET(p"/admin/log/set/$packageName/$level") => adminCtrl.setLogLevel(packageName, level) + case GET(p"/admin/check/stats") => adminCtrl.checkStats + case GET(p"/admin/check/$name/trigger") => adminCtrl.triggerCheck(name) + case GET(p"/admin/index/status") => adminCtrl.indexStatus + case GET(p"/admin/index/$name/reindex") => adminCtrl.reindex(name) + case GET(p"/admin/log/set/$packageName/$level") => adminCtrl.setLogLevel(packageName, level) + case POST(p"/admin/schema/repair/$schemaName" ? q_o"select=$select" ? q_o"filter=$filter") => adminCtrl.schemaRepair(schemaName, select, filter) + case POST(p"/admin/schema/info/$schemaName" ? q_o"select=$select" ? q_o"filter=$filter") => adminCtrl.schemaInfo(schemaName, select, filter) // GET /logout controllers.AuthenticationCtrl.logout() case GET(p"/logout") => authenticationCtrl.logout diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 32ae88ec67..3b12ab85b1 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -173,7 +173,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { traversal.removeProperty("colour").iterate() Success(()) } - .removeProperty("Tag", "colour", usedOnlyByThisModel = true) + .removeProperty[Int]("Tag", "colour", usedOnlyByThisModel = true) .addProperty[String]("Tag", "colour") .updateGraph("Add property colour for Tags ", "Tag") { traversal => traversal.raw.property("colour", "#000000").iterate() @@ -393,7 +393,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { traversal.removeProperty("deleted") Success(()) } - .removeProperty(model = "Log", propertyName = "deleted", usedOnlyByThisModel = true) + .removeProperty[Boolean](model = "Log", propertyName = "deleted", usedOnlyByThisModel = true) .updateGraph("Make shared dashboard writable", "Dashboard") { traversal => traversal.outE("OrganisationDashboard").raw.property("writable", true).iterate() Success(()) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 6b9cbb4c04..8d91fd568f 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -243,7 +243,7 @@ class AlertSrv @Inject() ( caseTemplate <- alert .caseTemplate - .map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).richCaseTemplate.getOrFail("CaseTemplate")) + .map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.richCaseTemplate.getOrFail("CaseTemplate")) .flip customField = alert.customFields.map(f => InputCustomFieldValue(f.name, f.value, f.order)) case0 = Case( diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index 3508249bd7..259c751da9 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -19,6 +19,7 @@ import org.thp.thehive.services.CaseTemplateOps._ import org.thp.thehive.services.DashboardOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ import org.thp.thehive.services.notification.AuditNotificationMessage import play.api.libs.json.{JsObject, JsValue, Json} @@ -315,7 +316,7 @@ class AuditSrv @Inject() ( object AuditOps { implicit class VertexDefs(traversal: Traversal[Vertex, Vertex, IdentityConverter[Vertex]]) { - def share: Traversal.V[Share] = traversal.coalesceIdent(_.in[ShareObservable], _.in[ShareTask], _.in[ShareCase]).v[Share] + def share: Traversal.V[Share] = traversal.coalesceIdent(_.in[ShareObservable], _.in[ShareTask], _.in[ShareCase], _.identity).v[Share] } implicit class AuditOpsDefs(traversal: Traversal.V[Audit]) { @@ -401,6 +402,7 @@ object AuditOps { .option("Organisation", _.v[Organisation]._id) .option("CaseTemplate", _.v[CaseTemplate].organisation._id) .option("Dashboard", _.v[Dashboard].organisation._id) + .option("Share", _.v[Share].organisation._id) ) .domainMap(EntityId.apply) @@ -412,6 +414,7 @@ object AuditOps { .option("Case", _.v[Case]._id) .option("Observable", _.v[Observable].value(_.relatedId).widen[AnyRef]) .option("Task", _.v[Task].value(_.relatedId).widen[AnyRef]) + .option("Share", _.v[Share].`case`._id) ) .domainMap(EntityId.apply) @@ -426,6 +429,7 @@ object AuditOps { .option("Organisation", _.v[Organisation].current.widen[Any]) .option("CaseTemplate", _.v[CaseTemplate].visible.widen[Any]) .option("Dashboard", _.v[Dashboard].visible.widen[Any]) + .option("Share", _.v[Share].organisation.current.widen[Any]) ) ) diff --git a/thehive/app/org/thp/thehive/services/CaseNumber.scala b/thehive/app/org/thp/thehive/services/CaseNumber.scala new file mode 100644 index 0000000000..7366151eae --- /dev/null +++ b/thehive/app/org/thp/thehive/services/CaseNumber.scala @@ -0,0 +1,76 @@ +package org.thp.thehive.services + +import akka.actor.typed.scaladsl.Behaviors +import akka.actor.typed.scaladsl.adapter.ClassicActorSystemOps +import akka.actor.typed.{ActorRefResolver, Behavior, ActorRef => TypedActorRef} +import akka.actor.{ActorSystem, ExtendedActorSystem} +import akka.cluster.typed.{ClusterSingleton, SingletonActor} +import akka.serialization.Serializer +import org.thp.scalligraph.models.Database +import org.thp.scalligraph.traversal.TraversalOps._ +import org.thp.thehive.GuiceAkkaExtension +import org.thp.thehive.services.CaseOps._ + +import java.io.NotSerializableException +import javax.inject.{Inject, Provider, Singleton} + +object CaseNumberActor { + sealed trait Message + sealed trait Request extends Message + sealed trait Response extends Message + case class GetNextNumber(replyTo: TypedActorRef[Response]) extends Request + case class NextNumber(number: Int) extends Response + + val behavior: Behavior[Request] = Behaviors.setup[Request] { context => + val injector = GuiceAkkaExtension(context.system).injector + val db = injector.getInstance(classOf[Database]) + val caseSrv = injector.getInstance(classOf[CaseSrv]) + db.roTransaction { implicit graph => + caseNumberProvider(caseSrv.startTraversal.getLast.headOption.fold(0)(_.number) + 1) + } + } + + def caseNumberProvider(nextNumber: Int): Behavior[Request] = + Behaviors.receiveMessage { + case GetNextNumber(replyTo) => + replyTo ! NextNumber(nextNumber) + caseNumberProvider(nextNumber + 1) + } +} + +@Singleton +class CaseNumberActorProvider @Inject() (system: ActorSystem) extends Provider[TypedActorRef[CaseNumberActor.Request]] { + override lazy val get: TypedActorRef[CaseNumberActor.Request] = + ClusterSingleton(system.toTyped) + .init(SingletonActor(CaseNumberActor.behavior, "CaseNumberLeader")) +} + +class CaseNumberSerializer(system: ExtendedActorSystem) extends Serializer { + import CaseNumberActor._ + + private val actorRefResolver = ActorRefResolver(system.toTyped) + + override def identifier: Int = 9739323 + + override def toBinary(o: AnyRef): Array[Byte] = + o match { + case GetNextNumber(replyTo) => 0.toByte +: actorRefResolver.toSerializationFormat(replyTo).getBytes + case NextNumber(number) => + Array(1.toByte, ((number >> 24) % 0xff).toByte, ((number >> 16) % 0xff).toByte, ((number >> 8) % 0xff).toByte, (number % 0xff).toByte) + case _ => throw new NotSerializableException + } + + override def includeManifest: Boolean = false + + override def fromBinary(bytes: Array[Byte], manifest: Option[Class[_]]): AnyRef = + bytes(0) match { + case 0 => GetNextNumber(actorRefResolver.resolveActorRef(new String(bytes.tail))) + case 1 => + NextNumber( + (bytes(2) << 24) + + (bytes(3) << 16) + + (bytes(4) << 8) + + bytes(5) + ) + } +} diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 17ecd57d73..a57cef3a73 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -1,6 +1,9 @@ package org.thp.thehive.services import akka.actor.ActorRef +import akka.actor.typed.scaladsl.AskPattern._ +import akka.actor.typed.{Scheduler, ActorRef => TypedActorRef} +import akka.util.Timeout import org.apache.tinkerpop.gremlin.process.traversal.{Order, P} import org.apache.tinkerpop.gremlin.structure.Vertex import org.thp.scalligraph.auth.{AuthContext, Permission} @@ -30,6 +33,8 @@ import play.api.libs.json.{JsNull, JsObject, JsValue, Json} import java.lang.{Long => JLong} import java.util.{Date, List => JList, Map => JMap} import javax.inject.{Inject, Named, Provider, Singleton} +import scala.concurrent.duration.DurationInt +import scala.concurrent.{Await, ExecutionContext, Future} import scala.util.{Failure, Success, Try} @Singleton @@ -48,7 +53,10 @@ class CaseSrv @Inject() ( userSrv: UserSrv, alertSrvProvider: Provider[AlertSrv], @Named("integrity-check-actor") integrityCheckActor: ActorRef, - cache: SyncCacheApi + @Named("case-number-actor") caseNumberActor: TypedActorRef[CaseNumberActor.Request], + cache: SyncCacheApi, + implicit val ec: ExecutionContext, + implicit val scheduler: Scheduler ) extends VertexSrv[Case] { lazy val alertSrv: AlertSrv = alertSrvProvider.get @@ -124,7 +132,15 @@ class CaseSrv @Inject() ( .map { case (InputCustomFieldValue(name, value, _), i) => InputCustomFieldValue(name, value, Some(i)) } } - def nextCaseNumber(implicit graph: Graph): Int = startTraversal.getLast.headOption.fold(0)(_.number) + 1 + def nextCaseNumberAsync: Future[Int] = { + implicit val timeout: Timeout = Timeout(1.minute) + caseNumberActor.ask[CaseNumberActor.Response](replyTo => CaseNumberActor.GetNextNumber(replyTo)).map { + case CaseNumberActor.NextNumber(caseNumber) => caseNumber + } + } + + def nextCaseNumber: Int = + Await.result(nextCaseNumberAsync, 1.minute) override def exists(e: Case)(implicit graph: Graph): Boolean = startTraversal.getByNumber(e.number).exists diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 6797a1dea8..4365f75750 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -17,7 +17,7 @@ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.{JsObject, JsString, Json} import java.util.{Map => JMap} import javax.inject.{Inject, Provider, Singleton} @@ -136,28 +136,42 @@ class ObservableSrv @Inject() ( _ <- auditSrv.observable.update(observable, Json.obj("tags" -> tags)) } yield (tagsToAdd, tagsToRemove) - override def delete(observable: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = + override def delete(observable: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + def observableDetail(attachment: Option[Attachment with Entity]): JsObject = + JsObject( + "dataType" -> JsString(observable.dataType) :: + attachment.map { a => + "attachment" -> Json.obj( + "name" -> a.name, + "id" -> a.attachmentId, + "size" -> a.size, + "contentType" -> a.contentType, + "hashes" -> a.hashes.map(_.toString) + ) + }.toList ::: observable.data.map(d => "data" -> JsString(d)).toList + ) + get(observable).alert.headOption match { case None => get(observable) - .share + .project(_.by(_.share).by(_.attachments.option)) .toIterator .toTry { - case share if share.owner => + case (share, attachment) if share.owner => get(observable) .shares .toIterator .toTry { share => auditSrv .observable - .delete(observable, share) + .delete(observable, share, Some(observableDetail(attachment))) } .map(_ => get(observable).remove()) - case share => + case (share, attachment) => for { organisation <- organisationSrv.current.getOrFail("Organisation") _ <- shareSrv.unshareObservable(observable, organisation) - _ <- auditSrv.observable.delete(observable, share) + _ <- auditSrv.observable.delete(observable, share, Some(observableDetail(attachment))) } yield () } .map(_ => ()) @@ -165,6 +179,7 @@ class ObservableSrv @Inject() ( get(observable).remove() auditSrv.observableInAlert.delete(observable, alert) } + } override def update( traversal: Traversal.V[Observable], diff --git a/thehive/conf/play/reference-overrides.conf b/thehive/conf/play/reference-overrides.conf index 1c0cb430ae..048f95b8d0 100644 --- a/thehive/conf/play/reference-overrides.conf +++ b/thehive/conf/play/reference-overrides.conf @@ -29,6 +29,7 @@ akka.actor { //thehive-schema-updater = "org.thp.thehive.models.SchemaUpdaterSerializer" flow = "org.thp.thehive.services.FlowSerializer" integrity = "org.thp.thehive.services.IntegrityCheckSerializer" + caseNumber = "org.thp.thehive.services.CaseNumberSerializer" } serialization-bindings { @@ -37,5 +38,6 @@ akka.actor { //"org.thp.thehive.models.SchemaUpdaterMessage" = thehive-schema-updater "org.thp.thehive.services.FlowMessage" = flow "org.thp.thehive.services.IntegrityCheckMessage" = integrity + "org.thp.thehive.services.CaseNumberActor$Message" = caseNumber } }