diff --git a/migration/src/main/scala/org/thp/thehive/migration/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/Input.scala index 0470160a6f..1347390d3c 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Input.scala @@ -120,6 +120,8 @@ trait Input { def listJobObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed] def countAction(filter: Filter): Future[Long] def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed] - def countAudit(filter: Filter): Future[Long] + def countAudits(filter: Filter): Future[Long] def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed] + def countDashboards(filter: Filter): Future[Long] + def listDashboards(filter: Filter): Source[Try[InputDashboard], NotUsed] } diff --git a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala index 6880087f0d..ae174496d6 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala @@ -454,7 +454,7 @@ trait MigrationOps { input.countJobs(filter).foreach(count => migrationStats.setTotal("Job", count)) input.countJobObservables(filter).foreach(count => migrationStats.setTotal("Job/Observable", count)) input.countAction(filter).foreach(count => migrationStats.setTotal("Action", count)) - input.countAudit(filter).foreach(count => migrationStats.setTotal("Audit", count)) + input.countAudits(filter).foreach(count => migrationStats.setTotal("Audit", count)) migrationStats.stage = "Prepare database" output.startMigration().flatMap { _ => @@ -474,6 +474,8 @@ trait MigrationOps { migrate(output)("ObservableType", input.listObservableTypes(filter), output.createObservableTypes, output.observableTypeExists) migrationStats.stage = "Migrate case templates" migrateWholeCaseTemplates(input, output, filter) + migrationStats.stage = "Migrate dashboards" + migrate(output)("Dashboard", input.listDashboards(filter), output.createDashboard, output.dashboardExists) migrationStats.stage = "Migrate cases and alerts" migrateCasesAndAlerts(input, output, filter) migrationStats.stage = "Finalisation" diff --git a/migration/src/main/scala/org/thp/thehive/migration/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/Output.scala index d8e2f3f199..f9df45d83a 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Output.scala @@ -39,4 +39,6 @@ trait Output[TX] { def createAlertObservable(tx: TX, alertId: EntityId, inputObservable: InputObservable): Try[IdMapping] def createAction(tx: TX, objectId: EntityId, inputAction: InputAction): Try[IdMapping] def createAudit(tx: TX, contextId: EntityId, inputAudit: InputAudit): Try[Unit] + def dashboardExists(tx: TX, inputDashboard: InputDashboard): Boolean + def createDashboard(tx: TX, inputDashboard: InputDashboard): Try[IdMapping] } diff --git a/migration/src/main/scala/org/thp/thehive/migration/dto/InputDashboard.scala b/migration/src/main/scala/org/thp/thehive/migration/dto/InputDashboard.scala new file mode 100644 index 0000000000..a1ece1fa9b --- /dev/null +++ b/migration/src/main/scala/org/thp/thehive/migration/dto/InputDashboard.scala @@ -0,0 +1,5 @@ +package org.thp.thehive.migration.dto + +import org.thp.thehive.models.Dashboard + +case class InputDashboard(metaData: MetaData, organisation: Option[(String, Boolean)], dashboard: Dashboard) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index b47edc7f90..34bfc3da95 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -584,4 +584,14 @@ trait Conversion { ) ) } + implicit val dashboardReads: Reads[InputDashboard] = Reads[InputDashboard] { json => + for { + metaData <- json.validate[MetaData] + title <- (json \ "title").validate[String] + description <- (json \ "description").validate[String] + definitionString <- (json \ "definition").validate[String] + definition <- Json.parse(definitionString).validate[JsObject] + status <- (json \ "status").validate[String] + } yield InputDashboard(metaData, if (status == "Shared") Some(mainOrganisation -> true) else None, Dashboard(title, description, definition)) + } } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala index 190db790a9..3ebf904c37 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala @@ -41,7 +41,7 @@ object Input { } @Singleton -class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient, implicit val ec: ExecutionContext, implicit val mat: Materializer) +class Input @Inject() (configuration: Configuration, elasticClient: ElasticClient, implicit val ec: ExecutionContext, implicit val mat: Materializer) extends migration.Input with Conversion { lazy val logger: Logger = Logger(getClass) @@ -58,7 +58,7 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient override def readAttachment(id: String): Source[ByteString, NotUsed] = Source.unfoldAsync(0) { chunkNumber => - elaticClient + elasticClient .get("data", s"${id}_$chunkNumber") .map { json => (json \ "binary").asOpt[String].map(s => chunkNumber + 1 -> ByteString(Base64.getDecoder.decode(s))) @@ -86,31 +86,31 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient } override def listCases(filter: Filter): Source[Try[InputCase], NotUsed] = - elaticClient("case", searchQuery(bool(caseFilter(filter)), "-createdAt")) + elasticClient("case", searchQuery(bool(caseFilter(filter)), "-createdAt")) .read[InputCase] override def countCases(filter: Filter): Future[Long] = - elaticClient.count("case", searchQuery(bool(caseFilter(filter)))) + elasticClient.count("case", searchQuery(bool(caseFilter(filter)))) override def countCaseObservables(filter: Filter): Future[Long] = - elaticClient.count("case_artifact", searchQuery(hasParentQuery("case", bool(caseFilter(filter))))) + elasticClient.count("case_artifact", searchQuery(hasParentQuery("case", bool(caseFilter(filter))))) override def listCaseObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed] = - elaticClient("case_artifact", searchQuery(hasParentQuery("case", idsQuery(caseId)))) + elasticClient("case_artifact", searchQuery(hasParentQuery("case", idsQuery(caseId)))) .readWithParent[InputObservable](json => Try((json \ "_parent").as[String])) override def countCaseTasks(filter: Filter): Future[Long] = - elaticClient.count("case_task", searchQuery(hasParentQuery("case", bool(caseFilter(filter))))) + elasticClient.count("case_task", searchQuery(hasParentQuery("case", bool(caseFilter(filter))))) override def listCaseTasks(caseId: String): Source[Try[(String, InputTask)], NotUsed] = - elaticClient("case_task", searchQuery(hasParentQuery("case", idsQuery(caseId)))) + elasticClient("case_task", searchQuery(hasParentQuery("case", idsQuery(caseId)))) .readWithParent[InputTask](json => Try((json \ "_parent").as[String])) override def countCaseTaskLogs(filter: Filter): Future[Long] = countCaseTaskLogs(bool(caseFilter(filter))) override def listCaseTaskLogs(caseId: String): Source[Try[(String, InputLog)], NotUsed] = - elaticClient( + elasticClient( "case_task_log", searchQuery( bool( @@ -123,7 +123,7 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient .readWithParent[InputLog](json => Try((json \ "_parent").as[String])) private def countCaseTaskLogs(caseQuery: JsObject): Future[Long] = - elaticClient.count( + elasticClient.count( "case_task_log", searchQuery( bool( @@ -147,19 +147,20 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient bool(dateFilter ++ includeFilter, Nil, excludeFilter) } + override def listAlerts(filter: Filter): Source[Try[InputAlert], NotUsed] = - elaticClient("alert", searchQuery(alertFilter(filter), "-createdAt")) + elasticClient("alert", searchQuery(alertFilter(filter), "-createdAt")) .read[InputAlert] override def countAlerts(filter: Filter): Future[Long] = - elaticClient.count("alert", searchQuery(alertFilter(filter))) + elasticClient.count("alert", searchQuery(alertFilter(filter))) override def countAlertObservables(filter: Filter): Future[Long] = Future.failed(new NotImplementedError) override def listAlertObservables(alertId: String): Source[Try[(String, InputObservable)], NotUsed] = { val dummyMetaData = MetaData("no-id", "init", new Date, None, None) Source - .future(elaticClient.searchRaw("alert", searchQuery(idsQuery(alertId)))) + .future(elasticClient.searchRaw("alert", searchQuery(idsQuery(alertId)))) .via(JsonReader.select("$.hits.hits[*]._source.artifacts[*]")) .mapConcat { data => Try(Json.parse(data.toArray[Byte])) @@ -176,25 +177,25 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient } override def listUsers(filter: Filter): Source[Try[InputUser], NotUsed] = - elaticClient("user", searchQuery(matchAll)) + elasticClient("user", searchQuery(matchAll)) .read[InputUser] override def countUsers(filter: Filter): Future[Long] = - elaticClient.count("user", searchQuery(matchAll)) + elasticClient.count("user", searchQuery(matchAll)) override def listCustomFields(filter: Filter): Source[Try[InputCustomField], NotUsed] = - elaticClient("dblist", searchQuery(or(termQuery("dblist", "case_metrics"), termQuery("dblist", "custom_fields")))) + elasticClient("dblist", searchQuery(or(termQuery("dblist", "case_metrics"), termQuery("dblist", "custom_fields")))) .read[InputCustomField] override def countCustomFields(filter: Filter): Future[Long] = - elaticClient.count("dblist", searchQuery(or(termQuery("dblist", "case_metrics"), termQuery("dblist", "custom_fields")))) + elasticClient.count("dblist", searchQuery(or(termQuery("dblist", "case_metrics"), termQuery("dblist", "custom_fields")))) override def listObservableTypes(filter: Filter): Source[Try[InputObservableType], NotUsed] = - elaticClient("dblist", searchQuery(termQuery("dblist", "list_artifactDataType"))) + elasticClient("dblist", searchQuery(termQuery("dblist", "list_artifactDataType"))) .read[InputObservableType] override def countObservableTypes(filter: Filter): Future[Long] = - elaticClient.count("dblist", searchQuery(termQuery("dblist", "list_artifactDataType"))) + elasticClient.count("dblist", searchQuery(termQuery("dblist", "list_artifactDataType"))) override def listProfiles(filter: Filter): Source[Try[InputProfile], NotUsed] = Source.empty[Try[InputProfile]] @@ -212,18 +213,18 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient override def countResolutionStatus(filter: Filter): Future[Long] = Future.successful(0) override def listCaseTemplate(filter: Filter): Source[Try[InputCaseTemplate], NotUsed] = - elaticClient("caseTemplate", searchQuery(matchAll)) + elasticClient("caseTemplate", searchQuery(matchAll)) .read[InputCaseTemplate] override def countCaseTemplate(filter: Filter): Future[Long] = - elaticClient.count("caseTemplate", searchQuery(matchAll)) + elasticClient.count("caseTemplate", searchQuery(matchAll)) override def countCaseTemplateTask(filter: Filter): Future[Long] = Future.failed(new NotImplementedError) def listCaseTemplateTask(caseTemplateId: String): Source[Try[(String, InputTask)], NotUsed] = Source .futureSource { - elaticClient + elasticClient .get("caseTemplate", caseTemplateId) .map { json => val metaData = json.as[MetaData] @@ -238,16 +239,16 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient .mapMaterializedValue(_ => NotUsed) override def countJobs(filter: Filter): Future[Long] = - elaticClient.count("case_artifact_job", searchQuery(hasParentQuery("case_artifact", hasParentQuery("case", bool(caseFilter(filter)))))) + elasticClient.count("case_artifact_job", searchQuery(hasParentQuery("case_artifact", hasParentQuery("case", bool(caseFilter(filter)))))) override def listJobs(caseId: String): Source[Try[(String, InputJob)], NotUsed] = - elaticClient("case_artifact_job", searchQuery(hasParentQuery("case_artifact", hasParentQuery("case", idsQuery(caseId))))) + elasticClient("case_artifact_job", searchQuery(hasParentQuery("case_artifact", hasParentQuery("case", idsQuery(caseId))))) .readWithParent[InputJob](json => Try((json \ "_parent").as[String]))(jobReads, classTag[InputJob]) override def countJobObservables(filter: Filter): Future[Long] = Future.failed(new NotImplementedError) override def listJobObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed] = - elaticClient("case_artifact_job", searchQuery(hasParentQuery("case_artifact", hasParentQuery("case", idsQuery(caseId))))) + elasticClient("case_artifact_job", searchQuery(hasParentQuery("case_artifact", hasParentQuery("case", idsQuery(caseId))))) .map { json => Try { val metaData = json.as[MetaData] @@ -260,10 +261,10 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient } override def countAction(filter: Filter): Future[Long] = - elaticClient.count("action", searchQuery(matchAll)) + elasticClient.count("action", searchQuery(matchAll)) override def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed] = - elaticClient("action", searchQuery(termsQuery("objectId", entityIds))) + elasticClient("action", searchQuery(termsQuery("objectId", entityIds))) .read[(String, InputAction)] private def auditFilter(filter: Filter, objectIds: String*): JsObject = { @@ -283,10 +284,17 @@ class Input @Inject() (configuration: Configuration, elaticClient: ElasticClient bool(dateFilter ++ includeFilter ++ objectIdFilter, Nil, excludeFilter) } - override def countAudit(filter: Filter): Future[Long] = - elaticClient.count("audit", searchQuery(auditFilter(filter))) + override def countAudits(filter: Filter): Future[Long] = + elasticClient.count("audit", searchQuery(auditFilter(filter))) override def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed] = - elaticClient("audit", searchQuery(auditFilter(filter, entityIds: _*))) + elasticClient("audit", searchQuery(auditFilter(filter, entityIds: _*))) .read[(String, InputAudit)] + + override def countDashboards(filter: Filter): Future[Long] = + elasticClient.count("dashboard", searchQuery(matchAll)) + + override def listDashboards(filter: Filter): Source[Try[InputDashboard], NotUsed] = + elasticClient("dashboard", searchQuery(matchAll)) + .read[InputDashboard] }