Merge branch 'develop-th4' into feature/indexBackend

TheHive-Project · Feb 9, 2021 · fefc923 · fefc923
2 parents f9db070 + 497a9eb
commit fefc923
Show file tree

Hide file tree

Showing 106 changed files with 1,995 additions and 722 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,34 @@
 # Change Log
 
+## [4.0.5](https://github.com/TheHive-Project/TheHive/milestone/68) (2021-02-08)
+
+**Implemented enhancements:**
+
+- Support for using asterisks by tag-filtering [\#933](https://github.com/TheHive-Project/TheHive/issues/933)
+- "Close tasks and case" deletes tasks instead of closing them [\#1755](https://github.com/TheHive-Project/TheHive/issues/1755)
+- [Enhancement] Add schema update status in status API [\#1782](https://github.com/TheHive-Project/TheHive/issues/1782)
+
+**Closed issues:**
+
+- Running TheHive 4.0.1-1 it appears that application.log is no longer rotated. [\#1746](https://github.com/TheHive-Project/TheHive/issues/1746)
+
+**Fixed bugs:**
+
+- [Bug] RPM package does not create secret.conf file [\#1248](https://github.com/TheHive-Project/TheHive/issues/1248)
+- [Bug] More webhooks or more detailed webhook events [\#1739](https://github.com/TheHive-Project/TheHive/issues/1739)
+- [Bug] Webhooks opening infinite amount of files [\#1743](https://github.com/TheHive-Project/TheHive/issues/1743)
+- [Bug] Dashboards are always created as private [\#1754](https://github.com/TheHive-Project/TheHive/issues/1754)
+- [Bug]/Unable to get MISP organisation [\#1758](https://github.com/TheHive-Project/TheHive/issues/1758)
+- [Bug] TheHive 4 Cluster and Haproxy with roundrobin [\#1760](https://github.com/TheHive-Project/TheHive/issues/1760)
+- [Bug] TheHive -> MISP works. MISP -> TheHive not. [\#1761](https://github.com/TheHive-Project/TheHive/issues/1761)
+- [Bug] TheHive 4.0.4 cannot show tasks created in previous versions [\#1763](https://github.com/TheHive-Project/TheHive/issues/1763)
+- [Bug] `Imported` property in Alerts not taken into account  [\#1769](https://github.com/TheHive-Project/TheHive/issues/1769)
+- [Bug] Sort field list in dashboard widget filters [\#1771](https://github.com/TheHive-Project/TheHive/issues/1771)
+- [Bug] Dashboard on organisation (and other) doesn't work [\#1772](https://github.com/TheHive-Project/TheHive/issues/1772)
+- [BUG] Cannot link multiple organisations together [\#1773](https://github.com/TheHive-Project/TheHive/issues/1773)
+- [Bug] Fix pivoting from donuts to search pages on custom fields based widgets [\#1777](https://github.com/TheHive-Project/TheHive/issues/1777)
+- [Bug] Fix custom field filters in v0 APIs [\#1779](https://github.com/TheHive-Project/TheHive/issues/1779)
+
 ## [4.0.4](https://github.com/TheHive-Project/TheHive/milestone/67) (2021-01-12)
 
 **Implemented enhancements:**
@@ -269,7 +298,6 @@
 - SearchSrv.NotFoundError  [\#1242](https://github.com/TheHive-Project/TheHive/issues/1242)
 - Assignee is not changeable [\#1243](https://github.com/TheHive-Project/TheHive/issues/1243)
 - [Bug] In TheHive, a user is a member of one or more organisations. One user has a profile for each organisation and can have different profiles for different organisations.  [\#1247](https://github.com/TheHive-Project/TheHive/issues/1247)
-- [Bug] RPM package does not create secret.conf file [\#1248](https://github.com/TheHive-Project/TheHive/issues/1248)
 - [Bug] Unable to save new or imported dashboards in 4.0-RC1 [\#1250](https://github.com/TheHive-Project/TheHive/issues/1250)
 - [Bug] Header Variable authentication does not work [\#1251](https://github.com/TheHive-Project/TheHive/issues/1251)
 - Filtering by custom fields returns no results [\#1252](https://github.com/TheHive-Project/TheHive/issues/1252)

diff --git a/ScalliGraph b/ScalliGraph
diff --git a/build.sbt b/build.sbt
@@ -171,7 +171,10 @@ lazy val thehiveDto = (project in file("dto"))
   .dependsOn(scalligraph)
   .settings(
     name := "thehive-dto",
-    version := thehiveVersion
+    version := thehiveVersion,
+    libraryDependencies ++= Seq(
+      aix
+    )
   )
 
 lazy val thehiveClient = (project in file("client"))

diff --git a/conf/migration-logback.xml b/conf/migration-logback.xml
@@ -5,7 +5,7 @@
                     converterClass="play.api.libs.logback.ColoredLevel"/>
 
     <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>./logs/migration.log</file>
+        <file>${application.home:-.}/logs/migration.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
             <fileNamePattern>${application.home:-.}/logs/application.%i.log.zip</fileNamePattern>
             <minIndex>1</minIndex>

diff --git a/...ector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchemaDefinition.scala b/...ector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchemaDefinition.scala
@@ -16,8 +16,7 @@ import scala.reflect.runtime.{universe => ru}
 class CortexSchemaDefinition @Inject() () extends Schema with UpdatableSchema {
 
   lazy val logger: Logger    = Logger(getClass)
-  val name: String           = "thehive-cortex"
-  val operations: Operations = Operations(name)
+  val operations: Operations = Operations("thehive-cortex")
 
   lazy val reflectionClasses = new Reflections(
     new ConfigurationBuilder()

diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Connector.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Connector.scala
@@ -2,9 +2,13 @@ package org.thp.thehive.connector.cortex.services
 
 import akka.actor.ActorSystem
 import akka.stream.Materializer
+
+import javax.inject.{Inject, Singleton}
 import org.thp.cortex.client.{CortexClient, CortexClientConfig}
+import org.thp.scalligraph.models.SchemaStatus
 import org.thp.scalligraph.services.config.ApplicationConfig.finiteDurationFormat
 import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem}
+import org.thp.thehive.connector.cortex.models.CortexSchemaDefinition
 import org.thp.thehive.models.HealthStatus
 import org.thp.thehive.services.{Connector => TheHiveConnector}
 import play.api.libs.json.{JsObject, Json}
@@ -17,6 +21,7 @@ import scala.util.{Failure, Success}
 @Singleton
 class Connector @Inject() (
     appConfig: ApplicationConfig,
+    schemaDefinition: CortexSchemaDefinition,
     mat: Materializer,
     implicit val system: ActorSystem,
     implicit val ec: ExecutionContext
@@ -85,4 +90,5 @@ class Connector @Inject() (
       }
   updateStatus()
 
+  override def schemaStatus: Option[SchemaStatus] = schemaDefinition.schemaStatus
 }
diff --git a/...ex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/TestConnector.scala b/...ex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/TestConnector.scala
@@ -2,15 +2,24 @@ package org.thp.thehive.connector.cortex.services
 
 import akka.actor.ActorSystem
 import akka.stream.Materializer
+
+import javax.inject.{Inject, Singleton}
 import org.thp.cortex.client.CortexClient
 import org.thp.scalligraph.services.config.ApplicationConfig
+import org.thp.thehive.connector.cortex.models.CortexSchemaDefinition
 
 import javax.inject.{Inject, Singleton}
 import scala.concurrent.ExecutionContext
 
 @Singleton
-class TestConnector @Inject() (client: CortexClient, appConfig: ApplicationConfig, mat: Materializer, system: ActorSystem, ec: ExecutionContext)
-    extends Connector(appConfig, mat, system, ec) {
+class TestConnector @Inject() (
+    client: CortexClient,
+    appConfig: ApplicationConfig,
+    schemaDefinition: CortexSchemaDefinition,
+    mat: Materializer,
+    system: ActorSystem,
+    ec: ExecutionContext
+) extends Connector(appConfig, schemaDefinition, mat, system, ec) {
   override def clients: Seq[CortexClient] = Seq(client)
 
   override protected def updateHealth(): Unit = ()

diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Alert.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Alert.scala
@@ -1,5 +1,7 @@
 package org.thp.thehive.dto.v1
 
+import ai.x.play.json.Encoders.encoder
+import ai.x.play.json.Jsonx
 import org.thp.scalligraph.controllers.WithParser
 import play.api.libs.json._
 
@@ -54,87 +56,5 @@ case class OutputAlert(
 )
 
 object OutputAlert {
-  implicit val reads: Reads[OutputAlert] = Reads[OutputAlert] { json =>
-    for {
-      _id             <- (json \ "_id").validate[String]
-      _type           <- (json \ "_type").validate[String]
-      _createdBy      <- (json \ "_createdBy").validate[String]
-      _updatedBy      <- (json \ "_updatedBy").validateOpt[String]
-      _createdAt      <- (json \ "_createdAt").validate[Date]
-      _updatedAt      <- (json \ "_updatedAt").validateOpt[Date]
-      tpe             <- (json \ "type").validate[String]
-      source          <- (json \ "source").validate[String]
-      sourceRef       <- (json \ "sourceRef").validate[String]
-      externalLink    <- (json \ "externalLink").validateOpt[String]
-      title           <- (json \ "title").validate[String]
-      description     <- (json \ "description").validate[String]
-      severity        <- (json \ "severity").validate[Int]
-      date            <- (json \ "date").validate[Date]
-      tags            <- (json \ "tags").validate[Set[String]]
-      tlp             <- (json \ "tlp").validate[Int]
-      pap             <- (json \ "pap").validate[Int]
-      read            <- (json \ "read").validate[Boolean]
-      follow          <- (json \ "follow").validate[Boolean]
-      customFields    <- (json \ "customFields").validate[Seq[OutputCustomFieldValue]]
-      caseTemplate    <- (json \ "caseTemplate").validateOpt[String]
-      observableCount <- (json \ "observableCount").validate[Long]
-      caseId          <- (json \ "caseId").validateOpt[String]
-      extraData       <- (json \ "extraData").validate[JsObject]
-    } yield OutputAlert(
-      _id,
-      _type,
-      _createdBy,
-      _updatedBy,
-      _createdAt,
-      _updatedAt,
-      tpe,
-      source,
-      sourceRef,
-      externalLink,
-      title,
-      description,
-      severity,
-      date,
-      tags,
-      tlp,
-      pap,
-      read,
-      follow,
-      customFields,
-      caseTemplate,
-      observableCount,
-      caseId,
-      extraData
-    )
-  }
-  implicit val writes: OWrites[OutputAlert] = OWrites[OutputAlert] { outputAlert =>
-    Json.obj(
-      "_id"             -> outputAlert._id,
-      "_type"           -> outputAlert._type,
-      "_createdBy"      -> outputAlert._createdBy,
-      "_updatedBy"      -> outputAlert._updatedBy,
-      "_createdAt"      -> outputAlert._createdAt,
-      "_updatedAt"      -> outputAlert._updatedAt,
-      "type"            -> outputAlert.`type`,
-      "source"          -> outputAlert.source,
-      "sourceRef"       -> outputAlert.sourceRef,
-      "externalLink"    -> outputAlert.externalLink,
-      "title"           -> outputAlert.title,
-      "description"     -> outputAlert.description,
-      "severity"        -> outputAlert.severity,
-      "date"            -> outputAlert.date,
-      "tags"            -> outputAlert.tags,
-      "tlp"             -> outputAlert.tlp,
-      "pap"             -> outputAlert.pap,
-      "read"            -> outputAlert.read,
-      "follow"          -> outputAlert.follow,
-      "customFields"    -> outputAlert.customFields,
-      "caseTemplate"    -> outputAlert.caseTemplate,
-      "observableCount" -> outputAlert.observableCount,
-      "caseId"          -> outputAlert.caseId,
-      "extraData"       -> outputAlert.extraData
-    )
-  }
-
-  implicit val format: OFormat[OutputAlert] = OFormat(reads, writes)
+  implicit val format: OFormat[OutputAlert] = Jsonx.formatCaseClass[OutputAlert]
 }
diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala
@@ -1,5 +1,9 @@
 package org.thp.thehive.dto.v1
 
+import ai.x.play.json.Encoders.encoder
+import ai.x.play.json.Jsonx
+
+import java.util.Date
 import org.thp.scalligraph.controllers.WithParser
 import play.api.libs.json._
 
@@ -53,86 +57,5 @@ case class OutputCase(
 )
 
 object OutputCase {
-
-  val reads: Reads[OutputCase] = Reads[OutputCase] { json =>
-    for {
-      _id              <- (json \ "_id").validate[String]
-      _type            <- (json \ "_type").validate[String]
-      _createdBy       <- (json \ "_createdBy").validate[String]
-      _updatedBy       <- (json \ "_updatedBy").validateOpt[String]
-      _createdAt       <- (json \ "_createdAt").validate[Date]
-      _updatedAt       <- (json \ "_updatedAt").validateOpt[Date]
-      number           <- (json \ "number").validate[Int]
-      title            <- (json \ "title").validate[String]
-      description      <- (json \ "description").validate[String]
-      severity         <- (json \ "severity").validate[Int]
-      startDate        <- (json \ "startDate").validate[Date]
-      endDate          <- (json \ "endDate").validateOpt[Date]
-      tags             <- (json \ "tags").validate[Set[String]]
-      flag             <- (json \ "flag").validate[Boolean]
-      tlp              <- (json \ "tlp").validate[Int]
-      pap              <- (json \ "pap").validate[Int]
-      status           <- (json \ "status").validate[String]
-      summary          <- (json \ "summary").validateOpt[String]
-      impactStatus     <- (json \ "impactStatus").validateOpt[String]
-      resolutionStatus <- (json \ "resolutionStatus").validateOpt[String]
-      assignee         <- (json \ "assignee").validateOpt[String]
-      customFields     <- (json \ "customFields").validate[Seq[OutputCustomFieldValue]]
-      extraData        <- (json \ "extraData").validate[JsObject]
-    } yield OutputCase(
-      _id,
-      _type,
-      _createdBy,
-      _updatedBy,
-      _createdAt,
-      _updatedAt,
-      number,
-      title,
-      description,
-      severity,
-      startDate,
-      endDate,
-      tags,
-      flag,
-      tlp,
-      pap,
-      status,
-      summary,
-      impactStatus,
-      resolutionStatus,
-      assignee,
-      customFields,
-      extraData
-    )
-  }
-
-  val writes: OWrites[OutputCase] = OWrites[OutputCase] { outputCase =>
-    Json.obj(
-      "_id"              -> outputCase._id,
-      "_type"            -> outputCase._type,
-      "_createdBy"       -> outputCase._createdBy,
-      "_updatedBy"       -> outputCase._updatedBy,
-      "_createdAt"       -> outputCase._createdAt,
-      "_updatedAt"       -> outputCase._updatedAt,
-      "number"           -> outputCase.number,
-      "title"            -> outputCase.title,
-      "description"      -> outputCase.description,
-      "severity"         -> outputCase.severity,
-      "startDate"        -> outputCase.startDate,
-      "endDate"          -> outputCase.endDate,
-      "tags"             -> outputCase.tags,
-      "flag"             -> outputCase.flag,
-      "tlp"              -> outputCase.tlp,
-      "pap"              -> outputCase.pap,
-      "status"           -> outputCase.status,
-      "summary"          -> outputCase.summary,
-      "impactStatus"     -> outputCase.impactStatus,
-      "resolutionStatus" -> outputCase.resolutionStatus,
-      "assignee"         -> outputCase.assignee,
-      "customFields"     -> outputCase.customFields,
-      "extraData"        -> outputCase.extraData
-    )
-  }
-
-  implicit val format: OFormat[OutputCase] = OFormat(reads, writes)
+  implicit val format: OFormat[OutputCase] = Jsonx.formatCaseClass[OutputCase]
 }
+7 −2		core/src/main/scala/org/thp/scalligraph/models/Database.scala
+10 −1		core/src/main/scala/org/thp/scalligraph/models/Schema.scala