Merge branch 'develop-th4' into feature-taxonomy

TheHive-Project · Jan 21, 2021 · 58e4ae9 · 58e4ae9
2 parents 5365d7e + d33fc97
commit 58e4ae9
Show file tree

Hide file tree

Showing 36 changed files with 905 additions and 296 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,21 +1,35 @@
 # Change Log
 
+## [4.0.4](https://github.com/TheHive-Project/TheHive/milestone/67) (2021-01-12)
+
+**Implemented enhancements:**
+
+- [Feature Request] Add alert observable API endpoints [\#1732](https://github.com/TheHive-Project/TheHive/issues/1732)
+- [Feature Request] Add alert import date property [\#1733](https://github.com/TheHive-Project/TheHive/issues/1733)
+- [Feature Request] Add handling duration properties to imported Alert type [\#1734](https://github.com/TheHive-Project/TheHive/issues/1734)
+
+**Fixed bugs:**
+
+- [Bug] TheHive doesn't start if cassandra is not ready [\#1725](https://github.com/TheHive-Project/TheHive/issues/1725)
+- [Bug] Alert imported multiple times (bis) [\#1738](https://github.com/TheHive-Project/TheHive/issues/1738)
+- [Bug] Cosmetic fix in alert observables list [\#1744](https://github.com/TheHive-Project/TheHive/issues/1744)
+
 ## [4.0.3](https://github.com/TheHive-Project/TheHive/milestone/66) (2020-12-22)
 
 **Implemented enhancements:**
 
 - Providing output details for Responders [\#1293](https://github.com/TheHive-Project/TheHive/issues/1293)
 - [Enhancement] Change artifacts by observables on the onMouseOver tooltip of the eye icon of observable [\#1695](https://github.com/TheHive-Project/TheHive/issues/1695)
-- [Bug] Enhance support of S3 for attachment storage [\#1705](https://github.com/TheHive-Project/TheHive/issues/1705)
-- Update the headers of basic info sections [\#1710](https://github.com/TheHive-Project/TheHive/issues/1710)
+- [Enhancement] Enhance support of S3 for attachment storage [\#1705](https://github.com/TheHive-Project/TheHive/issues/1705)
+- [Enhancement] Update the headers of basic info sections [\#1710](https://github.com/TheHive-Project/TheHive/issues/1710)
 - [Enhancement] Add poll duration config for UI Stream [\#1720](https://github.com/TheHive-Project/TheHive/issues/1720)
 
 **Fixed bugs:**
 
 - [Bug] MISP filters are not correctly implemented [\#1685](https://github.com/TheHive-Project/TheHive/issues/1685)
 - [Bug] The query "getObservable" doesn't work for alert observables [\#1691](https://github.com/TheHive-Project/TheHive/issues/1691)
-- Click analyzers mini-report does not load the full report [\#1694](https://github.com/TheHive-Project/TheHive/issues/1694)
-- [TH4] Import file observable in gui generate error [\#1697](https://github.com/TheHive-Project/TheHive/issues/1697)
+- [Bug] Click analyzers mini-report does not load the full report [\#1694](https://github.com/TheHive-Project/TheHive/issues/1694)
+- [Bug] Import file observable in gui generate error [\#1697](https://github.com/TheHive-Project/TheHive/issues/1697)
 - [Bug] Cannot search for alerts per observables [\#1707](https://github.com/TheHive-Project/TheHive/issues/1707)
 - [Bug] Serialization problem in cluster mode [\#1708](https://github.com/TheHive-Project/TheHive/issues/1708)
 - [Bug] Issue with sorting [\#1716](https://github.com/TheHive-Project/TheHive/issues/1716)

diff --git a/ScalliGraph b/ScalliGraph
diff --git a/build.sbt b/build.sbt
@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.0.3-1"
+val thehiveVersion         = "4.0.4-1"
 val scala212               = "2.12.12"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)

diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Share.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Share.scala
@@ -0,0 +1,48 @@
+package org.thp.thehive.dto.v1
+
+import org.thp.thehive.dto.v1.ObservablesFilter.ObservablesFilter
+import org.thp.thehive.dto.v1.TasksFilter.TasksFilter
+import play.api.libs.json.{Format, Json, Writes}
+
+import java.util.Date
+
+case class InputShare(organisationName: String, profile: String, tasks: TasksFilter, observables: ObservablesFilter)
+
+object TasksFilter extends Enumeration {
+  type TasksFilter = Value
+
+  val all: TasksFilter  = Value("all")
+  val none: TasksFilter = Value("none")
+
+  implicit val format: Format[TasksFilter] = Json.formatEnum(TasksFilter)
+}
+
+object ObservablesFilter extends Enumeration {
+  type ObservablesFilter = Value
+
+  val all: ObservablesFilter  = Value("all")
+  val none: ObservablesFilter = Value("none")
+
+  implicit val format: Format[ObservablesFilter] = Json.formatEnum(ObservablesFilter)
+}
+
+object InputShare {
+  implicit val writes: Writes[InputShare] = Json.writes[InputShare]
+}
+
+case class OutputShare(
+    _id: String,
+    _type: String,
+    _createdBy: String,
+    _updatedBy: Option[String] = None,
+    _createdAt: Date,
+    _updatedAt: Option[Date] = None,
+    caseId: String,
+    profileName: String,
+    organisationName: String,
+    owner: Boolean
+)
+
+object OutputShare {
+  implicit val format: Format[OutputShare] = Json.format[OutputShare]
+}
diff --git a/frontend/app/index.html b/frontend/app/index.html
@@ -212,6 +212,7 @@
     <script src="scripts/controllers/SearchCtrl.js"></script>
     <script src="scripts/controllers/SettingsCtrl.js"></script>
     <script src="scripts/directives/affixer.js"></script>
+    <script src="scripts/directives/alert-duration.js"></script>
     <script src="scripts/directives/auto-focus.js"></script>
     <script src="scripts/directives/case-duration.js"></script>
     <script src="scripts/directives/charts/c3Chart.js"></script>

diff --git a/frontend/app/scripts/directives/alert-duration.js b/frontend/app/scripts/directives/alert-duration.js
@@ -0,0 +1,15 @@
+(function() {
+    'use strict';
+    angular.module('theHiveDirectives').directive('alertDuration', function() {
+        return {
+            restrict: 'E',
+            scope: {
+                start: '=',
+                end: '=',
+                icon: '@',
+                indicator: '='
+            },
+            templateUrl: 'views/directives/alert-duration.html'
+        };
+    });
+})();
diff --git a/frontend/app/scripts/services/api/AlertingSrv.js b/frontend/app/scripts/services/api/AlertingSrv.js
@@ -139,7 +139,8 @@
                         onUpdate: callback || undefined,
                         operations: [
                             {'_name': 'listAlert'}
-                        ]
+                        ],
+                        extraData: ['importDate']
                     });
                 },
 

diff --git a/frontend/app/views/components/alert/observable-list.component.html b/frontend/app/views/components/alert/observable-list.component.html
@@ -32,7 +32,7 @@
                     </tr>
                 </thead>
                 <tbody>
-                    <tr ng-repeat="observable in $cmp.list.values" class="clickable">
+                    <tr ng-repeat="observable in $cmp.list.values">
                         <td class="nowrap">
                             <observable-flags observable="observable" inline="true" on-filter="$cmp.addFilterValue(fieldName, value)"></observable-flags>
                         </td>

diff --git a/frontend/app/views/directives/alert-duration.html b/frontend/app/views/directives/alert-duration.html
@@ -0,0 +1,8 @@
+<span>
+    <strong ng-if="end" class="text-success" uib-tooltip="Imported after">
+        <i class="fa" ng-class="icon"></i> {{indicator ? 'During ' : ''}}{{start | duration:end}}
+    </strong>
+    <strong ng-if="!end" class="text-danger" uib-tooltip="Created since">
+        <i class="fa" ng-class="icon"></i> {{start | duration}} {{indicator ? 'ago' : ''}}
+    </strong>
+</span>
diff --git a/frontend/app/views/partials/alert/list.html b/frontend/app/views/partials/alert/list.html
@@ -136,6 +136,9 @@ <h3 class="box-title">List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList
                                     <td class="text-center">{{::event.observableCount || 0}}</td>
                                     <td>
                                         <a href ng-click="$vm.addFilterValue('date', event.date)">{{event.date | shortDate}}</a>
+                                        <div ng-if="!!event.caseId">
+                                            <alert-duration start="event._createdAt" end="event.extraData.importDate" icon="fa-clock-o"></alert-duration>
+                                        </div>
                                     </td>
                                     <td class="clearfix">
                                         <div class="pull-right" if-permission="manageAlert">

diff --git a/frontend/bower.json b/frontend/bower.json
@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.0.3-1",
+  "version": "4.0.4-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

diff --git a/frontend/package.json b/frontend/package.json
@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.0.3-1",
+    "version": "4.0.4-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

diff --git a/.../connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala b/.../connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala
@@ -71,44 +71,44 @@ class MispImportSrvTest(implicit ec: ExecutionContext) extends PlaySpecification
     }
   }
 
-  "MISP service" should {
-    "import events" in testApp { app =>
-      app[Database].roTransaction { implicit graph =>
-        app[MispImportSrv].syncMispEvents(app[TheHiveMispClient])
-        app[AlertSrv].startTraversal.getBySourceId("misp", "ORGNAME", "1").visible.getOrFail("Alert")
-      } must beSuccessfulTry(
-        Alert(
-          `type` = "misp",
-          source = "ORGNAME",
-          sourceRef = "1",
-          externalLink = Some("https://misp.test/events/1"),
-          title = "#1 test1 -> 1.2",
-          description = s"Imported from MISP Event #1, created at ${Event.simpleDateFormat.parse("2019-08-23")}",
-          severity = 3,
-          date = Event.simpleDateFormat.parse("2019-08-23"),
-          lastSyncDate = new Date(1566913355000L),
-          tlp = 2,
-          pap = 2,
-          read = false,
-          follow = true
-        )
-      ).eventually(5, 100.milliseconds)
-
-      val observables = app[Database]
-        .roTransaction { implicit graph =>
-          app[OrganisationSrv]
-            .get(EntityName("admin"))
-            .alerts
-            .getBySourceId("misp", "ORGNAME", "1")
-            .observables
-            .richObservable
-            .toList
-        }
-        .map(o => (o.`type`.name, o.data.map(_.data), o.tlp, o.message, o.tags.map(_.toString).toSet))
-//        println(observables.mkString("\n"))
-      observables must contain(
-        ("filename", Some("plop"), 0, Some(""), Set("TEST", "TH-test", "misp:category=\"Artifacts dropped\"", "misp:type=\"filename\""))
-      )
-    }
-  }
+//  "MISP service" should {
+//    "import events" in testApp { app =>
+//      app[Database].roTransaction { implicit graph =>
+//        app[MispImportSrv].syncMispEvents(app[TheHiveMispClient])
+//        app[AlertSrv].startTraversal.getBySourceId("misp", "ORGNAME", "1").visible.getOrFail("Alert")
+//      } must beSuccessfulTry(
+//        Alert(
+//          `type` = "misp",
+//          source = "ORGNAME",
+//          sourceRef = "1",
+//          externalLink = Some("https://misp.test/events/1"),
+//          title = "#1 test1 -> 1.2",
+//          description = s"Imported from MISP Event #1, created at ${Event.simpleDateFormat.parse("2019-08-23")}",
+//          severity = 3,
+//          date = Event.simpleDateFormat.parse("2019-08-23"),
+//          lastSyncDate = new Date(1566913355000L),
+//          tlp = 2,
+//          pap = 2,
+//          read = false,
+//          follow = true
+//        )
+//      ).eventually(5, 100.milliseconds)
+//
+//      val observables = app[Database]
+//        .roTransaction { implicit graph =>
+//          app[OrganisationSrv]
+//            .get(EntityName("admin"))
+//            .alerts
+//            .getBySourceId("misp", "ORGNAME", "1")
+//            .observables
+//            .richObservable
+//            .toList
+//        }
+//        .map(o => (o.`type`.name, o.data.map(_.data), o.tlp, o.message, o.tags.map(_.toString).toSet))
+////        println(observables.mkString("\n"))
+//      observables must contain(
+//        ("filename", Some("plop"), 0, Some(""), Set("TEST", "TH-test", "misp:category=\"Artifacts dropped\"", "misp:type=\"filename\""))
+//      )
+//    }
+//  }
 }
diff --git a/thehive/app/org/thp/thehive/TheHiveModule.scala b/thehive/app/org/thp/thehive/TheHiveModule.scala
@@ -101,6 +101,7 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte
     integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
     integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
     integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
+    integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
     bind[ActorRef].annotatedWithName("integrity-check-actor").toProvider[IntegrityCheckActorProvider]
 
     bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider]

diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala
@@ -1,9 +1,6 @@
 package org.thp.thehive.controllers.v0
 
-import java.util.{Base64, List => JList, Map => JMap}
-
 import io.scalaland.chimney.dsl._
-import javax.inject.{Inject, Named, Singleton}
 import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers._
@@ -27,6 +24,8 @@ import org.thp.thehive.services._
 import play.api.libs.json.{JsArray, JsObject, Json}
 import play.api.mvc.{Action, AnyContent, Results}
 
+import java.util.{Base64, List => JList, Map => JMap}
+import javax.inject.{Inject, Named, Singleton}
 import scala.util.{Failure, Success, Try}
 
 @Singleton
@@ -450,5 +449,12 @@ class PublicAlert @Inject() (
         case _ => Failure(BadRequestError("Invalid custom fields format"))
       })
       .property("case", db.idMapping)(_.select(_.`case`._id).readonly)
+      .property("imported", UMapping.boolean)(_.select(_.imported).readonly)
+      .property("importDate", UMapping.date.optional)(_.select(_.importDate).readonly)
+      .property("computed.handlingDuration", UMapping.long)(_.select(_.handlingDuration).readonly)
+      .property("computed.handlingDurationInSeconds", UMapping.long)(_.select(_.handlingDuration.math("_ / 1000").domainMap(_.toLong)).readonly)
+      .property("computed.handlingDurationInMinutes", UMapping.long)(_.select(_.handlingDuration.math("_ / 60000").domainMap(_.toLong)).readonly)
+      .property("computed.handlingDurationInHours", UMapping.long)(_.select(_.handlingDuration.math("_ / 3600000").domainMap(_.toLong)).readonly)
+      .property("computed.handlingDurationInDays", UMapping.long)(_.select(_.handlingDuration.math("_ / 86400000").domainMap(_.toLong)).readonly)
       .build
 }
diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala
@@ -352,66 +352,11 @@ class PublicCase @Inject() (
             } yield Json.obj("customFields" -> values)
           case _ => Failure(BadRequestError("Invalid custom fields format"))
         })
-      .property("computed.handlingDurationInDays", UMapping.long)(
-        _.select(
-          _.coalesceIdent(
-            _.has(_.endDate)
-              .sack(
-                (_: JLong, endDate: JLong) => endDate,
-                _.by(_.value(_.endDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long))
-              )
-              .sack((_: Long) - (_: JLong), _.by(_.value(_.startDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long)))
-              .sack((_: Long) / (_: Long), _.by(_.constant(86400000L)))
-              .sack[Long],
-            _.constant(0L)
-          )
-        ).readonly
-      )
-      .property("computed.handlingDurationInHours", UMapping.long)(
-        _.select(
-          _.coalesceIdent(
-            _.has(_.endDate)
-              .sack(
-                (_: JLong, endDate: JLong) => endDate,
-                _.by(_.value(_.endDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long))
-              )
-              .sack((_: Long) - (_: JLong), _.by(_.value(_.startDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long)))
-              .sack((_: Long) / (_: Long), _.by(_.constant(3600000L)))
-              .sack[Long],
-            _.constant(0L)
-          )
-        ).readonly
-      )
-      .property("computed.handlingDurationInMinutes", UMapping.long)(
-        _.select(
-          _.coalesceIdent(
-            _.has(_.endDate)
-              .sack(
-                (_: JLong, endDate: JLong) => endDate,
-                _.by(_.value(_.endDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long))
-              )
-              .sack((_: Long) - (_: JLong), _.by(_.value(_.startDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long)))
-              .sack((_: Long) / (_: Long), _.by(_.constant(60000L)))
-              .sack[Long],
-            _.constant(0L)
-          )
-        ).readonly
-      )
-      .property("computed.handlingDurationInSeconds", UMapping.long)(
-        _.select(
-          _.coalesceIdent(
-            _.has(_.endDate)
-              .sack(
-                (_: JLong, endDate: JLong) => endDate,
-                _.by(_.value(_.endDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long))
-              )
-              .sack((_: Long) - (_: JLong), _.by(_.value(_.startDate).graphMap[Long, JLong, Converter[Long, JLong]](_.getTime, Converter.long)))
-              .sack((_: Long) / (_: Long), _.by(_.constant(1000L)))
-              .sack[Long],
-            _.constant(0L)
-          )
-        ).readonly
-      )
+      .property("computed.handlingDuration", UMapping.long)(_.select(_.handlingDuration).readonly)
+      .property("computed.handlingDurationInSeconds", UMapping.long)(_.select(_.handlingDuration.math("_ / 1000").domainMap(_.toLong)).readonly)
+      .property("computed.handlingDurationInMinutes", UMapping.long)(_.select(_.handlingDuration.math("_ / 60000").domainMap(_.toLong)).readonly)
+      .property("computed.handlingDurationInHours", UMapping.long)(_.select(_.handlingDuration.math("_ / 3600000").domainMap(_.toLong)).readonly)
+      .property("computed.handlingDurationInDays", UMapping.long)(_.select(_.handlingDuration.math("_ / 86400000").domainMap(_.toLong)).readonly)
       .property("viewingOrganisation", UMapping.string)(
         _.authSelect((cases, authContext) => cases.organisations.visible(authContext).value(_.name)).readonly
       )
+2 −1		core/src/main/scala/org/thp/scalligraph/query/Aggregation.scala
+0 −8		core/src/main/scala/org/thp/scalligraph/services/IntegrityCheckOps.scala
+9 −3		core/src/main/scala/org/thp/scalligraph/traversal/TraversalOps.scala