Merge branch 'release/3.1.0-RC3'

CHANGELOG.md

@@ -1,7 +1,29 @@
 # Change Log
 
-## [3.1.0-RC2](https://github.com/TheHive-Project/TheHive/tree/3.1.0-RC2) (2018-08-27)
+## [3.1.0-RC3](https://github.com/TheHive-Project/TheHive/tree/3.1.0-RC3) (2018-09-06)
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.1.0-RC2...3.1.0-RC3)
+
+**Implemented enhancements:**
+
+- Display task description via a collapsible row [\#709](https://github.com/TheHive-Project/TheHive/issues/709)
+- Allow task group auto complete in case template admin section [\#707](https://github.com/TheHive-Project/TheHive/issues/707)
+- Display task group in global task lists [\#705](https://github.com/TheHive-Project/TheHive/issues/705)
+- Make task group input optional [\#696](https://github.com/TheHive-Project/TheHive/issues/696)
+- Related Cases: See \(x\) more links [\#690](https://github.com/TheHive-Project/TheHive/issues/690)
+- Search section: Search for a string over all types of objects [\#689](https://github.com/TheHive-Project/TheHive/issues/689)
+- Filter on computedHandlingDuration in SearchDialog fails [\#688](https://github.com/TheHive-Project/TheHive/issues/688)
+- Extend Case Description Field [\#81](https://github.com/TheHive-Project/TheHive/issues/81)
+- Change layout of observable creation form [\#706](https://github.com/TheHive-Project/TheHive/pull/706) ([srilumpa](https://github.com/srilumpa))
 
+**Fixed bugs:**
+
+- .sbt build of current git version fails with x-pack-transport error [\#710](https://github.com/TheHive-Project/TheHive/issues/710)
+- PKI authentication fails if user name in certificate has the wrong case [\#700](https://github.com/TheHive-Project/TheHive/issues/700)
+- Error handling deletion and re creation of file observables [\#699](https://github.com/TheHive-Project/TheHive/issues/699)
+- Start waiting tasks when adding task logs [\#695](https://github.com/TheHive-Project/TheHive/issues/695)
+- Adding new observables to an alert retrospectively is impossible [\#511](https://github.com/TheHive-Project/TheHive/issues/511)
+
+## [3.1.0-RC2](https://github.com/TheHive-Project/TheHive/tree/3.1.0-RC2) (2018-08-27)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.1.0-RC1...3.1.0-RC2)
 
 **Implemented enhancements:**

thehive-backend/app/controllers/ArtifactCtrl.scala

@@ -125,14 +125,14 @@ class ArtifactCtrl @Inject() (
 
   @Timed
   def update(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒
-    artifactSrv.update(id, request.body)
+    artifactSrv.update(id, request.body.unset("attachment"))
       .map(artifact ⇒ renderer.toOutput(OK, artifact))
   }
 
   @Timed
   def bulkUpdate(): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒
     request.body.getStrings("ids").fold(Future.successful(Ok(JsArray()))) { ids ⇒
-      artifactSrv.bulkUpdate(ids, request.body.unset("ids")).map(multiResult ⇒ renderer.toMultiOutput(OK, multiResult))
+      artifactSrv.bulkUpdate(ids, request.body.unset("ids").unset("attachment")).map(multiResult ⇒ renderer.toMultiOutput(OK, multiResult))
     }
   }
 

thehive-backend/app/controllers/DescribeCtrl.scala

@@ -40,7 +40,7 @@ class DescribeCtrl @Inject() (
       .getOrElse(NotFound(s"Model $modelName not found"))
   }
 
-  private val allModels: Seq[String] = Seq("case", "case_artifact", "case_task", "alert", "case_artifact_job", "audit")
+  private val allModels: Seq[String] = Seq("case", "case_artifact", "case_task", "case_task_log", "alert", "case_artifact_job", "audit")
   def describeAll: Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
     val entityDefinitions = modelSrv.list
       .collect {

thehive-backend/app/controllers/LogCtrl.scala

@@ -1,20 +1,19 @@
 package controllers
 
-import javax.inject.{ Inject, Singleton }
-
 import scala.concurrent.ExecutionContext
 
 import play.api.http.Status
 import play.api.mvc._
 
+import javax.inject.{ Inject, Singleton }
 import models.Roles
 import services.LogSrv
 
-import org.elastic4play.Timed
 import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
 import org.elastic4play.models.JsonFormat.baseModelEntityWrites
-import org.elastic4play.services.JsonFormat.queryReads
-import org.elastic4play.services.{ QueryDSL, QueryDef }
+import org.elastic4play.services.JsonFormat.{ aggReads, queryReads }
+import org.elastic4play.services.{ Agg, QueryDSL, QueryDef }
+import org.elastic4play.{ BadRequestError, Timed }
 
 @Singleton
 class LogCtrl @Inject() (
@@ -70,4 +69,11 @@ class LogCtrl @Inject() (
     val (logs, total) = logSrv.find(query, range, sort)
     renderer.toOutput(OK, logs, total)
   }
+
+  @Timed
+  def stats(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
+    val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
+    val aggs = request.body.getValue("stats").getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]]
+    logSrv.stats(query, aggs).map(s ⇒ Ok(s))
+  }
 }

thehive-backend/app/models/Artifact.scala

@@ -34,7 +34,7 @@ trait ArtifactAttributes { _: AttributeDef ⇒
   val dataType: A[String] = attribute("dataType", F.listEnumFmt("artifactDataType")(dblists), "Type of the artifact", O.readonly)
   val message: A[Option[String]] = optionalAttribute("message", F.textFmt, "Description of the artifact in the context of the case")
   val startDate: A[Date] = attribute("startDate", F.dateFmt, "Creation date", new Date)
-  val attachment: A[Option[Attachment]] = optionalAttribute("attachment", F.attachmentFmt, "Artifact file content", O.readonly)
+  val attachment: A[Option[Attachment]] = optionalAttribute("attachment", F.attachmentFmt, "Artifact file content")
   val tlp: A[Long] = attribute("tlp", TlpAttributeFormat, "TLP level", 2L)
   val tags: A[Seq[String]] = multiAttribute("tags", F.stringFmt, "Artifact tags")
   val ioc: A[Boolean] = attribute("ioc", F.booleanFmt, "Artifact is an IOC", false)

thehive-backend/app/services/ArtifactSrv.scala

@@ -63,7 +63,6 @@ class ArtifactSrv @Inject() (
           fields
             .unset("data")
             .unset("dataType")
-            .unset("attachment")
             .set("status", "Ok"),
           modifyConfig)
       } yield updatedArtifact

thehive-backend/app/services/LogSrv.scala

@@ -1,15 +1,14 @@
 package services
 
-import javax.inject.{ Inject, Singleton }
-
+import javax.inject.{ Inject, Provider, Singleton }
 import scala.concurrent.{ ExecutionContext, Future }
 
 import play.api.libs.json.JsObject
 
 import akka.NotUsed
 import akka.stream.Materializer
 import akka.stream.scaladsl.{ Sink, Source }
-import models.{ Log, LogModel, Task, TaskModel }
+import models._
 
 import org.elastic4play.controllers.Fields
 import org.elastic4play.database.{ DBRemove, ModifyConfig }
@@ -20,6 +19,7 @@ class LogSrv @Inject() (
     logModel: LogModel,
     taskModel: TaskModel,
     auditSrv: AuditSrv,
+    taskSrvProvider: Provider[TaskSrv],
     createSrv: CreateSrv,
     getSrv: GetSrv,
     updateSrv: UpdateSrv,
@@ -30,11 +30,14 @@ class LogSrv @Inject() (
     implicit val ec: ExecutionContext,
     implicit val mat: Materializer) {
 
+  lazy val taskSrv: TaskSrv = taskSrvProvider.get
+
   def create(taskId: String, fields: Fields)(implicit authContext: AuthContext): Future[Log] =
     getSrv[TaskModel, Task](taskModel, taskId)
       .flatMap { task ⇒ create(task, fields) }
 
   def create(task: Task, fields: Fields)(implicit authContext: AuthContext): Future[Log] = {
+    if (task.status() == TaskStatus.Waiting) taskSrv.update(task, Fields.empty.set("status", TaskStatus.InProgress.toString))
     createSrv[LogModel, Log, Task](logModel, task, fields.addIfAbsent("owner", authContext.userId))
   }
 
@@ -69,5 +72,5 @@ class LogSrv @Inject() (
     findSrv[LogModel, Log](logModel, queryDef, range, sortBy)
   }
 
-  def stats(queryDef: QueryDef, agg: Agg*): Future[JsObject] = findSrv(logModel, queryDef, agg: _*)
+  def stats(queryDef: QueryDef, agg: Seq[Agg]): Future[JsObject] = findSrv(logModel, queryDef, agg: _*)
 }

thehive-backend/app/services/TaskSrv.scala

@@ -62,7 +62,7 @@ class TaskSrv @Inject() (
   def update(task: Task, fields: Fields, modifyConfig: ModifyConfig)(implicit authContext: AuthContext): Future[Task] = {
     // if update status from waiting to something else and owner is not set, then set owner to user
     val f = if (task.status() == TaskStatus.Waiting &&
-      !fields.getString("status").forall(_ == TaskStatus.Waiting.toString) &&
+      !fields.getString("status").contains(TaskStatus.Waiting.toString) &&
       !fields.contains("owner") &&
       task.owner().isEmpty)
       fields.set("owner", authContext.userId)

thehive-backend/app/services/UserSrv.scala

@@ -75,7 +75,7 @@ class UserSrv @Inject() (
     }
   }
 
-  override def get(id: String): Future[User] = getSrv[UserModel, User](userModel, id)
+  override def get(id: String): Future[User] = getSrv[UserModel, User](userModel, id.toLowerCase)
 
   def update(id: String, fields: Fields)(implicit authContext: AuthContext): Future[User] =
     update(id, fields, ModifyConfig.default)

thehive-backend/conf/routes

@@ -50,6 +50,7 @@ POST     /api/case/:caseId/task                   controllers.TaskCtrl.create(ca
 GET      /api/case/task/:taskId/log               controllers.LogCtrl.findInTask(taskId)
 POST     /api/case/task/:taskId/log/_search       controllers.LogCtrl.findInTask(taskId)
 POST     /api/case/task/log/_search               controllers.LogCtrl.find()
+POST     /api/case/task/log/_stats                controllers.LogCtrl.stats()
 POST     /api/case/task/:taskId/log               controllers.LogCtrl.create(taskId)
 PATCH    /api/case/task/log/:logId                controllers.LogCtrl.update(logId)
 DELETE   /api/case/task/log/:logId                controllers.LogCtrl.delete(logId)

ui/app/scripts/controllers/SearchCtrl.js

@@ -4,8 +4,10 @@
         .controller('SearchCtrl', function($scope, $q, $stateParams, $uibModal, PSearchSrv, CaseTemplateSrv, CaseTaskSrv, NotificationSrv, EntitySrv, UserInfoSrv, QueryBuilderSrv, GlobalSearchSrv, metadata) {
             $scope.metadata = metadata;
             $scope.toolbar = [
+                {name: 'all', label: 'All', icon: 'glyphicon glyphicon-search'},
                 {name: 'case', label: 'Cases', icon: 'glyphicon glyphicon-folder-open'},
                 {name: 'case_task', label: 'Tasks', icon: 'glyphicon glyphicon-tasks'},
+                {name: 'case_task_log', label: 'Tasks Logs', icon: 'glyphicon glyphicon-comment'},
                 {name: 'case_artifact', label: 'Observables', icon: 'glyphicon glyphicon-pushpin'},
                 {name: 'alert', label: 'Alerts', icon: 'glyphicon glyphicon-alert'},
                 {name: 'case_artifact_job', label: 'Jobs', icon: 'glyphicon glyphicon-cog'},
@@ -103,16 +105,23 @@
                 $scope.search();
             };
 
+            $scope.filterFields = function(entity) {
+                return _.filter($scope.metadata[entity].attributes, function(value, key) {
+                    return !key.startsWith('computed.');
+                });
+            };
+
             $scope.search = function() {
-                var entity = $scope.config.entity,
-                    search = $scope.config[entity].search,
-                    filters = $scope.config[entity].filters || [],
+                var entityName = $scope.config.entity,
+                    entity = $scope.config[entityName] || {},
+                    search = entity.search,
+                    filters = entity.filters || [],
                     filters_query = null,
                     search_query = null;
 
                 try {
-                    if(filters.length > 0) {
-                        filters_query = QueryBuilderSrv.buildFiltersQuery($scope.metadata[entity].attributes, filters);
+                    if(entityName !== 'all' && filters.length > 0) {
+                        filters_query = QueryBuilderSrv.buildFiltersQuery($scope.metadata[entityName].attributes, filters);
                     }
 
                     if(search) {
@@ -125,11 +134,11 @@
                     if(query) {
                         GlobalSearchSrv.save($scope.config);
 
-                        $scope.searchResults = PSearchSrv(undefined, $scope.metadata[entity].path, {
+                        $scope.searchResults = PSearchSrv(undefined, entityName === 'all' ? 'any' : $scope.metadata[entityName].path, {
                             filter: query,
                             baseFilter: $scope.baseFilter,
                             nparent: 10,
-                            nstats: entity === 'audit',
+                            nstats: entityName === 'audit',
                             skipStream: true
                         });
                     } else {

ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js

@@ -202,10 +202,15 @@
                             return action;
                         },
                         task: function() {
-                            return _.extend({}, {group: 'default'}, task);
+                            return _.extend({}, task);
                         },
                         users: function() {
                             return UserSrv.list({ status: 'Ok' });
+                        },
+                        groups: function() {
+                            var existingGroups = _.uniq(_.pluck(self.template.tasks, 'group').sort());
+
+                            return existingGroups.length === 0 ? ['default'] : existingGroups;
                         }
                     }
                 });
@@ -366,10 +371,11 @@
                     });
             };
         })
-        .controller('AdminCaseTemplateTasksCtrl', function($scope, $uibModalInstance, action, task, users) {
+        .controller('AdminCaseTemplateTasksCtrl', function($scope, $uibModalInstance, action, task, users, groups) {
             $scope.task = task || {};
             $scope.action = action;
             $scope.users = users;
+            $scope.groups = groups;
 
             $scope.cancel = function() {
                 $uibModalInstance.dismiss();

ui/app/scripts/controllers/case/CaseLinksCtrl.js

@@ -9,6 +9,7 @@
             $scope.sorting = {
               field: '-startDate'
             }
+            $scope.displayOptions = {};
             var tabName = 'links-' + $scope.caseId;
 
             // Add tab
@@ -79,6 +80,14 @@
 
             $scope.$watch('links', function(data){
                 $scope.linkStats = $scope.initStats(data);
+
+                _.each(data, function(link) {
+                    if($scope.displayOptions[link.id] === undefined) {
+                        $scope.displayOptions[link.id] = 5;
+                    }
+                });
+
+
             });
         }
     );

ui/app/scripts/controllers/case/CaseTasksCtrl.js

@@ -17,6 +17,7 @@
             status: 'Waiting'
         };
         $scope.taskResponders = null;
+        $scope.collapseOptions = {};
 
         $scope.tasks = PSearchSrv($scope.caseId, 'case_task', {
             scope: $scope,

ui/app/scripts/controllers/case/CaseTasksItemCtrl.js

@@ -178,6 +178,11 @@
                 TaskLogSrv.save({
                     'taskId': $scope.task.id
                 }, $scope.newLog, function () {
+                    if($scope.task.status === 'Waiting') {
+                        // Reload the task
+                        $scope.reloadTask();
+                    }
+
                     delete $scope.newLog.attachment;
                     $scope.state.attachmentCollapsed = true;
                     $scope.newLog.message = '';
@@ -227,6 +232,16 @@
                   });
             };
 
+            $scope.reloadTask = function() {
+                CaseTaskSrv.get({
+                    'taskId': $scope.task.id
+                }, function(data) {
+                    $scope.task = data;
+                }, function(response) {
+                    NotificationSrv.error('taskDetails', response.data, response.status);
+                });
+            }
+
             // Add tabs
             CaseTabsSrv.addTab($scope.tabName, {
                 name: $scope.tabName,

ui/app/scripts/controllers/case/ObservableCreationCtrl.js

@@ -11,7 +11,6 @@
             $scope.pendingAsync = false;
             $scope.step = 'form';
             $scope.params = params || {
-                bulk: false,
                 ioc: false,
                 sighted: false,
                 isZip: false,
@@ -81,14 +80,8 @@
                     };
 
                 if (params.data) {
-
-                    if ($scope.params.bulk) {
-                        postData.data = params.data.split('\n');
-                        count = postData.length;
-                    } else {
-                        postData.data = params.data;
-                    }
-
+                    postData.data = params.data.split('\n');
+                    count = postData.length;
                 } else if (params.attachment) {
                     postData.attachment = params.attachment;
 

ui/app/styles/main.css

@@ -658,3 +658,12 @@ table tr td.task-actions span.action-button {
     max-height: 550px;
     overflow-y: auto;
 }
+
+.table.tasks-table>tbody+tbody {
+  border: none;
+}
+
+.table.tasks-table .filter-panel {
+  margin-left:40px;
+  margin-right:40px;
+}

ui/app/views/directives/updatable-select.html

@@ -1,6 +1,6 @@
 <span ng-hide="updatable.updating" ng-init="active = false" ng-mouseenter="active = true" ng-mouseleave="active = false">
 	<span ng-if="value!==null && value !==''" style="vertical-align: top; white-space: pre-wrap" class="updatable-value">{{value}}</span>
-	<span ng-if="value === null || value === undefined" style="vertical-align: top;" class="updatable-value text-warning"><em>Not Specified</em></span>
+	<span ng-if="value === null || value === undefined || value === ''" style="vertical-align: top;" class="updatable-value text-warning"><em>Not Specified</em></span>
 	<small ng-show="active">
 		<a href class="clickable" tooltip-popup-delay='500' uib-tooltip="edit">
 			<i class="glyphicon glyphicon-pencil" ng-click="edit()"></i>

ui/app/views/partials/admin/case-templates.task.html

@@ -17,7 +17,9 @@ <h3 class="modal-title">{{action}} task</h3>
                 <i class="fa fa-asterisk text-danger"></i>
             </label>
             <div class="col-md-9">
-                <input class="form-control" ng-model="task.group" placeholder="Task group" required type="text">
+                <input class="form-control" ng-model="task.group" placeholder="Task group" required type="text"
+                    uib-typeahead="g for g in groups | filter:$viewValue"
+                    typeahead-min-length="0">
             </div>
         </div>