#466 Make Cortex connector ready for Cortex2

project/Dependencies.scala

@@ -20,6 +20,6 @@ object Dependencies {
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.4"
+    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.5-SNAPSHOT"
   }
 }

thehive-backend/app/services/StreamSrv.scala

@@ -198,7 +198,6 @@ class StreamActor(
 
     case Initialize(requestId) ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None)))
     case _: AuditOperation     ⇒
-    case message               ⇒ logger.warn(s"Unexpected message $message (${message.getClass})")
   }
 
   def receive: Receive = receiveWithState(None, Map.empty[String, Option[StreamMessageGroup[_]]])

thehive-cortex/app/connectors/cortex/models/Job.scala

@@ -1,17 +1,15 @@
 package connectors.cortex.models
 
 import java.util.Date
-
 import javax.inject.{ Inject, Singleton }
 
 import scala.concurrent.Future
 
-import play.api.libs.json.{ JsObject, JsValue, Json }
+import play.api.libs.json.{ JsObject, JsString, JsValue, Json }
 
 import org.elastic4play.JsonFormat.dateFormat
-import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, AttributeOption ⇒ O, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration }
+import org.elastic4play.models.{ AttributeDef, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F, AttributeOption ⇒ O }
 import org.elastic4play.utils.RichJson
-
 import connectors.cortex.models.JsonFormat.jobStatusFormat
 import models.{ Artifact, ArtifactModel }
 import services.AuditedModel
@@ -23,6 +21,8 @@ object JobStatus extends Enumeration with HiveEnumeration {
 
 trait JobAttributes { _: AttributeDef ⇒
   val analyzerId = attribute("analyzerId", F.stringFmt, "Analyzer", O.readonly)
+  val analyzerName = optionalAttribute("analyzerName", F.stringFmt, "Name of the analyzer", O.readonly)
+  val analyzerDefinition = optionalAttribute("analyzerDefinition", F.stringFmt, "Name of the analyzer definition", O.readonly)
   val status = attribute("status", F.enumFmt(JobStatus), "Status of the job", JobStatus.InProgress)
   val artifactId = attribute("artifactId", F.stringFmt, "Original artifact on which this job was executed", O.readonly)
   val startDate = attribute("startDate", F.dateFmt, "Timestamp of the job start") // , O.model)
@@ -41,10 +41,27 @@ class JobModel @Inject() (artifactModel: ArtifactModel) extends ChildModelDef[Jo
       .setIfAbsent("startDate", new Date)
   }
 }
-class Job(model: JobModel, attributes: JsObject) extends EntityDef[JobModel, Job](model, attributes) with JobAttributes {
+
+object Job {
+  def fixJobAttr(attr: JsObject): JsObject = {
+    val analyzerId = (attr \ "analyzerId").as[String]
+    val attrWithAnalyzerName = (attr \ "analyzerName").asOpt[String].fold(attr + ("analyzerName" -> JsString(analyzerId)))(_ ⇒ attr)
+    (attr \ "analyzerDefinition").asOpt[String].fold(attrWithAnalyzerName + ("analyzerDefinition" -> JsString(analyzerId)))(_ ⇒ attrWithAnalyzerName)
+  }
+}
+
+class Job(model: JobModel, attributes: JsObject) extends EntityDef[JobModel, Job](model, Job.fixJobAttr(attributes)) with JobAttributes {
   override def toJson = super.toJson + ("report" → report().fold[JsValue](JsObject.empty)(r ⇒ Json.parse(r))) // FIXME is parse fails (invalid report)
 }
 
-case class CortexJob(id: String, analyzerId: String, artifact: CortexArtifact, date: Date, status: JobStatus.Type, cortexIds: List[String] = Nil) {
+case class CortexJob(
+    id: String,
+    analyzerId: String,
+    analyzerName: String,
+    analyzerDefinition: String,
+    artifact: CortexArtifact,
+    date: Date,
+    status: JobStatus.Type,
+    cortexIds: List[String] = Nil) {
   def onCortex(cortexId: String) = copy(cortexIds = cortexId :: cortexIds)
 }

thehive-cortex/app/connectors/cortex/models/JsonFormat.scala

@@ -19,10 +19,12 @@ object JsonFormat {
     for {
       name ← (json \ "name").validate[String]
       version ← (json \ "version").validate[String]
-      id = (json \ "id").asOpt[String].getOrElse((name + "_" + version).replaceAll("\\.", "_"))
+      definition = (name + "_" + version).replaceAll("\\.", "_")
+      id = (json \ "id").asOpt[String].getOrElse(definition)
+      renamed = if (id == definition) definition else name
       description ← (json \ "description").validate[String]
       dataTypeList ← (json \ "dataTypeList").validate[Seq[String]]
-    } yield Analyzer(id, name, version, description, dataTypeList))
+    } yield Analyzer(id, renamed, version, description, dataTypeList))
   implicit val analyzerFormats: Format[Analyzer] = Format(analyzerReads, analyzerWrites)
 
   private val fileArtifactWrites = OWrites[FileArtifact](fileArtifact ⇒ Json.obj(
@@ -51,10 +53,12 @@ object JsonFormat {
     JsObject(attributes.flatMap(a ⇒ (json \ a).asOpt[JsValue].map(a -> _)))
   }
 
-  private val cortexJobReads = Reads[CortexJob](json ⇒
+  implicit val cortexJobReads = Reads[CortexJob](json ⇒
     for {
       id ← (json \ "id").validate[String]
       analyzerId ← (json \ "analyzerId").validate[String]
+      analyzerName = (json \ "analyzerName").validate[String].getOrElse(analyzerId)
+      analyzerDefinition = (json \ "analyzerDefinitionId").validate[String].getOrElse(analyzerId)
       attributes = filterObject(json.as[JsObject], "tlp", "message", "parameters")
       artifact = (json \ "artifact").validate[CortexArtifact]
         .getOrElse {
@@ -63,9 +67,7 @@ object JsonFormat {
         }
       date ← (json \ "date").validate[Date]
       status ← (json \ "status").validate[JobStatus.Type]
-    } yield CortexJob(id, analyzerId, artifact, date, status, Nil))
+    } yield CortexJob(id, analyzerId, analyzerName, analyzerDefinition, artifact, date, status, Nil))
 
-  private val cortexJobWrites = Json.writes[CortexJob]
-  implicit val cortexJobFormat: Format[CortexJob] = Format(cortexJobReads, cortexJobWrites)
   implicit val reportTypeFormat: Format[ReportType.Type] = enumFormat(ReportType)
 }

thehive-cortex/app/connectors/cortex/services/CortexClient.scala

@@ -16,6 +16,7 @@ import connectors.cortex.models.{ Analyzer, CortexArtifact, DataArtifact, FileAr
 import models.HealthStatus
 import services.CustomWSAPI
 
+import org.elastic4play.NotFoundError
 import org.elastic4play.utils.RichFuture
 
 object CortexAuthentication {
@@ -54,6 +55,23 @@ class CortexClient(val name: String, baseUrl: String, authentication: Option[Cor
 
   def getAnalyzer(analyzerId: String)(implicit ec: ExecutionContext): Future[Analyzer] = {
     request(s"api/analyzer/$analyzerId", _.get, _.json.as[Analyzer]).map(_.copy(cortexIds = List(name)))
+      .recoverWith { case _ ⇒ getAnalyzerByName(analyzerId) } // if get analyzer using cortex2 API fails, try using legacy API
+  }
+
+  def getAnalyzerByName(analyzerName: String)(implicit ec: ExecutionContext): Future[Analyzer] = {
+    val searchRequest = Json.obj(
+      "query" -> Json.obj(
+        "_field" -> "name",
+        "_value" -> analyzerName),
+      "range" -> "0-1")
+    request(s"api/analyzer/_search", _.post(searchRequest),
+      _.json.as[Seq[Analyzer]])
+      .flatMap { analyzers ⇒
+        analyzers.headOption
+          .fold[Future[Analyzer]](Future.failed(NotFoundError(s"analyzer $analyzerName not found"))) { analyzer ⇒
+            Future.successful(analyzer.copy(cortexIds = List(name)))
+          }
+      }
   }
 
   def listAnalyzer(implicit ec: ExecutionContext): Future[Seq[Analyzer]] = {

thehive-cortex/app/connectors/cortex/services/CortexSrv.scala

@@ -268,19 +268,25 @@ class CortexSrv @Inject() (
     ()
   }
 
-  def submitJob(cortexId: Option[String], analyzerId: String, artifactId: String)(implicit authContext: AuthContext): Future[Job] = {
-    val cortexClient = cortexId match {
-      case Some(id) ⇒ Future.successful(cortexConfig.instances.find(_.name == id))
-      case None ⇒ if (cortexConfig.instances.lengthCompare(1) <= 0) Future.successful(cortexConfig.instances.headOption)
-      else {
-        Future // If there are several cortex, select the first which has the analyzer
-          .traverse(cortexConfig.instances)(c ⇒ c.getAnalyzer(analyzerId).map(_ ⇒ Some(c)).recover { case _ ⇒ None })
-          .map(_.flatten.headOption)
-      }
+  def submitJob(cortexId: Option[String], analyzerName: String, artifactId: String)(implicit authContext: AuthContext): Future[Job] = {
+    val cortexClientAnalyzer = cortexId match {
+      case Some(id) ⇒
+        cortexConfig
+          .instances
+          .find(_.name == id)
+          .fold[Future[(CortexClient, Analyzer)]](Future.failed(NotFoundError(s"cortex $id not found"))) { c ⇒
+            c.getAnalyzer(analyzerName)
+              .map(c -> _)
+          }
+
+      case None ⇒
+        Future.firstCompletedOf {
+          cortexConfig.instances.map(c ⇒ c.getAnalyzer(analyzerName).map(c -> _))
+        }
     }
 
-    cortexClient.flatMap {
-      case Some(cortex) ⇒
+    cortexClientAnalyzer.flatMap {
+      case (cortex, analyzer) ⇒
         for {
           artifact ← artifactSrv.get(artifactId)
           artifactAttributes = Json.obj(
@@ -291,16 +297,17 @@ class CortexSrv @Inject() (
             case (None, Some(attachment)) ⇒ FileArtifact(attachmentSrv.source(attachment.id), artifactAttributes + ("attachment" → Json.toJson(attachment)))
             case _                        ⇒ throw InternalError(s"Artifact has invalid data : ${artifact.attributes}")
           }
-          cortexJobJson ← cortex.analyze(analyzerId, cortexArtifact)
+          cortexJobJson ← cortex.analyze(analyzer.id, cortexArtifact)
           cortexJob = cortexJobJson.as[CortexJob]
           job ← create(artifact, Fields.empty
             .set("analyzerId", cortexJob.analyzerId)
+            .set("analyzerName", cortexJob.analyzerName)
+            .set("analyzerDefinition", cortexJob.analyzerDefinition)
             .set("artifactId", artifactId)
             .set("cortexId", cortex.name)
             .set("cortexJobId", cortexJob.id))
           _ = updateJobWithCortex(job.id, cortexJob.id, cortex)
         } yield job
-      case None ⇒ Future.failed(NotFoundError(s"Cortex $cortexId not found"))
     }
   }
 }

ui/Gruntfile.js

@@ -114,7 +114,7 @@ module.exports = function(grunt) {
             },
             test: {
                 options: {
-                    port: 9001,
+                    port: 9000,
                     middleware: function(connect) {
                         return [
                             connect.static('.tmp'),

ui/app/scripts/controllers/case/CaseObservablesItemCtrl.js

@@ -79,15 +79,15 @@
             $scope.onJobsChange = function (updates) {
                 $scope.analyzerJobs = {};
 
-                _.each(_.keys($scope.analyzers).sort(), function(analyzerId) {
-                    $scope.analyzerJobs[analyzerId] = [];
+                _.each(_.keys($scope.analyzers).sort(), function(analyzerName) {
+                    $scope.analyzerJobs[analyzerName] = [];
                 });
 
                 angular.forEach($scope.jobs.values, function (job) {
-                    if (job.analyzerId in $scope.analyzerJobs) {
-                        $scope.analyzerJobs[job.analyzerId].push(job);
+                    if (job.analyzerName in $scope.analyzerJobs) {
+                        $scope.analyzerJobs[job.analyzerName].push(job);
                     } else {
-                        $scope.analyzerJobs[job.analyzerId] = [job];
+                        $scope.analyzerJobs[job.analyzerName] = [job];
                     }
                 });
 
@@ -117,7 +117,7 @@
                 CortexSrv.getJob(jobId).then(function(response) {
                     var job = response.data;
                     $scope.report = {
-                        template: job.analyzerId,
+                        template: job.analyzerDefinition,
                         content: job.report,
                         status: job.status,
                         startDate: job.startDate,
@@ -181,19 +181,19 @@
                 });
             };
 
-            $scope.runAnalyzer = function (analyzerId, serverId) {
+            $scope.runAnalyzer = function (analyzerName, serverId) {
                 var artifactName = $scope.artifact.data || $scope.artifact.attachment.name;
 
-                var promise = serverId ? $q.resolve(serverId) : CortexSrv.getServers([analyzerId])
+                var promise = serverId ? $q.resolve(serverId) : CortexSrv.getServers([analyzerName])
 
                 promise.then(function (serverId) {
-                        return $scope._runAnalyzer(serverId, analyzerId, $scope.artifact.id);
+                        return $scope._runAnalyzer(serverId, analyzerName, $scope.artifact.id);
                     })
                     .then(function () {
-                        NotificationSrv.log('Analyzer ' + analyzerId + ' has been successfully started for observable: ' + artifactName, 'success');
+                        NotificationSrv.log('Analyzer ' + analyzerName + ' has been successfully started for observable: ' + artifactName, 'success');
                     }, function (response) {
                         if (response && response.status) {
-                            NotificationSrv.log('Unable to run analyzer ' + analyzerId + ' for observable: ' + artifactName, 'error');
+                            NotificationSrv.log('Unable to run analyzer ' + analyzerName + ' for observable: ' + artifactName, 'error');
                         }
                     });
             };

ui/app/scripts/services/AnalyzerSrv.js

@@ -32,7 +32,7 @@
 
                             analyzers = _.indexBy(_.map(response, function(item) {
                                 return item.toJSON();
-                            }), 'id');
+                            }), 'name');
 
                             deferred.resolve(analyzers);
                         }, function (rejection) {

ui/app/views/partials/observables/details/artifact-details-analysers.html

@@ -13,13 +13,18 @@ <h3 class="pv-xxs pr-xxs text-primary">
                 <th width="60" ng-if="analysisEnabled">Action</th>
             </thead>
             <tbody>
-                <tr ng-repeat="(analyzerId, jobs) in analyzerJobs"
-                    ng-init="analyzer=analyzers[analyzerId]; analyzers[analyzerId].showRows=false;">
+                <tr ng-repeat="(analyzerName, jobs) in analyzerJobs"
+                    ng-init="analyzer=analyzers[analyzerName]; analyzers[analyzerName].showRows=false;">
                     <td>
-                        <a ng-if="jobs.length > 1" class="noline mr-xxs" href ng-click="analyzers[analyzerId].showRows = !analyzers[analyzerId].showRows">
-                            <i class="fa" ng-class="{ true:'fa-minus-square-o', false:'fa-plus-square-o' }[analyzers[analyzerId].showRows]"></i>
+                        <a ng-if="jobs.length > 1" class="noline mr-xxs" href ng-click="analyzers[analyzerName].showRows = !analyzers[analyzerName].showRows">
+                            <i class="fa" ng-class="{ true:'fa-minus-square-o', false:'fa-plus-square-o' }[analyzers[analyzerName].showRows]"></i>
                         </a>
-                        <span uib-tooltip="{{analyzer.description}}">{{analyzer.name ? (analyzer.name + ' ' + analyzer.version) : analyzerId}}</span>
+                        <span uib-tooltip="{{analyzer.description}}">{{analyzer.name || jobs[0].analyzerName}}</span>
+                        <!--pre>
+                          analyzer = {{analyzer}}
+                          analyzers = {{analyzers}}
+                          jobs = {{jobs}}
+                        </pre-->
                         <!-- <div class="text-muted">{{}}</div> -->
                     </td>
                     <!-- <td>
@@ -41,7 +46,7 @@ <h3 class="pv-xxs pr-xxs text-primary">
                     </td>
                     <td ng-if="analysisEnabled">
                         <span class="btn btn-xs" ng-class="{true: 'btn-warning', false: 'btn-danger'}[jobs.length > 0]"
-                            ng-click="runAnalyzer(analyzerId)"
+                            ng-click="runAnalyzer(analyzerName)"
                             ng-if="analyzer.cortexIds.length === 1 && analyzer.active">
                             <i class="glyphicon" ng-class="{true: 'glyphicon-repeat', false: 'glyphicon-fire'}[jobs.length > 0]"></i>
                         </span>
@@ -57,7 +62,7 @@ <h3 class="pv-xxs pr-xxs text-primary">
                             </button>
                             <ul class="dropdown-menu" uib-dropdown-menu>
                                 <li ng-repeat="srv in cortexServers">
-                                    <a href ng-click="runAnalyzer(analyzerId, srv.name)" ng-disabled="srv.status === 'ERROR'">
+                                    <a href ng-click="runAnalyzer(analyzerName, srv.name)" ng-disabled="srv.status === 'ERROR'">
                                         <div>
                                             <strong>{{srv.name}}</strong>
                                         </div>