diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5d526a8d46..010b5378f7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,6 +20,6 @@ object Dependencies { val reflections = "org.reflections" % "reflections" % "0.9.11" val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2" - val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.4" + val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.5-SNAPSHOT" } } diff --git a/thehive-backend/app/services/StreamSrv.scala b/thehive-backend/app/services/StreamSrv.scala index c38ef7e597..9a671f6a72 100644 --- a/thehive-backend/app/services/StreamSrv.scala +++ b/thehive-backend/app/services/StreamSrv.scala @@ -198,7 +198,6 @@ class StreamActor( case Initialize(requestId) ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None))) case _: AuditOperation ⇒ - case message ⇒ logger.warn(s"Unexpected message $message (${message.getClass})") } def receive: Receive = receiveWithState(None, Map.empty[String, Option[StreamMessageGroup[_]]]) diff --git a/thehive-cortex/app/connectors/cortex/models/Job.scala b/thehive-cortex/app/connectors/cortex/models/Job.scala index 87eed3a4da..1a8e47c80c 100644 --- a/thehive-cortex/app/connectors/cortex/models/Job.scala +++ b/thehive-cortex/app/connectors/cortex/models/Job.scala @@ -1,17 +1,15 @@ package connectors.cortex.models import java.util.Date - import javax.inject.{ Inject, Singleton } import scala.concurrent.Future -import play.api.libs.json.{ JsObject, JsValue, Json } +import play.api.libs.json.{ JsObject, JsString, JsValue, Json } import org.elastic4play.JsonFormat.dateFormat -import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, AttributeOption ⇒ O, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration } +import org.elastic4play.models.{ AttributeDef, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F, AttributeOption ⇒ O } import org.elastic4play.utils.RichJson - import connectors.cortex.models.JsonFormat.jobStatusFormat import models.{ Artifact, ArtifactModel } import services.AuditedModel @@ -23,6 +21,8 @@ object JobStatus extends Enumeration with HiveEnumeration { trait JobAttributes { _: AttributeDef ⇒ val analyzerId = attribute("analyzerId", F.stringFmt, "Analyzer", O.readonly) + val analyzerName = optionalAttribute("analyzerName", F.stringFmt, "Name of the analyzer", O.readonly) + val analyzerDefinition = optionalAttribute("analyzerDefinition", F.stringFmt, "Name of the analyzer definition", O.readonly) val status = attribute("status", F.enumFmt(JobStatus), "Status of the job", JobStatus.InProgress) val artifactId = attribute("artifactId", F.stringFmt, "Original artifact on which this job was executed", O.readonly) val startDate = attribute("startDate", F.dateFmt, "Timestamp of the job start") // , O.model) @@ -41,10 +41,27 @@ class JobModel @Inject() (artifactModel: ArtifactModel) extends ChildModelDef[Jo .setIfAbsent("startDate", new Date) } } -class Job(model: JobModel, attributes: JsObject) extends EntityDef[JobModel, Job](model, attributes) with JobAttributes { + +object Job { + def fixJobAttr(attr: JsObject): JsObject = { + val analyzerId = (attr \ "analyzerId").as[String] + val attrWithAnalyzerName = (attr \ "analyzerName").asOpt[String].fold(attr + ("analyzerName" -> JsString(analyzerId)))(_ ⇒ attr) + (attr \ "analyzerDefinition").asOpt[String].fold(attrWithAnalyzerName + ("analyzerDefinition" -> JsString(analyzerId)))(_ ⇒ attrWithAnalyzerName) + } +} + +class Job(model: JobModel, attributes: JsObject) extends EntityDef[JobModel, Job](model, Job.fixJobAttr(attributes)) with JobAttributes { override def toJson = super.toJson + ("report" → report().fold[JsValue](JsObject.empty)(r ⇒ Json.parse(r))) // FIXME is parse fails (invalid report) } -case class CortexJob(id: String, analyzerId: String, artifact: CortexArtifact, date: Date, status: JobStatus.Type, cortexIds: List[String] = Nil) { +case class CortexJob( + id: String, + analyzerId: String, + analyzerName: String, + analyzerDefinition: String, + artifact: CortexArtifact, + date: Date, + status: JobStatus.Type, + cortexIds: List[String] = Nil) { def onCortex(cortexId: String) = copy(cortexIds = cortexId :: cortexIds) } \ No newline at end of file diff --git a/thehive-cortex/app/connectors/cortex/models/JsonFormat.scala b/thehive-cortex/app/connectors/cortex/models/JsonFormat.scala index e4cff42ffa..cec56ac151 100644 --- a/thehive-cortex/app/connectors/cortex/models/JsonFormat.scala +++ b/thehive-cortex/app/connectors/cortex/models/JsonFormat.scala @@ -19,10 +19,12 @@ object JsonFormat { for { name ← (json \ "name").validate[String] version ← (json \ "version").validate[String] - id = (json \ "id").asOpt[String].getOrElse((name + "_" + version).replaceAll("\\.", "_")) + definition = (name + "_" + version).replaceAll("\\.", "_") + id = (json \ "id").asOpt[String].getOrElse(definition) + renamed = if (id == definition) definition else name description ← (json \ "description").validate[String] dataTypeList ← (json \ "dataTypeList").validate[Seq[String]] - } yield Analyzer(id, name, version, description, dataTypeList)) + } yield Analyzer(id, renamed, version, description, dataTypeList)) implicit val analyzerFormats: Format[Analyzer] = Format(analyzerReads, analyzerWrites) private val fileArtifactWrites = OWrites[FileArtifact](fileArtifact ⇒ Json.obj( @@ -51,10 +53,12 @@ object JsonFormat { JsObject(attributes.flatMap(a ⇒ (json \ a).asOpt[JsValue].map(a -> _))) } - private val cortexJobReads = Reads[CortexJob](json ⇒ + implicit val cortexJobReads = Reads[CortexJob](json ⇒ for { id ← (json \ "id").validate[String] analyzerId ← (json \ "analyzerId").validate[String] + analyzerName = (json \ "analyzerName").validate[String].getOrElse(analyzerId) + analyzerDefinition = (json \ "analyzerDefinitionId").validate[String].getOrElse(analyzerId) attributes = filterObject(json.as[JsObject], "tlp", "message", "parameters") artifact = (json \ "artifact").validate[CortexArtifact] .getOrElse { @@ -63,9 +67,7 @@ object JsonFormat { } date ← (json \ "date").validate[Date] status ← (json \ "status").validate[JobStatus.Type] - } yield CortexJob(id, analyzerId, artifact, date, status, Nil)) + } yield CortexJob(id, analyzerId, analyzerName, analyzerDefinition, artifact, date, status, Nil)) - private val cortexJobWrites = Json.writes[CortexJob] - implicit val cortexJobFormat: Format[CortexJob] = Format(cortexJobReads, cortexJobWrites) implicit val reportTypeFormat: Format[ReportType.Type] = enumFormat(ReportType) } \ No newline at end of file diff --git a/thehive-cortex/app/connectors/cortex/services/CortexClient.scala b/thehive-cortex/app/connectors/cortex/services/CortexClient.scala index 477b014b3a..68d9c70d34 100644 --- a/thehive-cortex/app/connectors/cortex/services/CortexClient.scala +++ b/thehive-cortex/app/connectors/cortex/services/CortexClient.scala @@ -16,6 +16,7 @@ import connectors.cortex.models.{ Analyzer, CortexArtifact, DataArtifact, FileAr import models.HealthStatus import services.CustomWSAPI +import org.elastic4play.NotFoundError import org.elastic4play.utils.RichFuture object CortexAuthentication { @@ -54,6 +55,23 @@ class CortexClient(val name: String, baseUrl: String, authentication: Option[Cor def getAnalyzer(analyzerId: String)(implicit ec: ExecutionContext): Future[Analyzer] = { request(s"api/analyzer/$analyzerId", _.get, _.json.as[Analyzer]).map(_.copy(cortexIds = List(name))) + .recoverWith { case _ ⇒ getAnalyzerByName(analyzerId) } // if get analyzer using cortex2 API fails, try using legacy API + } + + def getAnalyzerByName(analyzerName: String)(implicit ec: ExecutionContext): Future[Analyzer] = { + val searchRequest = Json.obj( + "query" -> Json.obj( + "_field" -> "name", + "_value" -> analyzerName), + "range" -> "0-1") + request(s"api/analyzer/_search", _.post(searchRequest), + _.json.as[Seq[Analyzer]]) + .flatMap { analyzers ⇒ + analyzers.headOption + .fold[Future[Analyzer]](Future.failed(NotFoundError(s"analyzer $analyzerName not found"))) { analyzer ⇒ + Future.successful(analyzer.copy(cortexIds = List(name))) + } + } } def listAnalyzer(implicit ec: ExecutionContext): Future[Seq[Analyzer]] = { diff --git a/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala b/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala index 23a1eebe85..d09a7fbd37 100644 --- a/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala +++ b/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala @@ -268,19 +268,25 @@ class CortexSrv @Inject() ( () } - def submitJob(cortexId: Option[String], analyzerId: String, artifactId: String)(implicit authContext: AuthContext): Future[Job] = { - val cortexClient = cortexId match { - case Some(id) ⇒ Future.successful(cortexConfig.instances.find(_.name == id)) - case None ⇒ if (cortexConfig.instances.lengthCompare(1) <= 0) Future.successful(cortexConfig.instances.headOption) - else { - Future // If there are several cortex, select the first which has the analyzer - .traverse(cortexConfig.instances)(c ⇒ c.getAnalyzer(analyzerId).map(_ ⇒ Some(c)).recover { case _ ⇒ None }) - .map(_.flatten.headOption) - } + def submitJob(cortexId: Option[String], analyzerName: String, artifactId: String)(implicit authContext: AuthContext): Future[Job] = { + val cortexClientAnalyzer = cortexId match { + case Some(id) ⇒ + cortexConfig + .instances + .find(_.name == id) + .fold[Future[(CortexClient, Analyzer)]](Future.failed(NotFoundError(s"cortex $id not found"))) { c ⇒ + c.getAnalyzer(analyzerName) + .map(c -> _) + } + + case None ⇒ + Future.firstCompletedOf { + cortexConfig.instances.map(c ⇒ c.getAnalyzer(analyzerName).map(c -> _)) + } } - cortexClient.flatMap { - case Some(cortex) ⇒ + cortexClientAnalyzer.flatMap { + case (cortex, analyzer) ⇒ for { artifact ← artifactSrv.get(artifactId) artifactAttributes = Json.obj( @@ -291,16 +297,17 @@ class CortexSrv @Inject() ( case (None, Some(attachment)) ⇒ FileArtifact(attachmentSrv.source(attachment.id), artifactAttributes + ("attachment" → Json.toJson(attachment))) case _ ⇒ throw InternalError(s"Artifact has invalid data : ${artifact.attributes}") } - cortexJobJson ← cortex.analyze(analyzerId, cortexArtifact) + cortexJobJson ← cortex.analyze(analyzer.id, cortexArtifact) cortexJob = cortexJobJson.as[CortexJob] job ← create(artifact, Fields.empty .set("analyzerId", cortexJob.analyzerId) + .set("analyzerName", cortexJob.analyzerName) + .set("analyzerDefinition", cortexJob.analyzerDefinition) .set("artifactId", artifactId) .set("cortexId", cortex.name) .set("cortexJobId", cortexJob.id)) _ = updateJobWithCortex(job.id, cortexJob.id, cortex) } yield job - case None ⇒ Future.failed(NotFoundError(s"Cortex $cortexId not found")) } } } diff --git a/ui/Gruntfile.js b/ui/Gruntfile.js index 72539ce0ae..d845e12104 100644 --- a/ui/Gruntfile.js +++ b/ui/Gruntfile.js @@ -114,7 +114,7 @@ module.exports = function(grunt) { }, test: { options: { - port: 9001, + port: 9000, middleware: function(connect) { return [ connect.static('.tmp'), diff --git a/ui/app/scripts/controllers/case/CaseObservablesItemCtrl.js b/ui/app/scripts/controllers/case/CaseObservablesItemCtrl.js index b4f2ad3076..b6e18ee125 100644 --- a/ui/app/scripts/controllers/case/CaseObservablesItemCtrl.js +++ b/ui/app/scripts/controllers/case/CaseObservablesItemCtrl.js @@ -79,15 +79,15 @@ $scope.onJobsChange = function (updates) { $scope.analyzerJobs = {}; - _.each(_.keys($scope.analyzers).sort(), function(analyzerId) { - $scope.analyzerJobs[analyzerId] = []; + _.each(_.keys($scope.analyzers).sort(), function(analyzerName) { + $scope.analyzerJobs[analyzerName] = []; }); angular.forEach($scope.jobs.values, function (job) { - if (job.analyzerId in $scope.analyzerJobs) { - $scope.analyzerJobs[job.analyzerId].push(job); + if (job.analyzerName in $scope.analyzerJobs) { + $scope.analyzerJobs[job.analyzerName].push(job); } else { - $scope.analyzerJobs[job.analyzerId] = [job]; + $scope.analyzerJobs[job.analyzerName] = [job]; } }); @@ -117,7 +117,7 @@ CortexSrv.getJob(jobId).then(function(response) { var job = response.data; $scope.report = { - template: job.analyzerId, + template: job.analyzerDefinition, content: job.report, status: job.status, startDate: job.startDate, @@ -181,19 +181,19 @@ }); }; - $scope.runAnalyzer = function (analyzerId, serverId) { + $scope.runAnalyzer = function (analyzerName, serverId) { var artifactName = $scope.artifact.data || $scope.artifact.attachment.name; - var promise = serverId ? $q.resolve(serverId) : CortexSrv.getServers([analyzerId]) + var promise = serverId ? $q.resolve(serverId) : CortexSrv.getServers([analyzerName]) promise.then(function (serverId) { - return $scope._runAnalyzer(serverId, analyzerId, $scope.artifact.id); + return $scope._runAnalyzer(serverId, analyzerName, $scope.artifact.id); }) .then(function () { - NotificationSrv.log('Analyzer ' + analyzerId + ' has been successfully started for observable: ' + artifactName, 'success'); + NotificationSrv.log('Analyzer ' + analyzerName + ' has been successfully started for observable: ' + artifactName, 'success'); }, function (response) { if (response && response.status) { - NotificationSrv.log('Unable to run analyzer ' + analyzerId + ' for observable: ' + artifactName, 'error'); + NotificationSrv.log('Unable to run analyzer ' + analyzerName + ' for observable: ' + artifactName, 'error'); } }); }; diff --git a/ui/app/scripts/services/AnalyzerSrv.js b/ui/app/scripts/services/AnalyzerSrv.js index 0b9405c439..f7ffea98bd 100644 --- a/ui/app/scripts/services/AnalyzerSrv.js +++ b/ui/app/scripts/services/AnalyzerSrv.js @@ -32,7 +32,7 @@ analyzers = _.indexBy(_.map(response, function(item) { return item.toJSON(); - }), 'id'); + }), 'name'); deferred.resolve(analyzers); }, function (rejection) { diff --git a/ui/app/views/partials/observables/details/artifact-details-analysers.html b/ui/app/views/partials/observables/details/artifact-details-analysers.html index 84a3370e52..57c2eae2cf 100644 --- a/ui/app/views/partials/observables/details/artifact-details-analysers.html +++ b/ui/app/views/partials/observables/details/artifact-details-analysers.html @@ -13,13 +13,18 @@

Action - + - - + + - {{analyzer.name ? (analyzer.name + ' ' + analyzer.version) : analyzerId}} + {{analyzer.name || jobs[0].analyzerName}} +