#53 Add cortex connector implementation

thehive-cortex/app/connectors/cortex/CortexConnector.scala

@@ -1,7 +1,5 @@
 package connectors.cortex
 
-import javax.inject.Singleton
-
 import play.api.{ Configuration, Environment, Logger }
 
 import connectors.ConnectorModule

thehive-cortex/app/connectors/cortex/CortextCtrl.scala

@@ -1,26 +1,23 @@
 package connectors.cortex
 
-import scala.concurrent.ExecutionContext
+import javax.inject.{ Inject, Singleton }
 
-import org.elastic4play.BadRequestError
-import org.elastic4play.NotFoundError
-import org.elastic4play.Timed
-import org.elastic4play.controllers.Authenticated
-import org.elastic4play.controllers.FieldsBodyParser
-import org.elastic4play.controllers.Renderer
-import org.elastic4play.services.Role
+import scala.concurrent.ExecutionContext
 
-import connectors.cortex.models.JsonFormat._
-import connectors.Connector
-import javax.inject.Inject
-import javax.inject.Singleton
 import play.api.Logger
 import play.api.http.Status
 import play.api.mvc.Controller
 import play.api.routing.SimpleRouter
-import play.api.routing.sird.GET
-import play.api.routing.sird.POST
-import play.api.routing.sird.UrlContext
+import play.api.routing.sird.{ GET, POST, UrlContext }
+
+import org.elastic4play.{ BadRequestError, NotFoundError, Timed }
+import org.elastic4play.controllers.{ Authenticated, FieldsBodyParser, Renderer }
+import org.elastic4play.models.JsonFormat.baseModelEntityWrites
+import org.elastic4play.services.{ QueryDef, QueryDSL, Role }
+import org.elastic4play.services.JsonFormat.queryReads
+
+import connectors.Connector
+import connectors.cortex.models.JsonFormat.{ analyzerFormats, cortexJobFormat }
 import connectors.cortex.services.CortexSrv
 
 @Singleton
@@ -35,7 +32,7 @@ class CortextCtrl @Inject() (
   val router = SimpleRouter {
     case POST(p"/job")                           ⇒ createJob
     case GET(p"/job/$jobId<[^/]*>")              ⇒ getJob(jobId)
-    case GET(p"/job")                            ⇒ listJob
+    case POST(p"/job/_search")                   ⇒ findJob
     case GET(p"/analyzer/$analyzerId<[^/]*>")    ⇒ getAnalyzer(analyzerId)
     case GET(p"/analyzer/type/$dataType<[^/]*>") ⇒ getAnalyzerFor(dataType)
     case GET(p"/analyzer")                       ⇒ listAnalyzer
@@ -46,7 +43,8 @@ class CortextCtrl @Inject() (
   def createJob = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒
     val analyzerId = request.body.getString("analyzerId").getOrElse(throw BadRequestError(s"analyzerId is missing"))
     val artifactId = request.body.getString("artifactId").getOrElse(throw BadRequestError(s"artifactId is missing"))
-    cortexSrv.createJob(analyzerId, artifactId).map { job ⇒
+    val cortexId = request.body.getString("cortexId").getOrElse(throw BadRequestError(s"cortexId is missing"))
+    cortexSrv.submitJob(cortexId, analyzerId, artifactId).map { job ⇒
       renderer.toOutput(OK, job)
     }
   }
@@ -59,10 +57,13 @@ class CortextCtrl @Inject() (
   }
 
   @Timed
-  def listJob = authenticated(Role.read).async { implicit request ⇒
-    cortexSrv.listJob.map { jobs ⇒
-      renderer.toOutput(OK, jobs)
-    }
+  def findJob = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒
+    val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
+    val range = request.body.getString("range")
+    val sort = request.body.getStrings("sort").getOrElse(Nil)
+
+    val (jobs, total) = cortexSrv.find(query, range, sort)
+    renderer.toOutput(OK, jobs, total)
   }
 
   @Timed
@@ -85,14 +86,4 @@ class CortextCtrl @Inject() (
       renderer.toOutput(OK, analyzers)
     }
   }
-
-  //*  POST     /api/case/artifact/:artifactId/job         controllers.JobCtrl.create(artifactId)
-  //POST     /api/case/artifact/job/_stats              controllers.JobCtrl.stats()
-  //POST     /api/case/artifact/job/_search             controllers.JobCtrl.find()
-  //GET      /api/case/artifact/:artifactId/job         controllers.JobCtrl.findInArtifact(artifactId)
-  //GET      /api/case/artifact/job/:jobId              controllers.JobCtrl.get(jobId)
-  //POST     /api/analyzer/_search                      controllers.AnalyzerCtrl.find()
-  //GET      /api/analyzer/:analyzerId                  controllers.AnalyzerCtrl.get(analyzerId)
-  //GET      /api/analyzer/:analyzerId/report/:flavor   controllers.AnalyzerCtrl.getReport(analyzerId, flavor)
-
 }

thehive-cortex/app/connectors/cortex/models/Analyzer.scala

@@ -1,7 +1,5 @@
 package connectors.cortex.models
 
-import connectors.cortex.services.CortexClient
-
 trait CortexModel[O] { self ⇒
   def onCortex(cortexId: String): O
 }

thehive-cortex/app/connectors/cortex/models/Artifact.scala

@@ -1,9 +1,11 @@
 package connectors.cortex.models
 
-import java.io.File
+import akka.NotUsed
+import akka.stream.scaladsl.Source
+import akka.util.ByteString
 
 import play.api.libs.json.JsObject
 
 sealed abstract class CortexArtifact(attributes: JsObject)
-case class FileArtifact(data: File, attributes: JsObject) extends CortexArtifact(attributes)
+case class FileArtifact(data: Source[ByteString, NotUsed], attributes: JsObject) extends CortexArtifact(attributes)
 case class DataArtifact(data: String, attributes: JsObject) extends CortexArtifact(attributes)

thehive-cortex/app/connectors/cortex/models/Job.scala

@@ -12,9 +12,8 @@ import org.elastic4play.JsonFormat.dateFormat
 import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, AttributeOption ⇒ O, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration }
 
 import JsonFormat.jobStatusFormat
-import models.ArtifactModel
+import models.{ Artifact, ArtifactModel }
 import services.AuditedModel
-import models.Artifact
 
 object JobStatus extends Enumeration with HiveEnumeration {
   type Type = Value
@@ -29,6 +28,7 @@ trait JobAttributes { _: AttributeDef ⇒
   val endDate = optionalAttribute("endDate", F.dateFmt, "Timestamp of the job completion (or fail)")
   val report = optionalAttribute("report", F.textFmt, "Analysis result", O.unaudited)
   val cortexId = optionalAttribute("cortexId", F.stringFmt, "Id of cortex where the job is run", O.readonly)
+  val cortexJobId = optionalAttribute("cortexJobId", F.stringFmt, "Id of job in cortex", O.readonly)
 
 }
 @Singleton
@@ -44,4 +44,6 @@ class Job(model: JobModel, attributes: JsObject) extends EntityDef[JobModel, Job
   override def toJson = super.toJson + ("report" → report().fold[JsValue](JsObject(Nil))(r ⇒ Json.parse(r)))
 }
 
-case class CortexJob(id: String, analyzerId: String, artifact: CortexArtifact, date: Date, status: JobStatus.Type)
+case class CortexJob(id: String, analyzerId: String, artifact: CortexArtifact, date: Date, status: JobStatus.Type, cortexIds: List[String] = Nil) extends CortexModel[CortexJob] {
+  def onCortex(cortexId: String) = copy(cortexIds = cortexId :: cortexIds)
+}

thehive-cortex/app/connectors/cortex/models/JsonFormat.scala

@@ -1,11 +1,12 @@
 package connectors.cortex.models
 
-import java.io.File
+import akka.stream.scaladsl.Source
 
-import play.api.libs.json.{ JsObject, JsString, Json, OFormat, OWrites, Reads, Writes }
+import play.api.libs.json.{ Format, JsObject, Json }
+import play.api.libs.json.{ OFormat, OWrites, Reads, Writes }
 import play.api.libs.json.Json.toJsFieldJsValueWrapper
+
 import org.elastic4play.models.JsonFormat.enumFormat
-import play.api.libs.json.Format
 
 object JsonFormat {
   val analyzerWrites = Writes[Analyzer](analyzer ⇒ Json.obj(
@@ -15,15 +16,21 @@ object JsonFormat {
     "description" → analyzer.description,
     "dataTypeList" → analyzer.dataTypeList,
     "cortexIds" → analyzer.cortexIds))
-  val analyzerReads = Json.reads[Analyzer]
+  val analyzerReads = Reads[Analyzer](json ⇒
+    for {
+      name ← (json \ "name").validate[String]
+      version ← (json \ "version").validate[String]
+      description ← (json \ "description").validate[String]
+      dataTypeList ← (json \ "dataTypeList").validate[Seq[String]]
+    } yield Analyzer(name, version, description, dataTypeList))
   implicit val analyzerFormats = Format(analyzerReads, analyzerWrites)
 
   val fileArtifactWrites = OWrites[FileArtifact](fileArtifact ⇒ Json.obj(
     "attributes" → fileArtifact.attributes))
 
   val fileArtifactReads = Reads[FileArtifact](json ⇒
     (json \ "attributes").validate[JsObject].map { attributes ⇒
-      FileArtifact(new File("dummy"), attributes)
+      FileArtifact(Source.empty, attributes)
     })
   val fileArtifactFormat = OFormat(fileArtifactReads, fileArtifactWrites)
   val dataArtifactFormat = Json.format[DataArtifact]
@@ -39,5 +46,5 @@ object JsonFormat {
 
   implicit val artifactFormat = OFormat(artifactReads, artifactWrites)
   implicit val jobStatusFormat = enumFormat(JobStatus)
-  implicit val jobFormat = Json.format[CortexJob]
+  implicit val cortexJobFormat = Json.format[CortexJob]
 }

thehive-cortex/app/connectors/cortex/services/CortexClient.scala

@@ -1,76 +1,58 @@
 package connectors.cortex.services
 
 import scala.concurrent.{ ExecutionContext, Future }
+import scala.concurrent.duration.Duration
 
-import akka.stream.scaladsl.{ FileIO, Source }
+import akka.stream.scaladsl.Source
 
-import play.api.libs.json.{ JsValue, Json }
+import play.api.libs.json.{ JsObject, Json }
 import play.api.libs.ws.{ WSClient, WSRequest, WSResponse }
 import play.api.mvc.MultipartFormData.{ DataPart, FilePart }
 
-import org.elastic4play.models.JsonFormat._
-import connectors.cortex.models.JsonFormat._
-import connectors.cortex.models.Analyzer
-import connectors.cortex.models.FileArtifact
-import connectors.cortex.models.DataArtifact
-import connectors.cortex.models.Job
-import connectors.cortex.models.CortexArtifact
-import play.api.libs.json.JsObject
 import org.elastic4play.NotFoundError
 
-class CortexClient(name: String, baseUrl: String, key: String) {
-  val fakeAnalyzers = Seq(
-    Analyzer("DNSDB_DomainName", "1.1", "DNSDB Passive DNS query for Domain Names : Provides history records for a domain", Seq("domain"), List(name)),
-    Analyzer("DNSDB_IPHistory", "1.0", "DNSDB Passive DNS query for IP history : Provides history records for an IP", Seq("ip"), List(name)),
-    Analyzer("DNSDB_NameHistory", "1.0", "DNSDB Passive DNS query for domain/host name history : Provides history records for an domain/host", Seq("fqdn"), List(name)),
-    Analyzer("DomainTools_ReverseIP", "1.0", "DomainTools Reverse IP: provides a list of domain names that share the same Internet host", Seq("ip"), List(name)),
-    Analyzer("DomainTools_ReverseNameServer", "1.0", "DomainTools Reverse Name server: provides a list of domain names that share the same primary or secondary name server", Seq("domain"), List(name)),
-    Analyzer("DomainTools_ReverseWhois", "1.0", "Domaintools Reverse Whois lookup : provides a list of domain names that share the same Registrant Information.", Seq("mail", "ip", "domain", "other"), List(name)),
-    Analyzer("DomainTools_WhoisHistory", "1.0", "DomainTools Whois History: provides a list of historic Whois records for a domain name", Seq("domain"), List(name)),
-    Analyzer("DomainTools_WhoisLookup", "1.0", "DomainTools Whois Lookup: provides the ownership record for a domain name with basic registration details", Seq("domain"), List(name)),
-    Analyzer("DomainTools_WhoisLookup_IP", "1.0", "DomainTools Whois Lookup IP: provides the ownership record for a IP address with basic registration details", Seq("ip"), List(name)),
-    Analyzer("Hipposcore", "1.0", "Hippocampe Score report: provides the last report for an IP, domain or a URL", Seq("ip", "domain", "fqdn", "url"), List(name)),
-    Analyzer("HippoMore", "1.0", "Hippocampe detailed report: provides the last detailed report for an IP, domain or a URL", Seq("ip", "domain", "fqdn", "url"), List(name)),
-    Analyzer("MaxMind_GeoIP", "2.0", "MaxMind: Geolocation", Seq("ip"), List(name)),
-    Analyzer("Msg_Parser", "1.0", "Outlook .msg file parser", Seq("file"), List(name)),
-    Analyzer("Olevba_Report", "1.0", "Olevba analysis report. Submit a Microsoft Office File.", Seq("file"), List(name)),
-    Analyzer("URLCategory", "1.0", "URL Category query: checks the category of a specific URL or domain", Seq("url", "domain"), List(name)),
-    Analyzer("VirusTotal_GetReport", "1.0", "VirusTotal get file report: provides the last report of a file. Submit a hash(md5/sha1/sha256)", Nil, List(name)),
-    Analyzer("VirusTotal_GetReport", "2.0", "VirusTotal get report: provides the last report of a file, hash, domain or ip", Seq("file", "hash", "domain", "ip"), List(name)),
-    Analyzer("VirusTotal_Scan", "2.0", "VirusTotal scan file or url", Seq("file", "url"), List(name)),
-    Analyzer("VirusTotal_UrlReport", "1.0", "VirusTotal get url report: provides the last report of a url or site. Submit a url", Nil, List(name)))
+import connectors.cortex.models.{ Analyzer, CortexArtifact, DataArtifact, FileArtifact }
+import connectors.cortex.models.JsonFormat._
+import play.api.Logger
+
+class CortexClient(val name: String, baseUrl: String, key: String) {
+
+  lazy val logger = Logger(getClass)
 
+  logger.info(s"new Cortex($name, $baseUrl, $key)")
   def request[A](uri: String, f: WSRequest ⇒ Future[WSResponse], t: WSResponse ⇒ A)(implicit ws: WSClient, ec: ExecutionContext) = {
-    f(ws.url(baseUrl + "/" + uri).withHeaders("auth" → key)).map {
+    val url = (baseUrl + uri)
+    logger.info(s"Requesting Cortex $url")
+    f(ws.url(url).withHeaders("auth" → key)).map {
       case response if response.status / 100 == 2 ⇒ t(response)
-      case error                                  ⇒ ???
+      case error ⇒
+        logger.error(s"Cortext error on $baseUrl/$uri (${error.status}) \n${error.body}")
+        sys.error("")
     }
   }
 
   def getAnalyzer(analyzerId: String)(implicit ws: WSClient, ec: ExecutionContext): Future[Analyzer] = {
-    //request(s"/api/analyzer/$analyzerId", _.get, _.json.as[Analyzer])
-    fakeAnalyzers.find(_.id == analyzerId).fold[Future[Analyzer]](Future.failed(NotFoundError("")))(a ⇒
-      Future.successful(a))
+    request(s"/api/analyzer/$analyzerId", _.get, _.json.as[Analyzer])
   }
 
   def listAnalyzer(implicit ws: WSClient, ec: ExecutionContext): Future[Seq[Analyzer]] = {
-    //request(s"/api/analyzer", _.get, _.json.as[Seq[Analyzer]])
-    Future.successful(fakeAnalyzers)
+    request(s"/api/analyzer", _.get, _.json.as[Seq[Analyzer]])
   }
 
   def analyze(analyzerId: String, artifact: CortexArtifact)(implicit ws: WSClient, ec: ExecutionContext) = {
     artifact match {
-      case FileArtifact(file, attributes) ⇒
-        val body = Source(FilePart("data", file.getName, None, FileIO.fromPath(file.toPath)) :: DataPart("_json", attributes.toString) :: Nil)
+      case FileArtifact(data, attributes) ⇒
+        val body = Source(List(
+          FilePart("data", (attributes \ "attachment" \ "name").asOpt[String].getOrElse("noname"), None, data),
+          DataPart("_json", attributes.toString)))
         request(s"/api/analyzer/$analyzerId", _.post(body), _.json)
       case a: DataArtifact ⇒
         request(s"/api/analyzer/$analyzerId", _.post(Json.toJson(a)), _.json.as[JsObject])
     }
   }
 
   def listAnalyzerForType(dataType: String)(implicit ws: WSClient, ec: ExecutionContext): Future[Seq[Analyzer]] = {
-    //request(s"/api/analyzer/type/$dataType", _.get, _.json.as[Seq[Analyzer]])
-    Future.successful(fakeAnalyzers.filter(_.dataTypeList.contains(dataType)))
+    request(s"/api/analyzer/type/$dataType", _.get, _.json.as[Seq[Analyzer]])
   }
 
   def listJob(implicit ws: WSClient, ec: ExecutionContext) = {
@@ -89,7 +71,7 @@ class CortexClient(name: String, baseUrl: String, key: String) {
     request(s"/api/job/$jobId/report", _.get, r ⇒ r.json.as[JsObject])
   }
 
-  def waitReport(jobId: String, atMost: String)(implicit ws: WSClient, ec: ExecutionContext) = {
-    request(s"/api/job/$jobId/waitreport", _.get, r ⇒ r.json.as[JsObject])
+  def waitReport(jobId: String, atMost: Duration)(implicit ws: WSClient, ec: ExecutionContext) = {
+    request(s"/api/job/$jobId/waitreport", _.withQueryString("atMost" → atMost.toString).get, r ⇒ r.json.as[JsObject])
   }
 }