-
Notifications
You must be signed in to change notification settings - Fork 640
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add custom HTTP client. It accepts proxy, ssl and timeout configurati…
…on, with stackable config. Use custom HTTP client in MISP connector (#143) and Cortex connector (#147)
- Loading branch information
Showing
4 changed files
with
177 additions
and
159 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| package services | ||
|
|
||
| import javax.inject.Inject | ||
|
|
||
| import akka.stream.Materializer | ||
| import play.api.inject.ApplicationLifecycle | ||
| import play.api.libs.ws.ahc.{ AhcWSAPI, AhcWSClient, AhcWSClientConfig, AhcWSClientConfigParser } | ||
| import play.api.libs.ws.ssl.TrustStoreConfig | ||
| import play.api.libs.ws._ | ||
| import play.api.{ Configuration, Environment, Logger } | ||
|
|
||
| object CustomWSAPI { | ||
| private[CustomWSAPI] lazy val logger = Logger(getClass) | ||
|
|
||
| def parseWSConfig(config: Configuration, environment: Environment): AhcWSClientConfig = { | ||
| new AhcWSClientConfigParser( | ||
| new WSConfigParser(config, environment).parse(), | ||
| config, | ||
| environment).parse() | ||
| } | ||
|
|
||
| def parseProxyConfig(config: Configuration): Option[WSProxyServer] = for { | ||
| proxyConfig ← config.getConfig("play.ws.proxy") | ||
| proxyHost ← proxyConfig.getString("host") | ||
| proxyPort ← proxyConfig.getInt("port") | ||
| proxyProtocol = proxyConfig.getString("protocol") | ||
| proxyPrincipal = proxyConfig.getString("user") | ||
| proxyPassword = proxyConfig.getString("password") | ||
| proxyNtlmDomain = proxyConfig.getString("ntlmDomain") | ||
| proxyEncoding = proxyConfig.getString("encoding") | ||
| proxyNonProxyHosts = proxyConfig.getStringSeq("nonProxyHosts") | ||
| } yield DefaultWSProxyServer(proxyHost, proxyPort, proxyProtocol, proxyPrincipal, proxyPassword, proxyNtlmDomain, proxyEncoding, proxyNonProxyHosts) | ||
|
|
||
| def getWS(config: Configuration, environment: Environment, lifecycle: ApplicationLifecycle, mat: Materializer): AhcWSAPI = { | ||
| val clientConfig = parseWSConfig(config, environment) | ||
| val clientConfigWithTruststore = config.getString("play.cert") match { | ||
| case Some(p) ⇒ | ||
| logger.warn( | ||
| """Use of "cert" parameter in configuration file is deprecated. Please use: | ||
| | ws.ssl { | ||
| | trustManager = { | ||
| | stores = [ | ||
| | { type = "PEM", path = "/path/to/cacert.crt" }, | ||
| | { type = "JKS", path = "/path/to/truststore.jks" } | ||
| | ] | ||
| | } | ||
| | } | ||
| """.stripMargin) | ||
| clientConfig.copy( | ||
| wsClientConfig = clientConfig.wsClientConfig.copy( | ||
| ssl = clientConfig.wsClientConfig.ssl.copy( | ||
| trustManagerConfig = clientConfig.wsClientConfig.ssl.trustManagerConfig.copy( | ||
| trustStoreConfigs = clientConfig.wsClientConfig.ssl.trustManagerConfig.trustStoreConfigs :+ TrustStoreConfig(filePath = Some(p.toString), data = None))))) | ||
| case None ⇒ clientConfig | ||
| } | ||
| new AhcWSAPI(environment, clientConfigWithTruststore, lifecycle)(mat) | ||
| } | ||
|
|
||
| def getConfig(config: Configuration, path: String): Configuration = { | ||
| Configuration( | ||
| config.getConfig(s"play.$path").getOrElse(Configuration.empty).underlying.withFallback( | ||
| config.getConfig(path).getOrElse(Configuration.empty).underlying)) | ||
| } | ||
| } | ||
|
|
||
| class CustomWSAPI(ws: AhcWSAPI, val proxy: Option[WSProxyServer], config: Configuration, environment: Environment, lifecycle: ApplicationLifecycle, mat: Materializer) extends WSAPI { | ||
| private[CustomWSAPI] lazy val logger = Logger(getClass) | ||
|
|
||
| @Inject() def this(config: Configuration, environment: Environment, lifecycle: ApplicationLifecycle, mat: Materializer) = | ||
| this( | ||
| CustomWSAPI.getWS(config, environment, lifecycle, mat), | ||
| CustomWSAPI.parseProxyConfig(config), | ||
| config, environment, lifecycle, mat) | ||
|
|
||
| override def url(url: String): WSRequest = { | ||
| val req = ws.url(url) | ||
| proxy.fold(req)(req.withProxyServer) | ||
| } | ||
|
|
||
| override def client: AhcWSClient = ws.client | ||
|
|
||
| def withConfig(subConfig: Configuration): CustomWSAPI = { | ||
| logger.debug(s"Override WS configuration using $subConfig") | ||
| new CustomWSAPI( | ||
| Configuration(subConfig.underlying.atKey("play").withFallback(config.underlying)), | ||
| environment, lifecycle, mat) | ||
| } | ||
| } |
67 changes: 31 additions & 36 deletions
67
thehive-cortex/app/connectors/cortex/services/CortexClient.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,77 +1,72 @@ | ||
| package connectors.cortex.services | ||
|
|
||
| import scala.concurrent.{ ExecutionContext, Future } | ||
| import scala.concurrent.duration.Duration | ||
|
|
||
| import akka.stream.scaladsl.Source | ||
|
|
||
| import play.api.libs.json.{ JsObject, Json } | ||
| import play.api.libs.ws.{ WSClient, WSRequest, WSResponse } | ||
| import play.api.mvc.MultipartFormData.{ DataPart, FilePart } | ||
|
|
||
| import org.elastic4play.NotFoundError | ||
|
|
||
| import connectors.cortex.models.{ Analyzer, CortexArtifact, DataArtifact, FileArtifact } | ||
| import connectors.cortex.models.JsonFormat._ | ||
| import play.api.Logger | ||
| import play.api.libs.json.{ JsObject, JsValue, Json } | ||
| import play.api.libs.ws.{ WSRequest, WSResponse } | ||
| import play.api.mvc.MultipartFormData.{ DataPart, FilePart } | ||
| import services.CustomWSAPI | ||
|
|
||
| import scala.concurrent.duration.Duration | ||
| import scala.concurrent.{ ExecutionContext, Future } | ||
|
|
||
| class CortexClient(val name: String, baseUrl: String, key: String) { | ||
| class CortexClient(val name: String, baseUrl: String, key: String, ws: CustomWSAPI) { | ||
|
|
||
| lazy val logger = Logger(getClass) | ||
|
|
||
| logger.info(s"new Cortex($name, $baseUrl, $key)") | ||
| def request[A](uri: String, f: WSRequest ⇒ Future[WSResponse], t: WSResponse ⇒ A)(implicit ws: WSClient, ec: ExecutionContext): Future[A] = { | ||
| val url = (baseUrl + uri) | ||
| logger.info(s"Requesting Cortex $url") | ||
| f(ws.url(url).withHeaders("auth" → key)).map { | ||
| def request[A](uri: String, f: WSRequest ⇒ Future[WSResponse], t: WSResponse ⇒ A)(implicit ec: ExecutionContext): Future[A] = { | ||
| logger.info(s"Requesting Cortex $baseUrl") | ||
| f(ws.url(s"$baseUrl/$uri").withHeaders("auth" → key)).map { | ||
| case response if response.status / 100 == 2 ⇒ t(response) | ||
| case error ⇒ | ||
| logger.error(s"Cortex error on $url (${error.status}) \n${error.body}") | ||
| logger.error(s"Cortex error on $baseUrl (${error.status}) \n${error.body}") | ||
| sys.error("") | ||
| } | ||
| } | ||
|
|
||
| def getAnalyzer(analyzerId: String)(implicit ws: WSClient, ec: ExecutionContext): Future[Analyzer] = { | ||
| request(s"/api/analyzer/$analyzerId", _.get, _.json.as[Analyzer]).map(_.copy(cortexIds = List(name))) | ||
| def getAnalyzer(analyzerId: String)(implicit ec: ExecutionContext): Future[Analyzer] = { | ||
| request(s"api/analyzer/$analyzerId", _.get, _.json.as[Analyzer]).map(_.copy(cortexIds = List(name))) | ||
| } | ||
|
|
||
| def listAnalyzer(implicit ws: WSClient, ec: ExecutionContext): Future[Seq[Analyzer]] = { | ||
| request(s"/api/analyzer", _.get, _.json.as[Seq[Analyzer]]).map(_.map(_.copy(cortexIds = List(name)))) | ||
| def listAnalyzer(implicit ec: ExecutionContext): Future[Seq[Analyzer]] = { | ||
| request(s"api/analyzer", _.get, _.json.as[Seq[Analyzer]]).map(_.map(_.copy(cortexIds = List(name)))) | ||
| } | ||
|
|
||
| def analyze(analyzerId: String, artifact: CortexArtifact)(implicit ws: WSClient, ec: ExecutionContext) = { | ||
| def analyze(analyzerId: String, artifact: CortexArtifact)(implicit ec: ExecutionContext): Future[JsValue] = { | ||
| artifact match { | ||
| case FileArtifact(data, attributes) ⇒ | ||
| val body = Source(List( | ||
| FilePart("data", (attributes \ "attachment" \ "name").asOpt[String].getOrElse("noname"), None, data), | ||
| DataPart("_json", attributes.toString))) | ||
| request(s"/api/analyzer/$analyzerId/run", _.post(body), _.json) | ||
| request(s"api/analyzer/$analyzerId/run", _.post(body), _.json) | ||
| case a: DataArtifact ⇒ | ||
| request(s"/api/analyzer/$analyzerId/run", _.post(Json.toJson(a)), _.json.as[JsObject]) | ||
| request(s"api/analyzer/$analyzerId/run", _.post(Json.toJson(a)), _.json.as[JsObject]) | ||
| } | ||
| } | ||
|
|
||
| def listAnalyzerForType(dataType: String)(implicit ws: WSClient, ec: ExecutionContext): Future[Seq[Analyzer]] = { | ||
| request(s"/api/analyzer/type/$dataType", _.get, _.json.as[Seq[Analyzer]]).map(_.map(_.copy(cortexIds = List(name)))) | ||
| def listAnalyzerForType(dataType: String)(implicit ec: ExecutionContext): Future[Seq[Analyzer]] = { | ||
| request(s"api/analyzer/type/$dataType", _.get, _.json.as[Seq[Analyzer]]).map(_.map(_.copy(cortexIds = List(name)))) | ||
| } | ||
|
|
||
| def listJob(implicit ws: WSClient, ec: ExecutionContext) = { | ||
| request(s"/api/job", _.get, _.json.as[Seq[JsObject]]) | ||
| def listJob(implicit ec: ExecutionContext): Future[Seq[JsObject]] = { | ||
| request(s"api/job", _.get, _.json.as[Seq[JsObject]]) | ||
| } | ||
|
|
||
| def getJob(jobId: String)(implicit ws: WSClient, ec: ExecutionContext) = { | ||
| request(s"/api/job/$jobId", _.get, _.json.as[JsObject]) | ||
| def getJob(jobId: String)(implicit ec: ExecutionContext): Future[JsObject] = { | ||
| request(s"api/job/$jobId", _.get, _.json.as[JsObject]) | ||
| } | ||
|
|
||
| def removeJob(jobId: String)(implicit ws: WSClient, ec: ExecutionContext) = { | ||
| request(s"/api/job/$jobId", _.delete, _ ⇒ ()) | ||
| def removeJob(jobId: String)(implicit ec: ExecutionContext): Future[Unit] = { | ||
| request(s"api/job/$jobId", _.delete, _ ⇒ ()) | ||
| } | ||
|
|
||
| def report(jobId: String)(implicit ws: WSClient, ec: ExecutionContext) = { | ||
| request(s"/api/job/$jobId/report", _.get, r ⇒ r.json.as[JsObject]) | ||
| def report(jobId: String)(implicit ec: ExecutionContext): Future[JsObject] = { | ||
| request(s"api/job/$jobId/report", _.get, r ⇒ r.json.as[JsObject]) | ||
| } | ||
|
|
||
| def waitReport(jobId: String, atMost: Duration)(implicit ws: WSClient, ec: ExecutionContext) = { | ||
| request(s"/api/job/$jobId/waitreport", _.withQueryString("atMost" → atMost.toString).get, r ⇒ r.json.as[JsObject]) | ||
| def waitReport(jobId: String, atMost: Duration)(implicit ec: ExecutionContext): Future[JsObject] = { | ||
| request(s"api/job/$jobId/waitreport", _.withQueryString("atMost" → atMost.toString).get, r ⇒ r.json.as[JsObject]) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.