Skip to content

Commit

Permalink
#623 #913 Update Elasic4play to add support of ElasticSearch 6
Browse files Browse the repository at this point in the history
  • Loading branch information
@To-om
To-om committed May 20, 2019
1 parent 5f8c0d6 commit 4c0df97
Show file tree
Hide file tree
Showing 8 changed files with 133 additions and 116 deletions.
4 changes: 2 additions & 2 deletions project/Dependencies.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ object Dependencies {
val guice = "com.typesafe.play" %% "play-guice" % version
}

val scalaGuice = "net.codingwell" %% "scala-guice" % "4.2.1"
val scalaGuice = "net.codingwell" %% "scala-guice" % "4.2.3"

val reflections = "org.reflections" % "reflections" % "0.9.11"
val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
val elastic4play = "org.thehive-project" %% "elastic4play" % "1.10.0"
val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.1"
val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.19"
val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.19"
}
Expand Down
157 changes: 82 additions & 75 deletions thehive-backend/app/controllers/ArtifactCtrl.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,27 @@ import java.io.FilterInputStream
import java.nio.file.Files

import scala.collection.JavaConverters._
import scala.concurrent.{ ExecutionContext, Future }
import scala.concurrent.{ExecutionContext, Future}

import play.api.{ Configuration, Logger }
import play.api.{Configuration, Logger}
import play.api.http.Status
import play.api.libs.json.JsArray
import play.api.mvc._

import javax.inject.{ Inject, Singleton }
import javax.inject.{Inject, Singleton}
import models.Roles
import net.lingala.zip4j.core.ZipFile
import net.lingala.zip4j.model.FileHeader
import services.ArtifactSrv

import org.elastic4play.controllers._
import org.elastic4play.models.JsonFormat.baseModelEntityWrites
import org.elastic4play.services.JsonFormat.{ aggReads, queryReads }
import org.elastic4play.services.JsonFormat.{aggReads, queryReads}
import org.elastic4play.services._
import org.elastic4play.{ BadRequestError, InternalError, Timed }
import org.elastic4play.{BadRequestError, InternalError, Timed}

@Singleton
class ArtifactCtrl @Inject() (
class ArtifactCtrl @Inject()(
artifactSrv: ArtifactSrv,
auxSrv: AuxSrv,
tempSrv: TempSrv,
Expand All @@ -33,7 +33,9 @@ class ArtifactCtrl @Inject() (
components: ControllerComponents,
fieldsBodyParser: FieldsBodyParser,
configuration: Configuration,
implicit val ec: ExecutionContext) extends AbstractController(components) with Status {
implicit val ec: ExecutionContext
) extends AbstractController(components)
with Status {

private[ArtifactCtrl] lazy val logger = Logger(getClass)

Expand All @@ -45,17 +47,15 @@ class ArtifactCtrl @Inject() (
val file = tempSrv.newTemporaryFile(fileName, "-fromZipFile")

val input = zipFile.getInputStream(header)
val size = header.getUncompressedSize
val size = header.getUncompressedSize
val sizedInput: FilterInputStream = new FilterInputStream(input) {
var totalRead = 0

override def read(): Int = {
override def read(): Int =
if (totalRead < size) {
totalRead += 1
super.read()
}
else throw BadRequestError("Error extracting file: output size doesn't match header")
}
} else throw BadRequestError("Error extracting file: output size doesn't match header")
}
Files.delete(file)
val fileSize = Files.copy(sizedInput, file)
Expand All @@ -70,67 +70,73 @@ class ArtifactCtrl @Inject() (
}

@Timed
def create(caseId: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) {
implicit request ⇒
val fields = request.body
val data = fields.getStrings("data")
.getOrElse(fields.getString("data").toSeq)
.map(_.trim) // most observables don't accept leading or trailing space
.filterNot(_.isEmpty)
// if data is not multivalued, use simple API (not bulk API)
if (data.isEmpty) {

fields.get("attachment")
.collect {
case FileInputValue(_, filepath, _) if fields.getBoolean("isZip").getOrElse(false) ⇒

val zipFile = new ZipFile(filepath.toFile)
val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]]

if (zipFile.isEncrypted) {
val pw = fields.getString("zipPassword")
.filterNot(_.isEmpty)
.getOrElse(configuration.get[String]("datastore.attachment.password"))
zipFile.setPassword(pw)
def create(caseId: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒
val fields = request.body
val data = fields
.getStrings("data")
.getOrElse(fields.getString("data").toSeq)
.map(_.trim) // most observables don't accept leading or trailing space
.filterNot(_.isEmpty)
// if data is not multivalued, use simple API (not bulk API)
if (data.isEmpty) {

fields
.get("attachment")
.collect {
case FileInputValue(_, filepath, _) if fields.getBoolean("isZip").getOrElse(false) ⇒
val zipFile = new ZipFile(filepath.toFile)
val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]]

if (zipFile.isEncrypted) {
val pw = fields
.getString("zipPassword")
.filterNot(_.isEmpty)
.getOrElse(configuration.get[String]("datastore.attachment.password"))
zipFile.setPassword(pw)
}

val multiFields = files
.filterNot(_.isDirectory)
.flatMap(extractAndCheckSize(zipFile, _))
.map { fiv ⇒
fields
.unset("isZip")
.unset("zipPassword")
.set("dataType", "file")
.set("attachment", fiv)
}

val multiFields = files.filterNot(_.isDirectory)
.flatMap(extractAndCheckSize(zipFile, _))
.map { fiv ⇒
fields
.unset("isZip")
.unset("zipPassword")
.set("dataType", "file")
.set("attachment", fiv)
}
artifactSrv.create(caseId, multiFields)
.map(multiResult ⇒ renderer.toMultiOutput(CREATED, multiResult))
}
.getOrElse {
artifactSrv.create(caseId, fields.unset("isZip").unset("zipPassword"))
.map(artifact ⇒ renderer.toOutput(CREATED, artifact))
}
}
else if (data.length == 1) {
artifactSrv.create(caseId, fields.set("data", data.head).unset("isZip").unset("zipPassword"))
.map(artifact ⇒ renderer.toOutput(CREATED, artifact))
}
else {
val multiFields = data.map(fields.set("data", _).unset("isZip").unset("zipPassword"))
artifactSrv.create(caseId, multiFields)
.map(multiResult ⇒ renderer.toMultiOutput(CREATED, multiResult))
}
artifactSrv
.create(caseId, multiFields)
.map(multiResult ⇒ renderer.toMultiOutput(CREATED, multiResult))
}
.getOrElse {
artifactSrv
.create(caseId, fields.unset("isZip").unset("zipPassword"))
.map(artifact ⇒ renderer.toOutput(CREATED, artifact))
}
} else if (data.length == 1) {
artifactSrv
.create(caseId, fields.set("data", data.head).unset("isZip").unset("zipPassword"))
.map(artifact ⇒ renderer.toOutput(CREATED, artifact))
} else {
val multiFields = data.map(fields.set("data", _).unset("isZip").unset("zipPassword"))
artifactSrv
.create(caseId, multiFields)
.map(multiResult ⇒ renderer.toMultiOutput(CREATED, multiResult))
}
}

@Timed
def get(id: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
artifactSrv.get(id)
artifactSrv
.get(id)
.map(artifact ⇒ renderer.toOutput(OK, artifact))
}

@Timed
def update(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒
artifactSrv.update(id, request.body.unset("attachment"))
artifactSrv
.update(id, request.body.unset("attachment"))
.map(artifact ⇒ renderer.toOutput(OK, artifact))
}

Expand All @@ -143,51 +149,52 @@ class ArtifactCtrl @Inject() (

@Timed
def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒
artifactSrv.delete(id)
artifactSrv
.delete(id)
.map(_ ⇒ NoContent)
}

@Timed
def findInCase(caseId: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
import org.elastic4play.services.QueryDSL._
val childQuery = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
val query = and(childQuery, "_parent" ~= caseId)
val range = request.body.getString("range")
val sort = request.body.getStrings("sort").getOrElse(Nil)
val query = and(childQuery, withParent("case", caseId))
val range = request.body.getString("range")
val sort = request.body.getStrings("sort").getOrElse(Nil)

val (artifacts, total) = artifactSrv.find(query, range, sort)
renderer.toOutput(OK, artifacts, total)
}

@Timed
def find(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
val range = request.body.getString("range")
val sort = request.body.getStrings("sort").getOrElse(Nil)
val nparent = request.body.getLong("nparent").getOrElse(0L).toInt
val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
val range = request.body.getString("range")
val sort = request.body.getStrings("sort").getOrElse(Nil)
val nparent = request.body.getLong("nparent").getOrElse(0L).toInt
val withStats = request.body.getBoolean("nstats").getOrElse(false)

val (artifacts, total) = artifactSrv.find(query, range, sort)
val artifactWithCase = auxSrv(artifacts, nparent, withStats, removeUnaudited = false)
val artifactWithCase = auxSrv(artifacts, nparent, withStats, removeUnaudited = false)
renderer.toOutput(OK, artifactWithCase, total)
}

@Timed
def findSimilar(artifactId: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
artifactSrv.get(artifactId).flatMap { artifact ⇒
val range = request.body.getString("range")
val sort = request.body.getStrings("sort").getOrElse(Nil)
val sort = request.body.getStrings("sort").getOrElse(Nil)

val (artifacts, total) = artifactSrv.findSimilar(artifact, range, sort)
val artifactWithCase = auxSrv(artifacts, 1, withStats = false, removeUnaudited = true)
val artifactWithCase = auxSrv(artifacts, 1, withStats = false, removeUnaudited = true)
renderer.toOutput(OK, artifactWithCase, total)
}
}

@Timed
def stats(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
val aggs = request.body.getValue("stats").getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]]
val aggs = request.body.getValue("stats").getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]]
artifactSrv.stats(query, aggs).map(s ⇒ Ok(s))
}
}
4 changes: 2 additions & 2 deletions thehive-backend/app/controllers/StatusCtrl.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import play.api.Configuration
import play.api.libs.json.{ JsBoolean, JsObject, JsString, Json }
import play.api.libs.json.Json.toJsFieldJsValueWrapper
import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
import com.sksamuel.elastic4s.ElasticDsl
import com.sksamuel.elastic4s.http.ElasticDsl
import connectors.Connector
import models.HealthStatus
import org.elastic4play.Timed
Expand Down Expand Up @@ -37,7 +37,7 @@ class StatusCtrl @Inject() (
"Elastic4Play" → getVersion(classOf[Timed]),
"Play" → getVersion(classOf[AbstractController]),
"Elastic4s" → getVersion(classOf[ElasticDsl]),
"ElasticSearch" → getVersion(classOf[org.elasticsearch.Build])),
"ElasticSearch" → getVersion(classOf[org.elasticsearch.client.Node])),
"connectors" → JsObject(connectors.map(c ⇒ c.name → c.status).toSeq),
"health" → Json.obj("elasticsearch" → clusterStatusName),
"config" → Json.obj(
Expand Down
Loading

0 comments on commit 4c0df97

Please sign in to comment.