From 276f79b7698d1ccfbd2f8ce4bec1dd2452edf38f Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 11 Aug 2020 16:44:22 +0200 Subject: [PATCH] #279 Add support of ES7 --- .../thp/cortex/controllers/StatusCtrl.scala | 4 +- app/org/thp/cortex/models/Migration.scala | 6 +- app/org/thp/cortex/models/Roles.scala | 4 +- .../thp/cortex/services/JobRunnerSrv.scala | 5 +- app/org/thp/cortex/services/JobSrv.scala | 2 +- app/org/thp/cortex/services/UserSrv.scala | 2 +- .../thp/cortex/services/WorkerConfigSrv.scala | 2 +- app/org/thp/cortex/services/WorkerSrv.scala | 4 +- build.sbt | 14 +- project/Bintray.scala | 167 ------------------ project/Dependencies.scala | 2 +- project/plugins.sbt | 5 +- 12 files changed, 15 insertions(+), 202 deletions(-) delete mode 100644 project/Bintray.scala diff --git a/app/org/thp/cortex/controllers/StatusCtrl.scala b/app/org/thp/cortex/controllers/StatusCtrl.scala index a9d85792a..9446ab646 100644 --- a/app/org/thp/cortex/controllers/StatusCtrl.scala +++ b/app/org/thp/cortex/controllers/StatusCtrl.scala @@ -8,12 +8,11 @@ import play.api.libs.json.Json.toJsFieldJsValueWrapper import play.api.libs.json.{JsBoolean, JsString, Json} import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents} -import com.sksamuel.elastic4s.http.ElasticDsl +import com.sksamuel.elastic4s.ElasticDsl import javax.inject.{Inject, Singleton} import org.elasticsearch.client.Node import org.thp.cortex.models.Worker -import org.elastic4play.database.DBIndex import org.elastic4play.services.AuthSrv import org.elastic4play.services.auth.MultiAuthSrv @@ -21,7 +20,6 @@ import org.elastic4play.services.auth.MultiAuthSrv class StatusCtrl @Inject()( configuration: Configuration, authSrv: AuthSrv, - dbIndex: DBIndex, components: ControllerComponents, implicit val ec: ExecutionContext ) extends AbstractController(components) diff --git a/app/org/thp/cortex/models/Migration.scala b/app/org/thp/cortex/models/Migration.scala index bad438b00..6ddaaa135 100644 --- a/app/org/thp/cortex/models/Migration.scala +++ b/app/org/thp/cortex/models/Migration.scala @@ -11,14 +11,14 @@ import org.thp.cortex.services.{OrganizationSrv, UserSrv, WorkerSrv} import org.elastic4play.controllers.Fields import org.elastic4play.services.Operation._ -import org.elastic4play.services.{DatabaseState, IndexType, MigrationOperations, Operation} +import org.elastic4play.services.{DatabaseState, MigrationOperations, Operation} import org.elastic4play.utils.Hasher @Singleton class Migration @Inject()(userSrv: UserSrv, organizationSrv: OrganizationSrv, workerSrv: WorkerSrv, implicit val ec: ExecutionContext) extends MigrationOperations { - lazy val logger = Logger(getClass) + lazy val logger: Logger = Logger(getClass) def beginMigration(version: Int): Future[Unit] = Future.successful(()) def endMigration(version: Int): Future[Unit] = @@ -28,8 +28,6 @@ class Migration @Inject()(userSrv: UserSrv, organizationSrv: OrganizationSrv, wo .transform(_ ⇒ Success(())) // ignore errors (already exist) } - override def indexType(version: Int): IndexType.Value = if (version > 3) IndexType.indexWithoutMappingTypes else IndexType.indexWithMappingTypes - val operations: PartialFunction[DatabaseState, Seq[Operation]] = { case DatabaseState(1) ⇒ val hasher = Hasher("MD5") diff --git a/app/org/thp/cortex/models/Roles.scala b/app/org/thp/cortex/models/Roles.scala index 0c906f2b4..8aeded20b 100644 --- a/app/org/thp/cortex/models/Roles.scala +++ b/app/org/thp/cortex/models/Roles.scala @@ -2,8 +2,8 @@ package org.thp.cortex.models import play.api.libs.json.{JsString, JsValue} -import com.sksamuel.elastic4s.http.ElasticDsl.keywordField -import com.sksamuel.elastic4s.mappings.KeywordField +import com.sksamuel.elastic4s.ElasticDsl.keywordField +import com.sksamuel.elastic4s.requests.mappings.KeywordField import org.scalactic.{Every, Good, One, Or} import org.elastic4play.{AttributeError, InvalidFormatAttributeError} diff --git a/app/org/thp/cortex/services/JobRunnerSrv.scala b/app/org/thp/cortex/services/JobRunnerSrv.scala index b1d172d03..b2bdc6826 100644 --- a/app/org/thp/cortex/services/JobRunnerSrv.scala +++ b/app/org/thp/cortex/services/JobRunnerSrv.scala @@ -100,10 +100,7 @@ class JobRunnerSrv @Inject()( attachmentSrv .source(attachment.id) .runWith(FileIO.toPath(attachmentFile)) - .flatMap { - case ioresult if ioresult.status.isSuccess ⇒ Future.successful(Some(attachmentFile)) - case ioresult ⇒ Future.failed(ioresult.getError) - } + .map(_ => Some(attachmentFile)) } .getOrElse(Future.successful(None)) .map { diff --git a/app/org/thp/cortex/services/JobSrv.scala b/app/org/thp/cortex/services/JobSrv.scala index 1b2b6944b..c5c458c30 100644 --- a/app/org/thp/cortex/services/JobSrv.scala +++ b/app/org/thp/cortex/services/JobSrv.scala @@ -94,7 +94,7 @@ class JobSrv( private def withUserFilter[A](userId: String)(x: String ⇒ (Source[A, NotUsed], Future[Long])): (Source[A, NotUsed], Future[Long]) = { val a = userSrv.getOrganizationId(userId).map(x) - val aSource = Source.fromFutureSource(a.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val aSource = Source.futureSource(a.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) val aTotal = a.flatMap(_._2) aSource → aTotal } diff --git a/app/org/thp/cortex/services/UserSrv.scala b/app/org/thp/cortex/services/UserSrv.scala index 47f137de3..b4bc7fdc2 100644 --- a/app/org/thp/cortex/services/UserSrv.scala +++ b/app/org/thp/cortex/services/UserSrv.scala @@ -162,7 +162,7 @@ class UserSrv( } yield findForOrganization(organizationId, queryDef, range, sortBy)) .recover { case NotFoundError("user init not found") ⇒ Source.empty → Future.successful(0L) } - val userSource = Source.fromFutureSource(users.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val userSource = Source.futureSource(users.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) val userTotal = users.flatMap(_._2) userSource → userTotal } diff --git a/app/org/thp/cortex/services/WorkerConfigSrv.scala b/app/org/thp/cortex/services/WorkerConfigSrv.scala index 02208b50a..3b7d8ebf5 100644 --- a/app/org/thp/cortex/services/WorkerConfigSrv.scala +++ b/app/org/thp/cortex/services/WorkerConfigSrv.scala @@ -109,7 +109,7 @@ trait WorkerConfigSrv { val configs = userSrv .getOrganizationId(userId) .map(organizationId ⇒ findForOrganization(organizationId, queryDef, range, sortBy)) - val configSource = Source.fromFutureSource(configs.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val configSource = Source.futureSource(configs.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) val configTotal = configs.flatMap(_._2) configSource → configTotal } diff --git a/app/org/thp/cortex/services/WorkerSrv.scala b/app/org/thp/cortex/services/WorkerSrv.scala index 719d22375..1515d8ed9 100644 --- a/app/org/thp/cortex/services/WorkerSrv.scala +++ b/app/org/thp/cortex/services/WorkerSrv.scala @@ -94,7 +94,7 @@ class WorkerSrv @Inject()( user ← userSrv.get(userId) organizationId = user.organization() } yield findForOrganization(organizationId, and(queryDef, "type" ~= WorkerType.analyzer), range, sortBy) - val analyserSource = Source.fromFutureSource(analyzers.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val analyserSource = Source.futureSource(analyzers.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) val analyserTotal = analyzers.flatMap(_._2) analyserSource → analyserTotal } @@ -110,7 +110,7 @@ class WorkerSrv @Inject()( user ← userSrv.get(userId) organizationId = user.organization() } yield findForOrganization(organizationId, and(queryDef, "type" ~= WorkerType.responder), range, sortBy) - val analyserSource = Source.fromFutureSource(responders.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val analyserSource = Source.futureSource(responders.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) val analyserTotal = responders.flatMap(_._2) analyserSource → analyserTotal } diff --git a/build.sbt b/build.sbt index bd6fc016e..77b674855 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,6 @@ import Common._ lazy val cortex = (project in file(".")) .enablePlugins(PlayScala) - .enablePlugins(Bintray) .settings(projectSettings) libraryDependencies ++= Seq( @@ -33,15 +32,4 @@ packageBin := { (packageBin in Debian).value (packageBin in Rpm).value (packageBin in Universal).value -} - -// Bintray // -bintrayOrganization := Some("thehive-project") -bintrayRepository := "cortex" -publish := { - (publish in Docker).value - publishRelease.value - publishLatest.value - publishRpm.value - publishDebian.value -} +} \ No newline at end of file diff --git a/project/Bintray.scala b/project/Bintray.scala deleted file mode 100644 index b2a22b324..000000000 --- a/project/Bintray.scala +++ /dev/null @@ -1,167 +0,0 @@ -import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.duration.Duration - -import bintray.BintrayCredentials -import bintray.BintrayKeys.{ bintrayEnsureCredentials, bintrayOrganization, bintrayPackage } -import bintry.Client -import com.typesafe.sbt.packager.Keys._ -import com.typesafe.sbt.packager.debian.DebianPlugin.autoImport.Debian -import com.typesafe.sbt.packager.rpm.RpmPlugin.autoImport.Rpm -import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport.Universal -import dispatch.{ FunctionHandler, Http } -import sbt.Keys._ -import sbt._ - -object Bintray extends AutoPlugin { - - object autoImport { - val publishRelease: TaskKey[Unit] = taskKey[Unit]("Publish binary in bintray") - val publishLatest: TaskKey[Unit] = taskKey[Unit]("Publish latest binary in bintray") - val publishDebian: TaskKey[Unit] = taskKey[Unit]("publish debian package in Bintray") - val publishRpm: TaskKey[Unit] = taskKey[Unit]("publish rpm package in Bintray") - val rpmReleaseFile = taskKey[File]("The rpm release package file") - } - - import autoImport._ - - override lazy val projectSettings = Seq( - - publishRelease in ThisBuild := { - val file = (packageBin in Universal).value - btPublish(file.getName, - file, - bintrayEnsureCredentials.value, - bintrayOrganization.value, - "binary", - bintrayPackage.value, - (version in ThisBuild).value, - sLog.value) - }, - - publishLatest in ThisBuild := Def.taskDyn { - if ((version in ThisBuild).value.endsWith("-SNAPSHOT")) sys.error("Snapshot version can't be released") - val file = (packageBin in Universal).value - val latestVersion = if (version.value.contains('-')) "latest-beta" else "latest" - val latestName = file.getName.replace(version.value, latestVersion) - if (latestName == file.getName) - Def.task { - sLog.value.warn(s"Latest package name can't be built using package name [$latestName], publish aborted") - } - else Def.task { - removeVersion(bintrayEnsureCredentials.value, - bintrayOrganization.value, - "binary", - bintrayPackage.value, - latestVersion, - sLog.value) - btPublish(latestName, - file, - bintrayEnsureCredentials.value, - bintrayOrganization.value, - "binary", - bintrayPackage.value, - latestVersion, - sLog.value) - } - } - .value, - - publishDebian in ThisBuild := { - if ((version in ThisBuild).value.endsWith("-SNAPSHOT")) sys.error("Snapshot version can't be released") - val file = (debianSign in Debian).value - val bintrayCredentials = bintrayEnsureCredentials.value - btPublish(file.getName, - file, - bintrayCredentials, - bintrayOrganization.value, - "debian-beta", - bintrayPackage.value, - version.value, - sLog.value, - "deb_distribution" → "any", - "deb_component" → "main", - "deb_architecture" → "all" - ) - if (!version.value.contains('-')) - btPublish(file.getName, - file, - bintrayCredentials, - bintrayOrganization.value, - "debian-stable", - bintrayPackage.value, - version.value, - sLog.value, - "deb_distribution" → "any", - "deb_component" → "main", - "deb_architecture" → "all" - ) - }, - - publishRpm in ThisBuild := { - if ((version in ThisBuild).value.endsWith("-SNAPSHOT")) sys.error("Snapshot version can't be released") - val file = (packageBin in Rpm).value - val bintrayCredentials = bintrayEnsureCredentials.value - btPublish(file.getName, - file, - bintrayCredentials, - bintrayOrganization.value, - "rpm-beta", - bintrayPackage.value, - (version in Rpm).value + '-' + (rpmRelease in Rpm).value, - sLog.value) - if (!version.value.contains('-')) - btPublish(file.getName, - file, - bintrayCredentials, - bintrayOrganization.value, - "rpm-stable", - bintrayPackage.value, - (version in Rpm).value + '-' + (rpmRelease in Rpm).value, - sLog.value) - } - ) - - private def asStatusAndBody = new FunctionHandler({ r => (r.getStatusCode, r.getResponseBody) }) - - def removeVersion(credential: BintrayCredentials, - org: Option[String], - repoName: String, - packageName: String, - version: String, - log: Logger): Unit = { - val BintrayCredentials(user, key) = credential - val client: Client = Client(user, key, new Http()) - val repo: Client#Repo = client.repo(org.getOrElse(user), repoName) - Await.result(repo.get(packageName).version(version).delete(asStatusAndBody), Duration.Inf) match { - case (status, body) => log.info(s"Delete version $packageName $version: $status ($body)") - } - } - - private def btPublish(filename: String, - file: File, - credential: BintrayCredentials, - org: Option[String], - repoName: String, - packageName: String, - version: String, - log: Logger, - additionalParams: (String, String)*): Unit = { - val BintrayCredentials(user, key) = credential - val owner: String = org.getOrElse(user) - val client: Client = Client(user, key, new Http()) - val repo: Client#Repo = client.repo(org.getOrElse(user), repoName) - - - val params = additionalParams - .map { case (k, v) => s"$k=$v" } - .mkString(";", ";", "") - val upload = repo.get(packageName).version(version).upload(filename + params, file) - - log.info(s"Uploading $file ... (${org.getOrElse(user)}/$repoName/$packageName/$version/$filename$params)") - Await.result(upload(asStatusAndBody), Duration.Inf) match { - case (201, _) => log.info(s"$file was uploaded to $owner/$packageName@$version") - case (_, fail) => sys.error(s"failed to upload $file to $owner/$packageName@$version: $fail") - } - } -} \ No newline at end of file diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 92a9f4ad3..97e36a569 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -18,7 +18,7 @@ object Dependencies { val reflections = "org.reflections" % "reflections" % "0.9.11" val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2" - val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.5" + val elastic4play = "org.thehive-project" %% "elastic4play" % "1.12.0-SNAPSHOT" val dockerClient = "com.spotify" % "docker-client" % "8.14.4" val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.21" } diff --git a/project/plugins.sbt b/project/plugins.sbt index 838f26ac7..dcbd25249 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,6 +2,5 @@ logLevel := Level.Info // The Play plugin -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.23") -addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.1") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.0.0") \ No newline at end of file +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.2") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") \ No newline at end of file