From 3d0f5f95e23913ed3886b6676c5542d92811d424 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 13 Dec 2018 10:21:18 +0100 Subject: [PATCH] #156 Add configuration for drone --- .drone.yml | 163 ++++++++++++++++++++++++ debian.sbt | 13 +- docker.sbt | 20 ++- project/Common.scala | 25 ++-- rpm.sbt | 37 +++++- test/org/thp/cortex/AnalyzersSpec.scala | 79 ------------ version.sbt | 2 +- www/package.json | 3 +- 8 files changed, 234 insertions(+), 108 deletions(-) create mode 100644 .drone.yml delete mode 100644 test/org/thp/cortex/AnalyzersSpec.scala diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 000000000..2c2129208 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,163 @@ +--- +kind: pipeline +name: default + +# Disable default clone +clone: + disable: true + +steps: + # This clone step doesn't use "root" user + - name: clone + image: plugins/git:next + + # Restore cache of downloaded dependencies + - name: restore cache + image: drillster/drone-volume-cache + settings: + restore: true + mount: + - .sbt + - .ivy2 + - www/node_modules + volumes: [{name: cache, path: /cache}] + + # Run project tests + - name: run tests and build stage + image: thehiveproject/drone-scala-node + commands: + - . ~/.nvm/nvm.sh + - sbt -Duser.home=$PWD test stage + + # Build packages + - name: build packages + image: thehiveproject/drone-scala-node + settings: + pgp_key: {from_secret: pgp_key} + commands: + - | + . ~/.nvm/nvm.sh + [ -n "$PLUGIN_PGP_KEY" ] && gpg --batch --import - <<< $PLUGIN_PGP_KEY + sbt -Duser.home=$PWD docker:stage debian:packageBin rpm:packageBin universal:packageBin + when: + event: [tag] + + # Save external libraries in cache + - name: save cache + image: drillster/drone-volume-cache + settings: + rebuild: true + mount: + - .sbt + - .ivy2 + - www/node_modules + volumes: [{name: cache, path: /cache}] + + - name: publish packages + image: thehiveproject/drone-bintray + settings: + user: {from_secret: bintray_user} + key: {from_secret: bintray_key} + subject: thehive-project + package: cortex + commands: + - | + export PLUGIN_USER + export PLUGIN_KEY + export PLUGIN_SUBJECT + export PLUGIN_PACKAGE + export PLUGIN_VERSION=$(cut -d\" -f2 version.sbt) + echo "Publishing package version $PLUGIN_VERSION" + + if echo $PLUGIN_VERSION | grep -qvi -E \ + -e '^[0-9]+\.[0-9]+\.[0-9]+$' \ + -e '^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+$' \ + -e '^[0-9]+\.[0-9]+\.[0-9]+-RC[0-9]+$'; then + echo The version $PLUGIN_VERSION has invalid format + exit 1 + fi + + CHANNEL=stable + echo $PLUGIN_VERSION | grep -qi rc && CHANNEL=beta + + DEB_FILE=target/cortex_$${PLUGIN_VERSION}_all.deb + RPM_FILE=target/rpm/RPMS/noarch/cortex-$${PLUGIN_VERSION}.noarch.rpm + ZIP_FILE=target/universal/cortex-$${PLUGIN_VERSION}.zip + + upload \ + --file $DEB_FILE \ + --repo debian-beta \ + --extra-param deb_distribution=any \ + --extra-param deb_component=main \ + --extra-param deb_architecture=all + + [ $CHANNEL = stable ] && upload \ + --file $DEB_FILE \ + --repo debian-stable \ + --extra-param deb_distribution=any \ + --extra-param deb_component=main \ + --extra-param deb_architecture=all + + upload \ + --file $RPM_FILE \ + --repo rpm-beta + + [ $CHANNEL = stable ] && upload \ + --file $RPM_FILE \ + --repo rpm-stable + + upload \ + --file $ZIP_FILE \ + --repo binary + + LATEST_VERSION=latest + [ $CHANNEL = beta ] && LATEST_VERSION=latest-beta + + removeVersion \ + --repo binary \ + --version $LATEST_VERSION + + upload \ + --file $ZIP_FILE \ + --repo binary \ + --version $LATEST_VERSION \ + --dest-file cortex-$${LATEST_VERSION}.zip + when: + event: [tag] + + # Publish docker image + - name: docker + image: plugins/docker + settings: + context: target/docker/stage + dockerfile: target/docker/stage/Dockerfile + repo: tooom/cortex + auto_tag: true + username: {from_secret: docker_username} + password: {from_secret: docker_password} + when: + event: [tag] + + - name: copy binaries + image: appleboy/drone-scp + settings: + host: {from_secret: deploy_beta_host} + username: {from_secret: deploy_username} + key: {from_secret: deploy_key} + target: ./cortex-builds/${DRONE_BUILD_NUMBER} + source: target/universal/stage + strip_components: 3 + + - name: deploy binaries + image: appleboy/drone-ssh + settings: + host: {from_secret: deploy_beta_host} + username: {from_secret: deploy_username} + key: {from_secret: deploy_key} + script: + - ./start cortex ${DRONE_BUILD_NUMBER} + +volumes: + - name: cache + host: + path: /opt/drone/cache diff --git a/debian.sbt b/debian.sbt index 798a7905f..c4d194d5b 100644 --- a/debian.sbt +++ b/debian.sbt @@ -1,6 +1,15 @@ -import Common._ +import Common.{stableVersion, betaVersion, snapshotVersion} -version in Debian := getVersion(version.value) + '-' + getRelease(version.value) +linuxPackageMappings in Debian += packageMapping(file("LICENSE") -> "/usr/share/doc/cortex/copyright").withPerms("644") +version in Debian := { + version.value match { + case stableVersion(_, _) => version.value + case betaVersion(v1, v2) => v1 + "-0.1RC" + v2 + case snapshotVersion(_, _) => version.value + "-SNAPSHOT" + case _ => sys.error("Invalid version: " + version.value) + } +} +debianPackageRecommends := Seq("elasticsearch") debianPackageDependencies += "java8-runtime | java8-runtime-headless" maintainerScripts in Debian := maintainerScriptsFromDirectory( baseDirectory.value / "package" / "debian", diff --git a/docker.sbt b/docker.sbt index df6ed9ef8..e4f86999b 100644 --- a/docker.sbt +++ b/docker.sbt @@ -1,20 +1,26 @@ -import Common._ -import com.typesafe.sbt.packager.docker.{ Cmd, ExecCmd } +import Common.{betaVersion, snapshotVersion, stableVersion} +import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd} -version in Docker := getVersion(version.value) + '-' + getRelease(version.value) +version in Docker := { + version.value match { + case stableVersion(_, _) => version.value + case betaVersion(v1, v2) => v1 + "-0.1RC" + v2 + case snapshotVersion(_, _) => version.value + "-SNAPSHOT" + case _ => sys.error("Invalid version: " + version.value) + } +} defaultLinuxInstallLocation in Docker := "/opt/cortex" dockerRepository := Some("thehiveproject") -dockerUpdateLatest := true +dockerUpdateLatest := !version.value.toUpperCase.contains("RC") dockerEntrypoint := Seq("/opt/cortex/entrypoint") -dockerExposedPorts := Seq(9001) +dockerExposedPorts := Seq(9000) mappings in Docker ++= Seq( file("package/docker/entrypoint") -> "/opt/cortex/entrypoint", - file("conf/logback.xml") -> "/etc/cortex/logback.xml", + file("package/logback.xml") -> "/etc/cortex/logback.xml", file("package/empty") -> "/var/log/cortex/application.log") mappings in Docker ~= (_.filterNot { case (_, filepath) => filepath == "/opt/cortex/conf/application.conf" }) - dockerCommands ~= { dc => val (dockerInitCmds, dockerTailCmds) = dc .collect { diff --git a/project/Common.scala b/project/Common.scala index 836f979dd..a382ef5ee 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -1,14 +1,17 @@ -import sbt._ +import scala.util.matching.Regex + import sbt.Keys._ +import sbt._ object Common { val projectSettings = Seq( organizationName := "TheHive-Project", organization := "org.thehive-project", - licenses += "AGPL-V3" -> url("https://www.gnu.org/licenses/agpl-3.0.html"), + licenses += "AGPL-V3" → url("https://www.gnu.org/licenses/agpl-3.0.html"), organizationHomepage := Some(url("http://thehive-project.org/")), resolvers += Resolver.bintrayRepo("thehive-project", "maven"), + resolvers += "elasticsearch-releases" at "https://artifacts.elastic.co/maven", scalaVersion := Dependencies.scalaVersion, scalacOptions ++= Seq( "-deprecation", // Emit warning and location for usages of deprecated APIs. @@ -22,7 +25,7 @@ object Common { "-Ywarn-nullary-override", // Warn when non-nullary overrides nullary, e.g. def foo() over def foo. "-Ywarn-numeric-widen" // Warn when numerics are widened. ), - scalacOptions in Test ~= { options => + scalacOptions in Test ~= { options ⇒ options filterNot (_ == "-Ywarn-dead-code") // Allow dead code in tests (to support using mockito). }, parallelExecution in Test := false, @@ -34,13 +37,15 @@ object Common { excludeDependencies += "org.apache.logging.log4j" % "log4j-core" ) - def getVersion(version: String): String = version.takeWhile(_ != '-') - - def getRelease(version: String): String = { - version.dropWhile(_ != '-').dropWhile(_ == '-') match { - case "" => "1" - case r if r.contains('-') => sys.error("Version can't have more than one dash") - case r => s"0.1$r" + val stableVersion: Regex = "(\\d+\\.\\d+\\.\\d+)-(\\d+)".r + val betaVersion: Regex = "(\\d+\\.\\d+\\.\\d+)-[Rr][Cc](\\d+)".r + object snapshotVersion { + def unapplySeq(version: String): Option[List[String]] = { + if (version.endsWith("-SNAPSHOT")) { + val v = version.dropRight(9) + stableVersion.unapplySeq(v) orElse betaVersion.unapplySeq(v) + } + else None } } } \ No newline at end of file diff --git a/rpm.sbt b/rpm.sbt index 888e87752..0f5ba9ba9 100644 --- a/rpm.sbt +++ b/rpm.sbt @@ -1,23 +1,46 @@ -import Common._ +import Common.{stableVersion, betaVersion, snapshotVersion} -version in Rpm := getVersion(version.value) -rpmRelease := getRelease(version.value) -rpmVendor := "TheHive Project" +version in Rpm := { + version.value match { + case stableVersion(v1, v2) => v1 + case betaVersion(v1, v2) => v1 + case snapshotVersion(v1, v2) => v1 + case _ => sys.error("Invalid version: " + version.value) + } +} +rpmRelease := { + version.value match { + case stableVersion(_, v2) => v2 + case betaVersion(v1, v2) => "0.1RC" + v2 + case snapshotVersion(v1, v2) => v2 + "-SNAPSHOT" + case _ => sys.error("Invalid version: " + version.value) + } +} +rpmVendor := organizationName.value rpmUrl := organizationHomepage.value.map(_.toString) rpmLicense := Some("AGPL") rpmRequirements += "java-1.8.0-openjdk-headless" + maintainerScripts in Rpm := maintainerScriptsFromDirectory( baseDirectory.value / "package" / "rpm", Seq(RpmConstants.Pre, RpmConstants.Preun, RpmConstants.Postun) ) -linuxPackageMappings in Rpm := configWithNoReplace((linuxPackageMappings in Rpm).value) + linuxPackageSymlinks in Rpm := Nil rpmPrefix := Some(defaultLinuxInstallLocation.value) linuxEtcDefaultTemplate in Rpm := (baseDirectory.value / "package" / "etc_default_cortex").asURL + +linuxPackageMappings in Rpm := configWithNoReplace((linuxPackageMappings in Rpm).value) + packageBin in Rpm := { import scala.sys.process._ - val rpmFile = (packageBin in Rpm).value - s"rpm --addsign $rpmFile".!! + Process("rpm" :: + "--define" :: "_gpg_name TheHive Project" :: + "--define" :: "_signature gpg" :: + "--define" :: "__gpg_check_password_cmd /bin/true" :: + "--define" :: "__gpg_sign_cmd %{__gpg} gpg --batch --no-verbose --no-armor --use-agent --no-secmem-warning -u \"%{_gpg_name}\" -sbo %{__signature_filename} %{__plaintext_filename}" :: + "--addsign" :: rpmFile.toString :: + Nil).!! rpmFile } diff --git a/test/org/thp/cortex/AnalyzersSpec.scala b/test/org/thp/cortex/AnalyzersSpec.scala deleted file mode 100644 index df0c93827..000000000 --- a/test/org/thp/cortex/AnalyzersSpec.scala +++ /dev/null @@ -1,79 +0,0 @@ -package org.thp.cortex - -import java.nio.file.Paths - -import scala.concurrent.Future - -import play.api.Application -import play.api.inject.bind -import play.api.inject.guice.GuiceApplicationBuilder -import play.api.libs.json.{ JsValue, Json } -import play.api.mvc.RequestHeader -import play.api.test._ - -import org.mockito.Matchers -import org.specs2.mock._ -import org.thp.cortex.controllers.AnalyzerCtrl -import org.thp.cortex.models.{ Organization, Roles } -import org.thp.cortex.services.{ OrganizationSrv, UserSrv } - -import org.elastic4play.services.AuthContext - -class AnalyzersSpec extends PlaySpecification with Mockito { - - abstract class WithTheHiveApp(app: Application) extends WithApplication(app) with Injecting { - - def this(builder: GuiceApplicationBuilder ⇒ GuiceApplicationBuilder) { - this({ - val UserSrv = mock[UserSrv].verbose - val adminUser = mock[AuthContext].verbose - adminUser.roles returns Seq(Roles.read, Roles.analyze, Roles.orgAdmin) - adminUser.userId returns "admin" - UserSrv.getFromId(Matchers.any[RequestHeader], Matchers.eq("admin")) returns Future.successful(adminUser) - - builder(GuiceApplicationBuilder().configure("analyzer.path" → Seq(Paths.get("test/resources/analyzers").toAbsolutePath.toString)) - .configure("search.index" → "TEST") - .overrides(bind[UserSrv].toInstance(UserSrv))) - .build() - }) - } - - def this() = this((builder: GuiceApplicationBuilder) ⇒ builder) - } - - "analyzer" should { - - "scan and read definitions" in new WithTheHiveApp { - private val analyzerCtrl = inject[AnalyzerCtrl] - private val result = analyzerCtrl.listDefinitions()(FakeRequest() - .withSession("username" → "admin")) - - status(result) must equalTo(OK) - contentType(result) must beSome("application/json") - contentAsJson(result) - .as[Seq[JsValue]] - .map(j ⇒ (j \ "id").as[String]) must contain(allOf("fakeAnalyzer_1_0")) - } - - "be created from definition" in new WithTheHiveApp(builder ⇒ { - val organizationSrv = mock[OrganizationSrv].verbose - val fakeOrganization = mock[Organization] - organizationSrv.get("fakeOrganization") returns Future.successful(fakeOrganization) - builder.overrides(bind[OrganizationSrv].toInstance(organizationSrv)) - }) { - - private val analyzerCtrl = inject[AnalyzerCtrl] - private val result = analyzerCtrl.create("fakeAnalyzer_1_0")(FakeRequest() - .withSession("username" → "admin") - .withHeaders("Content-Type" → "application/json") - .withJsonBody(Json.obj("name" → "myFakeAnalyzerInstance"))) - - //println(s"DEBUG: body=${contentAsString(result)}") - status(result) must equalTo(CREATED) - contentType(result) must beSome("application/json") - // contentAsJson(result) must_=== Json.obj( - // "" → "") - - } - } -} diff --git a/version.sbt b/version.sbt index ee9594954..1ddd013d0 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "2.1.2" +version in ThisBuild := "3.0.0-RC1-SNAPSHOT" diff --git a/www/package.json b/www/package.json index d92084911..22c06c72e 100755 --- a/www/package.json +++ b/www/package.json @@ -52,14 +52,13 @@ "font-awesome": "^4.7.0", "html-loader": "^0.4.4", "html-webpack-plugin": "^2.22.0", - "imagemin-pngquant": "^5.0.0", "jquery": "^3.2.1", "lodash": "^4.17.4", "manifest-revision-webpack-plugin": "^0.3.0", "moment": "^2.20.1", "ng-storage": "^0.3.2", "ngtemplate-loader": "^1.3.1", - "node-sass": "^3.10.0", + "node-sass": "^4.11.0", "oclazyload": "^1.1.0", "postcss-loader": "^0.13.0", "sass-loader": "^4.0.2",