From 49daf371e92999af7b89fcaa5de927a8b9db5c97 Mon Sep 17 00:00:00 2001 From: Vincent Debergue Date: Thu, 27 Jul 2023 17:29:59 +0200 Subject: [PATCH] Update deps (#449) * ci: add github workflow * update server dependencies * update docker image * create a github release * update node version --- .github/workflows/build.yml | 115 +++++++++++++++++++++++++++++++ .github/workflows/check_code.yml | 22 ++++++ build.sbt | 9 +++ project/Dependencies.scala | 21 +++--- project/DockerSettings.scala | 105 +++++++++++++++------------- project/FrontEnd.scala | 2 +- project/build.properties | 2 +- project/plugins.sbt | 4 +- www/package.json | 7 +- 9 files changed, 224 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/check_code.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..5c4ce6463 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,115 @@ +name: Build +on: + push: + tags: ["*"] + +jobs: + build: + name: Build + runs-on: [ ubuntu-latest ] + steps: + - uses: actions/checkout@v3 + - name: Setup node + uses: actions/setup-node@v3 + timeout-minutes: 15 + continue-on-error: true + with: + node-version: 18.16 + - name: Install bower + run: npm install -g bower + - name: Import PGP Key + run: gpg --batch --import - <<< ${{ secrets.PGP_KEY }} + - name: Write version + id: version + run: | + V=$(sbt -no-colors --error "print version" | awk 'END{print $1}') + echo "version=$V" + echo "version=$V" >> $GITHUB_OUTPUT + - name: Build packages + run: sbt Docker/stage Debian/packageBin Rpm/packageBin Universal/packageBin cortexWithDeps/Docker/stage makeBom + - name: Move packages + run: | + mv target/rpm/RPMS/noarch/cortex*.rpm target/ + mv target/universal/cortex*.zip target/ + + - name: Write docker tags from version + id: tags + run: | + V=${{ steps.version.outputs.version }} + if ( echo $V | grep -qi rc ) + then + echo $( echo $V | sed -re 's/([0-9]+.[0-9]+.[0-9]+)-RC([0-9]+)-([0-9]+)/\1-RC\2,\1-RC\2-\3/' ) > target/tags + else + echo $( echo $V | sed -re 's/([0-9]+).([0-9]+).([0-9]+)-([0-9]+)/\1,\1.\2,\1.\2.\3,\1.\2.\3-\4,latest/' ) > target/tags + fi + echo "tags=$(cat target/tags)" >> $GITHUB_OUTPUT + + - name: Build list of additional tags + id: additional-tags + uses: actions/github-script@v6 + with: + script: | + core.setOutput('tags', `${{ steps.tags.outputs.tags }}`.split(",").join("\n")) + + - name: Generate full docker tags + id: meta + uses: docker/metadata-action@v4 + with: + images: | + name=${{ secrets.HARBOR_REGISTRY }}/thehiveproject/cortex + name=thehiveproject/cortex + tags: | + ${{ steps.additional-tags.outputs.tags }} + + - name: Login to Harbor + uses: docker/login-action@v2 + with: + registry: ${{ secrets.HARBOR_REGISTRY }} + username: ${{ secrets.HARBOR_USERNAME }} + password: ${{ secrets.HARBOR_PASSWORD }} + + - name: Login to Dockerhub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push docker + uses: docker/build-push-action@v3 + with: + context: target/docker/stage + push: true + tags: ${{ steps.meta.outputs.tags }} + + - name: Build and push fat docker + uses: docker/build-push-action@v3 + with: + context: target/docker-withdeps/target/docker/stage + push: true + tags: ${{ steps.meta.outputs.tags }} + + - name: Make github release + uses: "softprops/action-gh-release@v1" + id: make-release + with: + generate_release_notes: true + files: | + target/cortex*.deb + target/cortex*.rpm + target/cortex*.zip + target/cortex*.bom.xml + + notify: + needs: [ build ] + runs-on: [ ubuntu-latest ] + if: always() + steps: + - name: Slack notification + uses: Gamesight/slack-workflow-status@master + with: + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}} + channel: "#ci-cortex" + name: Cortex build + include_commit_message: true + include_jobs: true \ No newline at end of file diff --git a/.github/workflows/check_code.yml b/.github/workflows/check_code.yml new file mode 100644 index 000000000..f3aa44ffd --- /dev/null +++ b/.github/workflows/check_code.yml @@ -0,0 +1,22 @@ +name: Check Code +on: + workflow_dispatch: + workflow_call: + push: + branches: [master, develop] +jobs: + check: + name: Check + runs-on: [ ubuntu-latest ] + steps: + - uses: actions/checkout@v3 + - name: Setup node + uses: actions/setup-node@v3 + timeout-minutes: 15 + continue-on-error: true + with: + node-version: 18.16 + - name: Install bower + run: npm install -g bower + - name: Run tests + run: sbt test Universal/packageBin diff --git a/build.sbt b/build.sbt index bfa0e4367..94b60e4ae 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,14 @@ import Common._ +ThisBuild / scalaVersion := Dependencies.scalaVersion +ThisBuild / evictionErrorLevel := util.Level.Warn + +ThisBuild / dependencyOverrides ++= Seq( + Dependencies.Play.twirl, + "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5", + "org.apache.commons" % "commons-compress" % "1.23.0", + "com.google.guava" % "guava" % "32.1.1-jre" +) lazy val cortex = (project in file(".")) .enablePlugins(PlayScala) .settings(projectSettings) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1e4701281..7d07b526f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,21 +5,22 @@ object Dependencies { object Play { val version = play.core.PlayVersion.current - val ws = "com.typesafe.play" %% "play-ws" % version - val ahc = "com.typesafe.play" %% "play-ahc-ws" % version - val cache = "com.typesafe.play" %% "play-ehcache" % version - val test = "com.typesafe.play" %% "play-test" % version - val specs2 = "com.typesafe.play" %% "play-specs2" % version + val ws = "com.typesafe.play" %% "play-ws" % version exclude ("com.typesafe.play", "play-ws-standalone-xml") + val ahc = "com.typesafe.play" %% "play-ahc-ws" % version + val cache = "com.typesafe.play" %% "play-ehcache" % version + val test = "com.typesafe.play" %% "play-test" % version + val specs2 = "com.typesafe.play" %% "play-specs2" % version val filters = "com.typesafe.play" %% "filters-helpers" % version - val guice = "com.typesafe.play" %% "play-guice" % version + val guice = "com.typesafe.play" %% "play-guice" % version + val twirl = "com.typesafe.play" %% "twirl-api" % "1.5.2" } - val scalaGuice = "net.codingwell" %% "scala-guice" % "5.1.0" + val scalaGuice = "net.codingwell" %% "scala-guice" % "5.1.1" - val reflections = "org.reflections" % "reflections" % "0.10.2" - val zip4j = "net.lingala.zip4j" % "zip4j" % "2.10.0" + val reflections = "org.reflections" % "reflections" % "0.10.2" + val zip4j = "net.lingala.zip4j" % "zip4j" % "2.11.5" val elastic4play = "org.thehive-project" %% "elastic4play" % "1.13.6" - val dockerClient = "com.spotify" % "docker-client" % "8.14.4" + val dockerClient = "com.spotify" % "docker-client" % "8.16.0" val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % play.core.PlayVersion.akkaVersion val akkaClusterTyped = "com.typesafe.akka" %% "akka-cluster-typed" % play.core.PlayVersion.akkaVersion } diff --git a/project/DockerSettings.scala b/project/DockerSettings.scala index 10f8e26ba..8a0262eb9 100644 --- a/project/DockerSettings.scala +++ b/project/DockerSettings.scala @@ -29,14 +29,21 @@ object DockerSettings { case (_, filepath) => filepath == "/opt/cortex/conf/application.conf" }), dockerCommands := Seq( - Cmd("FROM", "openjdk:8-slim"), + Cmd("FROM", "debian:bullseye-slim"), Cmd("LABEL", "MAINTAINER=\"TheHive Project \"", "repository=\"https://github.com/TheHive-Project/TheHive\""), Cmd("WORKDIR", "/opt/cortex"), + Cmd("ENV", "JAVA_HOME", "/usr/lib/jvm/java-11-amazon-corretto"), // format: off Cmd("RUN", "apt", "update", "&&", "apt", "upgrade", "-y", "&&", - "apt", "install", "-y", "iptables", "lxc", "wget", "&&", + "apt", "install", "-y", "iptables", "lxc", "wget", "curl", "gnupg", "&&", + // install java corretto + "curl", "-fL", "https://apt.corretto.aws/corretto.key", "|", "gpg", "--dearmor", "-o", "/usr/share/keyrings/corretto.gpg", "&&", + "echo", "'deb [signed-by=/usr/share/keyrings/corretto.gpg] https://apt.corretto.aws stable main'", ">", "/etc/apt/sources.list.d/corretto.list", "&&", + "mkdir", "-p", "/usr/share/man/man1", "||", "true", "&&", + "apt", "update", "&&", "apt", "install", "-y", "java-11-amazon-corretto-jdk", "&&", + // setup for docker "apt", "autoclean", "-y", "-q", "&&", "apt", "autoremove", "-y", "-q", "&&", "wget", "-q", "-O", "-", "https://download.docker.com/linux/static/stable/x86_64/docker-18.09.0.tgz", "|", @@ -46,8 +53,10 @@ object DockerSettings { "addgroup", "--system", "docker", "&&", "echo", "dockremap:165536:65536", ">>", "/etc/subuid", "&&", "echo", "dockremap:165536:65536", ">>", "/etc/subgid", "&&", + // cleanup "rm", "-rf", "/var/lib/apt/lists/*", "&&", "(", "type", "groupadd", "1>/dev/null", "2>&1", "&&", + // setup cortex user "groupadd", "-g", "1001", "cortex", "||", "addgroup", "-g", "1001", "-S", "cortex", ")", "&&", @@ -72,52 +81,52 @@ object DockerSettings { Cmd( "RUN", """ - | apt update && - | apt upgrade -y && - | apt install -y -q --no-install-recommends --no-install-suggests - | wkhtmltopdf libfuzzy-dev libimage-exiftool-perl curl unzip - | libboost-regex-dev - | libboost-program-options-dev - | libboost-system-dev libboost-filesystem-dev libssl-dev - | build-essential cmake python3-dev python2-dev - | git python3 python3-pip libffi-dev libjpeg62-turbo-dev libtiff5-dev - | libopenjp2-7-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev - | tcl8.6-dev tk8.6-dev python3-tk libharfbuzz-dev libfribidi-dev - | libxcb1-dev python2.7 && - | rm -rf /var/lib/apt/lists/* && - | curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output /tmp/get-pip.py && - | python2.7 /tmp/get-pip.py && - | pip2 install -U setuptools && - | pip3 install -U setuptools && - | ln -sf python3 /usr/bin/python && - | hash -r && - | git clone https://github.com/JusticeRage/Manalyze.git /tmp/Manalyze && - | cd /tmp/Manalyze && - | cmake . && - | make -j5 && - | cd /tmp/Manalyze/bin/yara_rules && - | pip3 install requests && - | python3 update_clamav_signatures.py && - | cd /tmp/Manalyze && - | make install && - | cd / && - | rm -rf /tmp/Manalyze && - | curl -SL https://github.com/fireeye/flare-floss/releases/download/v1.7.0/floss-v1.7.0-linux.zip - | --output /tmp/floss.zip && - | unzip /tmp/floss.zip -d /usr/bin && - | rm /tmp/floss.zip && - | git clone https://github.com/TheHive-Project/Cortex-Analyzers.git /tmp/analyzers && - | cat $(find /tmp/analyzers -name requirements.txt) | sort -u | while read I ; - | do - | pip2 install $I || true && - | pip3 install $I || true ; - | done && - | for I in $(find /tmp/analyzers -name requirements.txt) ; - | do - | pip2 install -r $I || true && - | pip3 install -r $I || true ; - | done && - | rm -rf /tmp/analyzers + | apt update && + | apt upgrade -y && + | apt install -y -q --no-install-recommends --no-install-suggests + | wkhtmltopdf libfuzzy-dev libimage-exiftool-perl curl unzip + | libboost-regex-dev + | libboost-program-options-dev + | libboost-system-dev libboost-filesystem-dev libssl-dev + | build-essential cmake python3-dev python2-dev + | git python3 python3-pip libffi-dev libjpeg62-turbo-dev libtiff5-dev + | libopenjp2-7-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev + | tcl8.6-dev tk8.6-dev python3-tk libharfbuzz-dev libfribidi-dev + | libxcb1-dev python2.7 && + | rm -rf /var/lib/apt/lists/* && + | curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output /tmp/get-pip.py && + | python2.7 /tmp/get-pip.py && + | pip2 install -U setuptools && + | pip3 install -U setuptools && + | ln -sf python3 /usr/bin/python && + | hash -r && + | git clone https://github.com/JusticeRage/Manalyze.git /tmp/Manalyze && + | cd /tmp/Manalyze && + | cmake . && + | make -j5 && + | cd /tmp/Manalyze/bin/yara_rules && + | pip3 install requests && + | python3 update_clamav_signatures.py && + | cd /tmp/Manalyze && + | make install && + | cd / && + | rm -rf /tmp/Manalyze && + | curl -SL https://github.com/fireeye/flare-floss/releases/download/v1.7.0/floss-v1.7.0-linux.zip + | --output /tmp/floss.zip && + | unzip /tmp/floss.zip -d /usr/bin && + | rm /tmp/floss.zip && + | git clone https://github.com/TheHive-Project/Cortex-Analyzers.git /tmp/analyzers && + | cat $(find /tmp/analyzers -name requirements.txt) | sort -u | while read I ; + | do + | pip2 install $I || true && + | pip3 install $I || true ; + | done && + | for I in $(find /tmp/analyzers -name requirements.txt) ; + | do + | pip2 install -r $I || true && + | pip3 install -r $I || true ; + | done && + | rm -rf /tmp/analyzers """.stripMargin.split("\\s").filter(_.nonEmpty): _* ) ) diff --git a/project/FrontEnd.scala b/project/FrontEnd.scala index de418c40f..294a947aa 100644 --- a/project/FrontEnd.scala +++ b/project/FrontEnd.scala @@ -18,7 +18,7 @@ object FrontEnd extends AutoPlugin { val s = streams.value s.log.info("Building front-end ...") s.log.info("npm install") - Process("npm" :: "install" :: Nil, baseDirectory.value / "www") ! s.log + Process("npm" :: "install" :: "--legacy-peer-deps" :: Nil, baseDirectory.value / "www") ! s.log s.log.info("npm run build") Process("npm" :: "run" :: "build" :: Nil, baseDirectory.value / "www") ! s.log val dir = baseDirectory.value / "www" / "dist" diff --git a/project/build.properties b/project/build.properties index baf5ff3ec..40b3b8e7b 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.7 +sbt.version=1.9.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index 7207b3c38..758cc8da9 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,9 @@ // Comment to get more information during initialization logLevel := Level.Info +evictionErrorLevel := util.Level.Warn // The Play plugin -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.16") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.19") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") addSbtPlugin("org.thehive-project" % "sbt-github-changelog" % "0.4.0") +addSbtPlugin("io.github.siculo" %% "sbt-bom" % "0.3.0") diff --git a/www/package.json b/www/package.json index d38b1847a..c5f90d233 100755 --- a/www/package.json +++ b/www/package.json @@ -17,6 +17,8 @@ }, "dependencies": { "@uirouter/angularjs": "^1.0.22", + "@uirouter/core": "^6.1.0", + "@uirouter/rx": "^1.0.0", "angular": "^1.7.8", "angular-base64-upload": "^0.1.23", "angular-bootstrap-multiselect": "git+https://github.com/bentorfs/angular-bootstrap-multiselect.git", @@ -67,12 +69,13 @@ "html-webpack-plugin": "2.22.0", "manifest-revision-webpack-plugin": "0.3.0", "ngtemplate-loader": "^1.3.1", - "node-sass": "^4.12.0", + "node-sass": "npm:sass@^1.62.0", "postcss-loader": "^0.13.0", + "sass": "^1.62.0", "sass-loader": "^4.0.2", "style-loader": "^0.13.1", "url-loader": "^0.5.9", "webpack": "3.5.0", "webpack-dev-server": "2.2.0" } -} +} \ No newline at end of file