From 8cfa56bdbcc7ecbf4fb61250569149eb32e88ee4 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 2 May 2017 10:59:37 +0200 Subject: [PATCH] #20 package Cortex with rpm, deb and docker --- build.sbt | 157 +++++++++++++++++++++--------------- conf/logback.xml | 39 +++++++++ install/cortex | 116 ++++++++++++++++---------- install/cortex.conf | 74 ++++++++--------- install/cortex.service | 14 ++-- install/docker/entrypoint | 55 +++++++++++++ project/Bintray.scala | 6 +- project/BuildSettings.scala | 2 +- 8 files changed, 300 insertions(+), 163 deletions(-) create mode 100644 conf/logback.xml mode change 100644 => 100755 install/docker/entrypoint diff --git a/build.sbt b/build.sbt index b149c99fc..706bb08db 100644 --- a/build.sbt +++ b/build.sbt @@ -15,9 +15,9 @@ libraryDependencies ++= Seq( resolvers += "scalaz-bintray" at "http://dl.bintray.com/scalaz/releases" -releaseVersionUIFile := baseDirectory.value / "ui" / "package.json" +Release.releaseVersionUIFile := baseDirectory.value / "ui" / "package.json" -changelogFile := baseDirectory.value / "CHANGELOG.md" +Release.changelogFile := baseDirectory.value / "CHANGELOG.md" publishArtifact in (Compile, packageDoc) := false @@ -30,83 +30,110 @@ run := { (run in Compile).evaluated frontendDev.value } - mappings in packageBin in Assets ++= frontendFiles.value // Install files // - -mappings in Universal ++= { - val dir = baseDirectory.value / "install" - (dir.***) pair relativeTo(dir.getParentFile) +mappings in Universal ~= { + _.flatMap { + case (file, "conf/application.conf") => Nil + case (file, "conf/apllication.sample") => Seq(file -> "conf/application.conf") + case other => Seq(other) + } ++ Seq( + file("install/cortex.service") -> "install/cortex.service", + file("install/cortex.conf") -> "install/cortex.conf", + file("install/cortex") -> "install/cortex" + ) } -// Release // -import ReleaseTransformations._ - -import Release._ - -bintrayOrganization := Some("cert-bdf") - -bintrayRepository := "cortex" - -publish := { - publishRelease.value - publishLatest.value +// Package // +maintainer := "Thomas Franco + val mappings = pm.mappings.filterNot { + case (file, path) => path.startsWith("/opt/cortex/install") || path.startsWith("/opt/cortex/conf") + } + com.typesafe.sbt.packager.linux.LinuxPackageMapping(mappings, pm.fileData).withConfig() +} :+ packageMapping( + file("install/cortex.service") -> "/etc/systemd/system/cortex.service", + file("install/cortex.conf") -> "/etc/init/cortex.conf", + file("install/cortex") -> "/etc/init.d/cortex", + file("conf/application.sample") -> "/etc/cortex/application.conf", + file("conf/logback.xml") -> "/etc/cortex/logback.xml" +).withConfig() } -releaseProcess := Seq[ReleaseStep]( - checkUncommittedChanges, - checkSnapshotDependencies, - getVersionFromBranch, - runTest, - releaseMerge, - checkoutMaster, - setReleaseVersion, - setReleaseUIVersion, - generateChangelog, - commitChanges, - tagRelease, - publishArtifacts, - checkoutDevelop, - setNextVersion, - setNextUIVersion, - commitChanges, - //commitNextVersion, - pushChanges) - +packageBin := { + (packageBin in Universal).value + (packageBin in Debian).value + //(packageBin in Rpm).value +} +// DEB // +//debianPackageRecommends := Seq("elasticsearch") +debianPackageDependencies += "java8-runtime-headless | java8-runtime" +maintainerScripts in Debian := maintainerScriptsFromDirectory( + baseDirectory.value / "install" / "debian", + Seq(DebianConstants.Postinst, DebianConstants.Prerm, DebianConstants.Postrm) +) +linuxEtcDefaultTemplate in Debian := (baseDirectory.value / "install" / "etc_default_cortex").asURL +linuxMakeStartScript in Debian := None + +// RPM // +rpmRelease := "8" +rpmVendor in Rpm := "TheHive Project" +rpmUrl := Some("http://thehive-project.org/") +rpmLicense := Some("AGPL") +rpmRequirements += "java-1.8.0-openjdk-headless" +maintainerScripts in Rpm := maintainerScriptsFromDirectory( + baseDirectory.value / "install" / "rpm", + Seq(RpmConstants.Pre, RpmConstants.Preun, RpmConstants.Postun) +) +linuxPackageSymlinks in Rpm := Nil +rpmPrefix := Some(defaultLinuxInstallLocation.value) +linuxEtcDefaultTemplate in Rpm := (baseDirectory.value / "install" / "etc_default_cortex").asURL // DOCKER // +import com.typesafe.sbt.packager.docker.{ Cmd, ExecCmd } -dockerBaseImage := "certbdf/thehive:latest" - -packageName in Docker := "thehive-cortex" - +defaultLinuxInstallLocation in Docker := "/opt/cortex" dockerRepository := Some("certbdf") - dockerUpdateLatest := true - -mappings in Universal += file("docker/entrypoint") -> "bin/entrypoint" - -mappings in Universal ~= { _.filterNot { - case (_, name) => name.startsWith("conf/") && name != "conf/keepme" -}} -import com.typesafe.sbt.packager.docker.{ ExecCmd, Cmd } - -dockerCommands := dockerCommands.value.map { - case ExecCmd("ENTRYPOINT", _*) => ExecCmd("ENTRYPOINT", "bin/entrypoint") - case cmd => cmd +dockerEntrypoint := Seq("/opt/cortex/entrypoint") +dockerExposedPorts := Seq(9000) +mappings in Docker ++= Seq( + file("install/docker/entrypoint") -> "/opt/cortex/entrypoint", + file("conf/logback.xml") -> "/etc/cortex/logback.xml", + file("install/empty") -> "/var/log/cortex/application.log") +mappings in Docker ~= (_.filterNot { + case (_, filepath) => filepath == "/opt/cortex/conf/application.conf" +}) + +dockerCommands ~= { dc => + val (dockerInitCmds, dockerTailCmds) = dc.splitAt(4) + dockerInitCmds ++ + Seq( + Cmd("USER", "root"), + ExecCmd("RUN", "bash", "-c", + "apt-get update && " + + "apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git && " + + "cd /opt && " + + "git clone -b develop https://github.com/CERT-BDF/Cortex-Analyzers.git && " + + "pip install $(cat Cortex-Analyzers/analyzers/*/requirements.txt | grep -v hashlib | sort -u)"), + Cmd("ADD", "var", "/var"), + Cmd("ADD", "etc", "/etc"), + ExecCmd("RUN", "chown", "-R", "daemon:daemon", "/var/log/cortex")) ++ + dockerTailCmds } -dockerCommands := dockerCommands.value.head +: - Cmd("USER", "root") +: - ExecCmd("RUN", "bash", "-c", - "apt-get update && " + - "apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git && " + - "cd /opt && " + - "git clone https://github.com/CERT-BDF/Cortex-Analyzers.git && " + - "pip install $(cat Cortex-Analyzers/analyzers/*/requirements.txt | grep -v hashlib | sort -u)") +: - Cmd("EXPOSE", "9001") +: - dockerCommands.value.tail +// Bintray // +bintrayOrganization := Some("cert-bdf") +bintrayRepository := "cortex" +publish := { + (publish in Docker).value + PublishToBinTray.publishRelease.value + PublishToBinTray.publishLatest.value +} // Scalariform // import scalariform.formatter.preferences._ diff --git a/conf/logback.xml b/conf/logback.xml new file mode 100644 index 000000000..2f1ff1748 --- /dev/null +++ b/conf/logback.xml @@ -0,0 +1,39 @@ + + + + + + + /var/log/cortex/application.log + + %date [%level] from %logger in %thread - %message%n%xException + + + + + + %coloredLevel %logger{15} - %message%n%xException{10} + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/install/cortex b/install/cortex index 29b24aedc..17a896ad8 100755 --- a/install/cortex +++ b/install/cortex @@ -5,73 +5,101 @@ # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: Cortex: a FOSS Information Security Incident Management Platform +# Short-Description: cortex: Powerful Observable Analysis Engine ### END INIT INFO NAME=cortex -DAEMONDIR=/opt/cortex -DAEMON=$DAEMONDIR/bin/cortex -PIDDIR=/var/run/cortex -PIDFILE=$PIDDIR/pid -DAEMONUSER=cortex -CONFIGFILE=/etc/cortex/application.conf -LISTENPORT=9000 -DAEMON_OPTS= +DAEMON=/opt/cortex/bin/cortex +PIDFILE=/var/run/cortex/pid +DAEMON_USER=cortex +DAEMON_ARGS="-Dconfig.file=/etc/cortex/application.conf -Dlogger.file=/etc/cortex/logback.xml -Dpidfile.path=/dev/null" PATH=/sbin:/bin:/usr/sbin:/usr/bin -test -x $DAEMON || exit 0 +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 -if test -f /etc/default/$NAME -then - . /etc/default/$NAME -fi +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. . /lib/lsb/init-functions +# +# Function that starts the daemon/service +# +do_start() { + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --pidfile $PIDFILE --user $DAEMON_USER --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE --user $DAEMON_USER --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() { + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON + [ "$?" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + check_for_upstart() { if init_is_upstart; then + echo "Your are using upstart. This script should not be used" exit $1 fi } -check_config_file() { - if test ! -e $CONFIGFILE - then - mkdir -p $(dirname $CONFIGFILE) 2>/dev/null || true - cat >> $CONFIGFILE <<- _EOF_ - # Secret key - # ~~~~~ - # The secret key is used to secure cryptographics functions. - # If you deploy your application to several instances be sure to use the same key! - play.crypto.secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)" -_EOF_ - chown $DAEMONUSER $CONFIGFILE - chmod o-rwx $CONFIGFILE - fi -} - case "$1" in start) check_for_upstart 1 - check_config_file - log_daemon_msg "Starting Cortex" $NAME || true - if start-stop-daemon --start --quiet --oknodo --pidfile $PIDFILE --chdir $DAEMONDIR --exec $DAEMON -- -Dconfig.file=$CONFIGFILE -Dhttp.port=$LISTENPORT -Dpidfile.path=$PIDFILE $DAEMON_OPTS - then - log_end_msg 0 || true - else - log_end_msg 1 || true + if [ "$ENABLED" = no ]; then + echo "The service cortex is disabled (/etc/default/cortex)" + exit 1 fi + + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac ;; stop) check_for_upstart 1 - log_daemon_msg "Stopping Cortex" $NAME || true - if start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE - then - log_end_msg 0 || true - else - log_end_msg 1 || true - fi + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac ;; status) diff --git a/install/cortex.conf b/install/cortex.conf index 4cf80c3db..c12bd23fe 100644 --- a/install/cortex.conf +++ b/install/cortex.conf @@ -1,49 +1,41 @@ -# cortex - a FOSS Information Security Incident Management Platform -# +# generated upstart config + +description "Powerful Observable Analysis Engine" +author "Thomas Franco " -description "Cortex" +# Stanzas +# +# Stanzas control when and how a process is started and stopped +# See a list of stanzas here: http://upstart.ubuntu.com/wiki/Stanzas#respawn +# When to start the service start on runlevel [2345] -stop on runlevel [!2345] + +# When to stop the service +stop on runlevel [016] + + +# Automatically restart process if crashed. Tries 1 times every 60 seconds respawn -respawn limit 10 5 -umask 022 - - -env DAEMON=/opt/cortex/bin/cortex -env PIDFILE=/var/run/cortex.pid -env DAEMONUSER=cortex -env CONFIGFILE=/etc/cortex/application.conf -env LISTENPORT=9000 -env DAEMON_OPTS= -env DEFAULTFILE=/etc/default/cortex - - -pre-start script - echo Starting Cortex - if [ -f "$DEFAULTFILE" ]; then - . "$DEFAULTFILE" - fi - - if test ! -e $CONFIGFILE - then - mkdir -p $(dirname $CONFIGFILE) 2>/dev/null || true - cat >> $CONFIGFILE <<- _EOF_ - # Secret key - # ~~~~~ - # The secret key is used to secure cryptographics functions. - # If you deploy your application to several instances be sure to use the same key! - play.crypto.secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)" -_EOF_ - chown $DAEMONUSER $CONFIGFILE - chmod o-rwx $CONFIGFILE - fi - - test -x $DAEMON || { echo $DAEMON not found; stop; exit 0; } -end script +respawn limit 1 60 + +normal exit 0 + +# set the working directory of the job processes +chdir /opt/cortex +# Start the process script - cd $(dirname $DAEMON)/.. - $DAEMON -Dconfig.file=$CONFIGFILE -Dhttp.port=$LISTENPORT -Dpidfile.path=$PIDFILE $DAEMON_OPTS + DAEMON_USER="cortex" + DAEMON_ARGS="-Dconfig.file=/etc/cortex/application.conf -Dlogger.file=/etc/cortex/logback.xml -Dpidfile.path=/dev/null" + + . /etc/default/cortex + + if [ "$ENABLED" = no ]; then + echo "The service cortex is disabled (/etc/default/cortex)" + exit 0 + fi + + exec sudo -u $DAEMON_USER bin/cortex $DAEMON_ARGS end script diff --git a/install/cortex.service b/install/cortex.service index e4e37d90c..3f235d827 100644 --- a/install/cortex.service +++ b/install/cortex.service @@ -1,23 +1,19 @@ [Unit] -Description=Cortex -Documentation=https://cortex-project.org +Description=cortex +Documentation=https://thehive-project.org Wants=network-online.target After=network-online.target [Service] -Environment=PID_DIR=/var/run/cortex WorkingDirectory=/opt/cortex User=cortex Group=cortex -RuntimeDirectory=cortex -RuntimeDirectoryMode=0750 - ExecStart=/opt/cortex/bin/cortex \ - -Dconfig.file=/etc/cortex/application.conf \ - -Dhttp.port=9000 \ - -Dpidfile.path=${PID_DIR}/pid + -Dconfig.file=/etc/cortex/application.conf \ + -Dlogger.file=/etc/cortex/logback.xml \ + -Dpidfile.path=/dev/null StandardOutput=journal StandardError=inherit diff --git a/install/docker/entrypoint b/install/docker/entrypoint old mode 100644 new mode 100755 index e69de29bb..08f2fd167 --- a/install/docker/entrypoint +++ b/install/docker/entrypoint @@ -0,0 +1,55 @@ +#!/bin/bash + +CONFIG_SECRET=1 +CONFIG=1 +CONFIG_FILE=/etc/cortex/application.conf +ANALYZER_PATH=/opt/Cortex-Analyzers/analyzers + +function usage { + cat <<- _EOF_ + Available options: + --no-config | do not try to configure Cortex (add secret and analyzers location) + --no-config-secret | do not add random secret to configuration + --secret | secret to secure sessions + --analyzer-path | where analyzers are located + _EOF_ + exit 1 +} + +STOP=0 +while test $# -gt 0 -o $STOP = 1 +do + case "$1" in + "--no-config") CONFIG=0;; + "--no-config-secret") CONFIG_SECRET=0;; + "--secret") shift; SECRET=$1;; + "--analyzer-path") shift; ANALYZER_PATH=$1;; + "--") STOP=1;; + *) usage + esac + shift +done + +if test $CONFIG = 1 +then + CONFIG_FILE=$(mktemp).conf + if test $CONFIG_SECRET = 1 + then + if test -z "$SECRET" + then + SECRET=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1) + fi + echo Using secret: $SECRET + echo play.crypto.secret=\"$SECRET\" >> $CONFIG_FILE + fi + + echo analyzer.path=\"$ANALYZER_PATH\" >> $CONFIG_FILE + + echo 'include file("/etc/cortex/application.conf")' >> $CONFIG_FILE +fi + +exec bin/cortex \ + -Dconfig.file=$CONFIG_FILE \ + -Dlogger.file=/etc/cortex/logback.xml \ + -Dpidfile.path=/dev/null \ + $@ diff --git a/project/Bintray.scala b/project/Bintray.scala index 7c3ad44e2..0d12083f2 100644 --- a/project/Bintray.scala +++ b/project/Bintray.scala @@ -15,8 +15,8 @@ import bintray.BintrayKeys.{ bintrayEnsureCredentials, bintrayOrganization, bint import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport.Universal object PublishToBinTray extends Plugin { - val publishRelease = taskKey[Unit]("Publish binary in bintray") - val publishLatest = taskKey[Unit]("Publish latest binary in bintray") + val publishRelease: TaskKey[Unit] = taskKey[Unit]("Publish binary in bintray") + val publishLatest: TaskKey[Unit] = taskKey[Unit]("Publish latest binary in bintray") override def settings = Seq( publishRelease := { @@ -54,7 +54,7 @@ object PublishToBinTray extends Plugin { private def asStatusAndBody = new FunctionHandler({ r => (r.getStatusCode, r.getResponseBody) }) - def removeVersion(credential: BintrayCredentials, org: Option[String], repoName: String, packageName: String, version: String, log: Logger) = { + def removeVersion(credential: BintrayCredentials, org: Option[String], repoName: String, packageName: String, version: String, log: Logger): Unit = { val BintrayCredentials(user, key) = credential val client: Client = Client(user, key, new Http()) val repo: Client#Repo = client.repo(org.getOrElse(user), repoName) diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index cbb82d3a2..01995195b 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -2,7 +2,7 @@ import sbt.Keys._ import sbt._ object BasicSettings extends AutoPlugin { - override def trigger = allRequirements + override def trigger: PluginTrigger = allRequirements override def projectSettings = Seq( organization := "org.cert-bdf",