From 334feb14f23f4b209ab74a55d528c377917d7532 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 12 Sep 2018 11:12:01 +0200 Subject: [PATCH 1/7] #128 Search analyzer using its name where MISP query a job --- app/org/thp/cortex/services/MispSrv.scala | 8 ++++++-- docker.sbt | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/org/thp/cortex/services/MispSrv.scala b/app/org/thp/cortex/services/MispSrv.scala index 31b2fc003..3883c10d6 100644 --- a/app/org/thp/cortex/services/MispSrv.scala +++ b/app/org/thp/cortex/services/MispSrv.scala @@ -17,6 +17,7 @@ import org.apache.commons.codec.binary.Base64 import org.thp.cortex.models._ import org.thp.cortex.services.AuditActor.Register +import org.elastic4play.NotFoundError import org.elastic4play.services._ @Singleton @@ -58,12 +59,15 @@ class MispSrv @Inject() ( } def query(module: String, mispType: String, data: String)(implicit authContext: AuthContext): Future[JsObject] = { + import org.elastic4play.services.QueryDSL._ + val artifact: Either[String, Attachment] = toArtifact(mispType, data) val duration = 20.minutes // TODO configurable for { - analyzer ← workerSrv.get(module) - job ← jobSrv.create(analyzer, mispType2dataType(mispType), artifact, 0, 0, "", JsObject.empty, None, force = false) + analyzer ← workerSrv.findAnalyzersForUser(authContext.userId, "name" ~= module, Some("0-1"), Nil)._1.runWith(Sink.headOption) + job ← analyzer.map(jobSrv.create(_, mispType2dataType(mispType), artifact, 0, 0, "", JsObject.empty, None, force = false)) + .getOrElse(Future.failed(NotFoundError(s"Module $module not found"))) _ ← auditActor.ask(Register(job.id, duration))(Timeout(duration)) updatedJob ← jobSrv.getForUser(authContext.userId, job.id) mispOutput ← toMispOutput(authContext.userId, updatedJob) diff --git a/docker.sbt b/docker.sbt index 7cbcd4f5c..ba4224de3 100644 --- a/docker.sbt +++ b/docker.sbt @@ -28,7 +28,7 @@ dockerCommands ~= { dc => ExecCmd("RUN", "bash", "-c", "apt-get update && " + "apt-get install -y --no-install-recommends python-pip python2.7-dev python3-pip python3-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev && " + - "pip install -U pip setuptools && " + + "pip2 install -U pip setuptools && " + "pip3 install -U pip setuptools && " + "cd /opt && " + "git clone https://github.com/TheHive-Project/Cortex-Analyzers.git && " + From 60956ae42b5d68eaee6420bda07adb362ae4cc02 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 12 Sep 2018 11:21:03 +0200 Subject: [PATCH 2/7] #129 remove temporary file when the job ends --- app/org/thp/cortex/services/JobSrv.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/org/thp/cortex/services/JobSrv.scala b/app/org/thp/cortex/services/JobSrv.scala index 32640dca6..5e36466f8 100644 --- a/app/org/thp/cortex/services/JobSrv.scala +++ b/app/org/thp/cortex/services/JobSrv.scala @@ -1,10 +1,10 @@ package org.thp.cortex.services import java.io.{ ByteArrayOutputStream, InputStream } -import java.nio.file.Files +import java.nio.file.{ Files, Paths } import java.util.Date -import javax.inject.{ Inject, Singleton } +import javax.inject.{ Inject, Singleton } import akka.NotUsed import akka.actor.ActorSystem import akka.stream.Materializer @@ -403,6 +403,11 @@ class JobSrv( val errorMessage = (error + output).take(8192) endJob(job, JobStatus.Failure, Some(s"Invalid output\n$errorMessage")) } + finally { + (input \ "file").asOpt[String].foreach { filename ⇒ + Files.deleteIfExists(Paths.get(filename)) + } + } }(executionContext) } From fb78c1b6d7833bd8a905c3e88b2e06e6f7176204 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 12 Sep 2018 14:35:18 +0200 Subject: [PATCH 3/7] #124 Display PAP in analyzers and responders admin pages --- www/src/app/core/directives/tlp/tlp.directive.js | 13 ++++++++----- .../components/analyzers/analyzers-list.html | 12 ++++++++---- .../components/responders/responders-list.html | 12 ++++++++---- www/src/app/pages/jobs/components/job.details.html | 2 +- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/www/src/app/core/directives/tlp/tlp.directive.js b/www/src/app/core/directives/tlp/tlp.directive.js index 247ad9793..642b38b8e 100644 --- a/www/src/app/core/directives/tlp/tlp.directive.js +++ b/www/src/app/core/directives/tlp/tlp.directive.js @@ -5,7 +5,7 @@ import _ from 'lodash/core'; import tpl from './tlp.html'; import './tlp.scss'; -export default function(app) { +export default function (app) { app.directive('tlp', tlp); function tlp(Tlps) { @@ -14,7 +14,8 @@ export default function(app) { return { templateUrl: tpl, scope: { - value: '=' + value: '=', + namespace: '@' }, replace: true, link: linkFn @@ -26,12 +27,14 @@ export default function(app) { scope.tlpClass = 'label-none'; scope.tlp = 'None'; } else { - const temp = (_.find(Tlps, { value: v }) || {}).key; + const temp = (_.find(Tlps, { + value: v + }) || {}).key; scope.tlpClass = `label-${(temp || '').toLowerCase()}`; - scope.tlp = `TLP:${temp}`; + scope.tlp = `${scope.namespace || 'TLP'}:${temp}`; } }); } } -} +} \ No newline at end of file diff --git a/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html b/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html index 4747c314a..7764ef11b 100644 --- a/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html +++ b/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html @@ -42,8 +42,9 @@

Analyzer
-
Max TLP
-
Rate Limit
+
Max TLP
+
Max PAP
+
Rate Limit
@@ -66,10 +67,13 @@

{{definition.description}}
-
+
-
+
+ +
+
{{$ctrl.activeAnalyzers[def].rate}} per {{$ctrl.activeAnalyzers[def].rateUnit}} diff --git a/www/src/app/pages/admin/organizations/components/responders/responders-list.html b/www/src/app/pages/admin/organizations/components/responders/responders-list.html index 9d5207dbb..ca8bc39a0 100644 --- a/www/src/app/pages/admin/organizations/components/responders/responders-list.html +++ b/www/src/app/pages/admin/organizations/components/responders/responders-list.html @@ -47,8 +47,9 @@

Responders
-
Max TLP
-
Rate Limit
+
Max TLP
+
Max PAP
+
Rate Limit
@@ -70,10 +71,13 @@

{{definition.description}}
-
+
-
+
+ +
+
{{$ctrl.activeResponders[def].rate}} per {{$ctrl.activeResponders[def].rateUnit}} diff --git a/www/src/app/pages/jobs/components/job.details.html b/www/src/app/pages/jobs/components/job.details.html index c46acb089..75674573c 100644 --- a/www/src/app/pages/jobs/components/job.details.html +++ b/www/src/app/pages/jobs/components/job.details.html @@ -31,7 +31,7 @@ PAP

- +

From 39fa9ac7f69eb9d5a141398ea1d38e7935a7f966 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 12 Sep 2018 15:01:06 +0200 Subject: [PATCH 4/7] #123 Display analyzer cache config in analyzer list --- .../organizations/components/analyzers/analyzers-list.html | 5 +++++ .../organizations/components/responders/responders-list.html | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html b/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html index 7764ef11b..388982b26 100644 --- a/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html +++ b/www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html @@ -45,6 +45,7 @@

Max TLP
Max PAP
Rate Limit
+
Cache

@@ -79,6 +80,10 @@

None

+
+ Default + {{$ctrl.activeAnalyzers[def].jobCache}} Minutes +
Edit diff --git a/www/src/app/pages/admin/organizations/components/responders/responders-list.html b/www/src/app/pages/admin/organizations/components/responders/responders-list.html index ca8bc39a0..a7ef9299c 100644 --- a/www/src/app/pages/admin/organizations/components/responders/responders-list.html +++ b/www/src/app/pages/admin/organizations/components/responders/responders-list.html @@ -49,7 +49,7 @@

Responders
Max TLP
Max PAP
-
Rate Limit
+
Rate Limit

From df72a6ce1fa7361f50917fd22e81cd1c4770994e Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Fri, 14 Sep 2018 17:00:06 +0200 Subject: [PATCH 5/7] Update resolvers in build.sbt to contain Maven as a dependency --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index f78b20236..0a87204bd 100644 --- a/build.sbt +++ b/build.sbt @@ -18,6 +18,7 @@ libraryDependencies ++= Seq( resolvers += Resolver.sbtPluginRepo("releases") resolvers += "scalaz-bintray" at "http://dl.bintray.com/scalaz/releases" +resolvers += "elasticsearch-releases" at "https://artifacts.elastic.co/maven" publishArtifact in (Compile, packageDoc) := false publishArtifact in packageDoc := false sources in (Compile,doc) := Seq.empty From ef93c8585251261d67130ed91ba6675d7d7ec7f6 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 25 Sep 2018 15:17:14 +0200 Subject: [PATCH 6/7] Update version and changelog --- CHANGELOG.md | 28 ++++++++++++++++++++++++---- project/Dependencies.scala | 2 +- version.sbt | 2 +- www/package.json | 2 +- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a8a2affd..5ee7c38e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Change Log +## [Unreleased](https://github.com/TheHive-Project/Cortex/tree/HEAD) + +[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.0-RC1...HEAD) + +**Implemented enhancements:** + +- Show PAP value in the Org \> Analyzers screen [\#124](https://github.com/TheHive-Project/Cortex/issues/124) +- Display cache configuration in analyzer admin page [\#123](https://github.com/TheHive-Project/Cortex/issues/123) + +**Fixed bugs:** + +- Temporary files are not removed at the end of job [\#129](https://github.com/TheHive-Project/Cortex/issues/129) +- MISP fails to run analyzers [\#128](https://github.com/TheHive-Project/Cortex/issues/128) +- MISP API fails [\#109](https://github.com/TheHive-Project/Cortex/issues/109) +- File\_Info issue [\#53](https://github.com/TheHive-Project/Cortex/issues/53) + +**Merged pull requests:** + +- Update resolvers in build.sbt to contain Maven as a dependency [\#130](https://github.com/TheHive-Project/Cortex/pull/130) ([adl1995](https://github.com/adl1995)) + ## [2.1.0-RC1](https://github.com/TheHive-Project/Cortex/tree/2.1.0-RC1) (2018-07-31) [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.4...2.1.0-RC1) @@ -9,22 +29,22 @@ **Fixed bugs:** +- Analyzer Configuration Only Showing Global Configuration [\#104](https://github.com/TheHive-Project/Cortex/issues/104) - First analyze of a "file" always fail, must re-run the analyze a second time [\#117](https://github.com/TheHive-Project/Cortex/issues/117) - Analyzers filter in Jobs History view is limited to 25 analyzers [\#116](https://github.com/TheHive-Project/Cortex/issues/116) - Fix redirection from Migration page to login on 401 error [\#114](https://github.com/TheHive-Project/Cortex/issues/114) -- Analyzer Configuration Only Showing Global Configuration [\#104](https://github.com/TheHive-Project/Cortex/issues/104) **Closed issues:** - Automatic observables extraction from analysis reports. [\#111](https://github.com/TheHive-Project/Cortex/issues/111) -- Automated response via Cortex [\#110](https://github.com/TheHive-Project/Cortex/issues/110) -- Consider providing checksums for the release files [\#105](https://github.com/TheHive-Project/Cortex/issues/105) - ImportError: No module named 'cortexutils' on V2.0.4 [\#102](https://github.com/TheHive-Project/Cortex/issues/102) - Error occur from thehive project request to cortex project [\#101](https://github.com/TheHive-Project/Cortex/issues/101) - Analyzers disappear after deactivation and can not get enabled [\#98](https://github.com/TheHive-Project/Cortex/issues/98) -- PAP as an analyzer restriction [\#65](https://github.com/TheHive-Project/Cortex/issues/65) - Application.conf doesn't have Yeti config nor allows for API Auth [\#54](https://github.com/TheHive-Project/Cortex/issues/54) - endless loop of cortex analyser call [\#36](https://github.com/TheHive-Project/Cortex/issues/36) +- Automated response via Cortex [\#110](https://github.com/TheHive-Project/Cortex/issues/110) +- Consider providing checksums for the release files [\#105](https://github.com/TheHive-Project/Cortex/issues/105) +- PAP as an analyzer restriction [\#65](https://github.com/TheHive-Project/Cortex/issues/65) **Merged pull requests:** diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1d7e2fad4..26b55b1a4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -17,6 +17,6 @@ object Dependencies { val reflections = "org.reflections" % "reflections" % "0.9.11" val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2" - val elastic4play = "org.thehive-project" %% "elastic4play" % "1.6.0" + val elastic4play = "org.thehive-project" %% "elastic4play" % "1.6.2" } diff --git a/version.sbt b/version.sbt index c3fc77c26..65e0ce175 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "2.1.0-RC1" +version in ThisBuild := "2.1.0" diff --git a/www/package.json b/www/package.json index 88c4e376f..4547d37b8 100755 --- a/www/package.json +++ b/www/package.json @@ -1,6 +1,6 @@ { "name": "cortex", - "version": "2.0.4", + "version": "2.1.0", "description": "A powerfull observable analysis engine", "license": "AGPL-v3", "homepage": "https://github.com/TheHive-Project/Cortex", From 47ce7f6666eddb7de4239c896d70c7fdd0fb8d52 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 25 Sep 2018 16:16:28 +0200 Subject: [PATCH 7/7] Fix bash cache failure after updating pip --- docker.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/docker.sbt b/docker.sbt index ba4224de3..df6ed9ef8 100644 --- a/docker.sbt +++ b/docker.sbt @@ -30,6 +30,7 @@ dockerCommands ~= { dc => "apt-get install -y --no-install-recommends python-pip python2.7-dev python3-pip python3-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev && " + "pip2 install -U pip setuptools && " + "pip3 install -U pip setuptools && " + + "hash -r && " + "cd /opt && " + "git clone https://github.com/TheHive-Project/Cortex-Analyzers.git && " + "for I in $(find Cortex-Analyzers -name 'requirements.txt'); do pip2 install -r $I; done && " +