diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e508f525..43969d449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,32 @@ # Change Log -## [2.0.2](https://github.com/TheHive-Project/Cortex/tree/2.02) +## [2.0.3](https://github.com/TheHive-Project/Cortex/tree/2.0.3) (2018-04-09) +[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.2...2.0.3) + +**Implemented enhancements:** + +- Allow arbitrary parameters for a job [\#86](https://github.com/TheHive-Project/Cortex/issues/86) +- Change of global config for proxy is not reflected in analyzer's configurations [\#81](https://github.com/TheHive-Project/Cortex/issues/81) + +**Fixed bugs:** + +- Refresh Analyzers button not working [\#83](https://github.com/TheHive-Project/Cortex/issues/83) +- Version Upgrade of Analyzer makes all Analyzers invisible for TheHive \(Cortex2\) [\#75](https://github.com/TheHive-Project/Cortex/issues/75) + +**Closed issues:** + +- Allow specifying a cache period per analyzer [\#85](https://github.com/TheHive-Project/Cortex/issues/85) +- Display existing analyzers with invalid definition [\#82](https://github.com/TheHive-Project/Cortex/issues/82) +- Allow configuring auto artifacts extraction per analyzer [\#80](https://github.com/TheHive-Project/Cortex/issues/80) + +## [2.0.2](https://github.com/TheHive-Project/Cortex/tree/2.0.2) (2018-04-04) [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.1...2.0.2) **Fixed bugs:** -- Silently failure when ElasticSearch is unreachable [\#76](https://github.com/TheHive-Project/Cortex/issues/76) - Coretxutils and TypeError: argument of type 'bool' is not iterable [\#73](https://github.com/TheHive-Project/Cortex/issues/73) +- Silently failure when ElasticSearch is unreachable [\#76](https://github.com/TheHive-Project/Cortex/issues/76) - Unable to disable analyzers [\#72](https://github.com/TheHive-Project/Cortex/issues/72) - Cortex 2 is not passing proxy variable to analyzers [\#71](https://github.com/TheHive-Project/Cortex/issues/71) - Session collision when TheHive & Cortex 2 share the same URL [\#70](https://github.com/TheHive-Project/Cortex/issues/70) @@ -157,4 +176,4 @@ ## [1.0.0](https://github.com/TheHive-Project/Cortex/tree/1.0.0) (2017-02-01) -\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* +\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* \ No newline at end of file diff --git a/app/org/thp/cortex/controllers/AnalyzerCtrl.scala b/app/org/thp/cortex/controllers/AnalyzerCtrl.scala index 297397bf5..adf802682 100644 --- a/app/org/thp/cortex/controllers/AnalyzerCtrl.scala +++ b/app/org/thp/cortex/controllers/AnalyzerCtrl.scala @@ -46,12 +46,12 @@ class AnalyzerCtrl @Inject() ( } private val emptyAnalyzerDefinitionJson = Json.obj( - "version" -> JsNull, - "description" -> JsNull, + "version" -> "0.0", + "description" -> "unknown", "dataTypeList" -> Nil, - "author" -> JsNull, - "url" -> JsNull, - "license" -> JsNull) + "author" -> "unknown", + "url" -> "unknown", + "license" -> "unknown") private def analyzerJson(analyzer: Analyzer, analyzerDefinition: Option[AnalyzerDefinition]) = { analyzer.toJson ++ analyzerDefinition.fold(emptyAnalyzerDefinitionJson) { ad ⇒ diff --git a/app/org/thp/cortex/models/Analyzer.scala b/app/org/thp/cortex/models/Analyzer.scala index af23ba4d9..d276c769f 100644 --- a/app/org/thp/cortex/models/Analyzer.scala +++ b/app/org/thp/cortex/models/Analyzer.scala @@ -27,6 +27,7 @@ trait AnalyzerAttributes { _: AttributeDef ⇒ val configuration = attribute("configuration", F.textFmt, "Configuration of analyzer", O.sensitive) val rate = optionalAttribute("rate", F.numberFmt, "Number ") val rateUnit = optionalAttribute("rateUnit", F.enumFmt(RateUnit), "") + val jobCache = optionalAttribute("jobCache", F.numberFmt, "") } @Singleton diff --git a/app/org/thp/cortex/services/JobSrv.scala b/app/org/thp/cortex/services/JobSrv.scala index edba7c31d..aabd5d0e0 100644 --- a/app/org/thp/cortex/services/JobSrv.scala +++ b/app/org/thp/cortex/services/JobSrv.scala @@ -19,7 +19,7 @@ import org.scalactic.{ Bad, Good, One, Or } import org.thp.cortex.models._ import play.api.libs.json._ import play.api.{ Configuration, Logger } -import scala.concurrent.duration.Duration +import scala.concurrent.duration._ import scala.concurrent.{ ExecutionContext, Future } import scala.sys.process.{ Process, ProcessIO } import scala.util.control.NonFatal @@ -300,7 +300,8 @@ class JobSrv( } def findSimilarJob(analyzer: Analyzer, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject): Future[Option[Job]] = { - if (jobCache.length == 0) { + val cache = analyzer.jobCache().fold(jobCache)(_.minutes) + if (cache.length == 0) { logger.info("Job cache is disabled") Future.successful(None) } @@ -312,7 +313,7 @@ class JobSrv( "analyzerId" ~= analyzer.id, "status" ~!= JobStatus.Failure, "status" ~!= JobStatus.Deleted, - "startDate" ~>= (now - jobCache.toMillis), + "startDate" ~>= (now - cache.toMillis), "dataType" ~= dataType, "tlp" ~= tlp, dataAttachment.fold(data ⇒ "data" ~= data, attachment ⇒ "attachment.id" ~= attachment.id), @@ -410,19 +411,19 @@ class JobSrv( "data" -> job.data().get) } .map { artifact ⇒ - val configAndParam = analyzer.config.deepMerge(job.params) (BaseConfig.global.items ++ BaseConfig.tlp.items ++ analyzerDefinition.configurationItems) - .validatedBy(_.read(configAndParam)) + .validatedBy(_.read(analyzer.config)) .map(cfg ⇒ Json.obj("config" -> JsObject(cfg).deepMerge(analyzerDefinition.configuration))) .map { cfg ⇒ val proxy_http = (cfg \ "config" \ "proxy_http").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" -> Json.obj("http" -> proxy)) } val proxy_https = (cfg \ "config" \ "proxy_https").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" -> Json.obj("https" -> proxy)) } - cfg.deepMerge(Json.obj("config" -> (proxy_http.deepMerge(proxy_https)))) + cfg.deepMerge(Json.obj("config" -> proxy_http.deepMerge(proxy_https))) } .map(_ deepMerge artifact + ("dataType" -> JsString(job.dataType())) + ("tlp" -> JsNumber(job.tlp())) + - ("message" -> JsString(job.message().getOrElse("")))) + ("message" -> JsString(job.message().getOrElse(""))) + + ("parameters" -> job.params)) .badMap(e ⇒ AttributeCheckingError("job", e.toSeq)) .toTry } diff --git a/build.sbt b/build.sbt index 625763176..94be3abc0 100644 --- a/build.sbt +++ b/build.sbt @@ -140,11 +140,13 @@ dockerCommands ~= { dc => Cmd("USER", "root"), ExecCmd("RUN", "bash", "-c", "apt-get update && " + - "apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev && " + + "apt-get install -y --no-install-recommends python-pip python2.7-dev python3-pip python3-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev && " + "pip install -U pip setuptools && " + + "pip3 install -U pip setuptools && " + "cd /opt && " + "git clone https://github.com/CERT-BDF/Cortex-Analyzers.git && " + - "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt)"), + "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt) && " + + "sort -u Cortex-Analyzers/analyzers/*/requirements.txt | grep -v ';python_version' | xargs -n 1 pip3 install || true"), Cmd("ADD", "var", "/var"), Cmd("ADD", "etc", "/etc"), ExecCmd("RUN", "chown", "-R", "daemon:root", "/var/log/cortex"), diff --git a/package/docker/entrypoint b/package/docker/entrypoint index 23b430e36..59e97e1ff 100755 --- a/package/docker/entrypoint +++ b/package/docker/entrypoint @@ -48,7 +48,7 @@ then SECRET=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1) fi echo Using secret: $SECRET - echo play.crypto.secret=\"$SECRET\" >> $CONFIG_FILE + echo play.http.secret.key=\"$SECRET\" >> $CONFIG_FILE fi if test $CONFIG_ES = 1 diff --git a/version.sbt b/version.sbt index 121071caf..3e00daab6 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "2.0.2" +version in ThisBuild := "2.0.3" diff --git a/www/package.json b/www/package.json index 40b4d0432..66fee3a1b 100755 --- a/www/package.json +++ b/www/package.json @@ -1,6 +1,6 @@ { "name": "cortex", - "version": "2.0.2", + "version": "2.0.3", "description": "A powerfull observable analysis engine", "license": "AGPL-v3", "homepage": "https://github.com/TheHive-Project/Cortex", @@ -21,6 +21,7 @@ "angular-bootstrap-multiselect": "git+https://github.com/bentorfs/angular-bootstrap-multiselect.git", "angular-clipboard": "^1.6.2", "angular-images-resizer": "^2.0.2", + "angular-input-masks": "^4.1.0", "angular-local-storage": "^0.7.1", "angular-messages": "1.6.8", "angular-moment": "^1.2.0", diff --git a/www/src/app/index.module.js b/www/src/app/index.module.js index 85b081c67..6a3e7f91c 100755 --- a/www/src/app/index.module.js +++ b/www/src/app/index.module.js @@ -33,6 +33,7 @@ const App = angular.module('cortex', [ 'btorfs.multiselect', 'LocalStorageModule', 'angularUtils.directives.dirPagination', + 'ui.utils.masks', // core coreModule.name, diff --git a/www/src/app/index.vendor.js b/www/src/app/index.vendor.js index 2380d377c..63ada7239 100755 --- a/www/src/app/index.vendor.js +++ b/www/src/app/index.vendor.js @@ -40,6 +40,8 @@ import 'angular-images-resizer'; import 'angular-base64-upload'; +import 'angular-input-masks'; + // local scripts //import "../assets/js/..."; diff --git a/www/src/app/pages/admin/organizations/components/analyzer-config-form.controller.js b/www/src/app/pages/admin/organizations/components/analyzer-config-form.controller.js index 995368b92..6e2a9988f 100644 --- a/www/src/app/pages/admin/organizations/components/analyzer-config-form.controller.js +++ b/www/src/app/pages/admin/organizations/components/analyzer-config-form.controller.js @@ -1,10 +1,33 @@ 'use strict'; +import _ from 'lodash/core'; + export default class AnalyzerConfigFormController { - constructor(Tlps) { + constructor($log, Tlps, AnalyzerService) { 'ngInject'; + this.AnalyzerService = AnalyzerService; this.Tlps = Tlps; this.rateUnits = ['Day', 'Month']; } + + $onInit() { + this.useGlobalCache = + this.analyzer.jobCache === null || this.analyzer.jobCache === undefined; + } + + applyConfig(config) { + _.forEach( + _.keys(config), + k => (this.analyzer.configuration[k] = config[k]) + ); + } + + applyGlobalConfig() { + this.applyConfig(this.globalConfig.config); + } + + applyBaseConfig() { + this.applyConfig(this.baseConfig.config); + } } diff --git a/www/src/app/pages/admin/organizations/components/analyzer-config-form.html b/www/src/app/pages/admin/organizations/components/analyzer-config-form.html index 20a2a26c8..287c058d1 100644 --- a/www/src/app/pages/admin/organizations/components/analyzer-config-form.html +++ b/www/src/app/pages/admin/organizations/components/analyzer-config-form.html @@ -10,12 +10,28 @@

Base details

-

Configuration

+

+ Configuration +
+ + + Apply defaults + +
+

-

Options

+

+ Options +
+ + + Apply defaults + +
+

@@ -42,10 +58,22 @@

Options

+
+ +
+
+ + +
+
+
+

Set to True to enable automatic observables extraction from analysis reports.

+
+
- +
+
+
+
+

Define the number minutes for analysis report caching, or use the globally defined value.

+
+
\ No newline at end of file diff --git a/www/src/app/pages/admin/organizations/components/analyzer.edit.controller.js b/www/src/app/pages/admin/organizations/components/analyzer.edit.controller.js index 234098147..c8f0f7bbf 100644 --- a/www/src/app/pages/admin/organizations/components/analyzer.edit.controller.js +++ b/www/src/app/pages/admin/organizations/components/analyzer.edit.controller.js @@ -7,6 +7,8 @@ export default class AnalyzerEditController { $log, $uibModalInstance, definition, + globalConfig, + baseConfig, configuration, analyzer, mode @@ -17,6 +19,8 @@ export default class AnalyzerEditController { this.$uibModalInstance = $uibModalInstance; this.mode = mode; this.definition = definition; + this.globalConfig = globalConfig; + this.baseConfig = baseConfig; this.configuration = configuration; this.analyzer = analyzer; } @@ -27,7 +31,8 @@ export default class AnalyzerEditController { name: this.definition.id, configuration: {}, rate: undefined, - rateUnit: undefined + rateUnit: undefined, + jobCache: null }; _.forEach(this.definition.configurationItems, item => { @@ -49,7 +54,9 @@ export default class AnalyzerEditController { _.forEach(globalConfig, cnf => { if (analyzer.configuration[cnf] === undefined) { analyzer.configuration[cnf] = - this.configuration.config[cnf] || undefined; + this.configuration.config[cnf] !== undefined + ? this.configuration.config[cnf] + : undefined; } }); diff --git a/www/src/app/pages/admin/organizations/components/analyzer.edit.modal.html b/www/src/app/pages/admin/organizations/components/analyzer.edit.modal.html index baa3277b4..c65f05f27 100644 --- a/www/src/app/pages/admin/organizations/components/analyzer.edit.modal.html +++ b/www/src/app/pages/admin/organizations/components/analyzer.edit.modal.html @@ -3,7 +3,11 @@

{{$modal.mode === 'create' ? 'Enable' : 'Edit'}} analyzer {{$modal.definition.id}}

- +
diff --git a/www/src/app/pages/admin/organizations/components/analyzers-list.controller.js b/www/src/app/pages/admin/organizations/components/analyzers-list.controller.js index 2f4301b2f..1a2e0d44e 100644 --- a/www/src/app/pages/admin/organizations/components/analyzers-list.controller.js +++ b/www/src/app/pages/admin/organizations/components/analyzers-list.controller.js @@ -33,37 +33,42 @@ export default class OrganizationAnalyzersController { $onInit() { this.activeAnalyzers = _.keyBy(this.analyzers, 'analyzerDefinitionId'); this.definitionsIds = _.keys(this.analyzerDefinitions).sort(); + this.invalidAnalyzers = _.filter(this.analyzers, a => + _.isEmpty(a.dataTypeList) + ); } openModal(mode, definition, analyzer) { - let promise; - - if (definition && definition.baseConfig) { - promise = this.AnalyzerService.getConfiguration(definition.baseConfig); - } else { - promise = this.$q.resolve({}); - } - return promise - .then( - analyzerConfig => analyzerConfig, - err => { - if (err.status === 404) { - return {}; - } - } - ) - .then(analyzerConfig => - this.AnalyzerService.getConfiguration('global').then(globalConfig => { - if (!analyzerConfig.config) { - analyzerConfig.config = {}; + let baseConfigName = definition ? definition.baseConfig : undefined; + return this.AnalyzerService.getBaseConfig(baseConfigName) + .then(baseConfig => { + let configs = { + globalConfig: {}, + baseConfig: baseConfig, + analyzerConfig: { + config: {} } + }; + + return this.AnalyzerService.getConfiguration('global').then( + globalConfig => { + configs.globalConfig = globalConfig; - _.merge(analyzerConfig.config, globalConfig.config); + if (!baseConfig.config) { + baseConfig.config = {}; + } - return analyzerConfig; - }) - ) - .then(analyzerConfig => { + _.merge( + configs.analyzerConfig.config, + configs.baseConfig.config, + configs.globalConfig.config + ); + + return configs; + } + ); + }) + .then(configs => { let modal = this.$uibModal.open({ animation: true, controller: AnalyzerEditController, @@ -72,7 +77,9 @@ export default class OrganizationAnalyzersController { size: 'lg', resolve: { definition: () => definition, - configuration: () => analyzerConfig, + globalConfig: () => configs.globalConfig, + baseConfig: () => configs.baseConfig, + configuration: () => configs.analyzerConfig, analyzer: () => angular.copy(analyzer), mode: () => mode } @@ -89,7 +96,14 @@ export default class OrganizationAnalyzersController { } else { return this.OrganizationService.updateAnalyzer( analyzer.id, - _.pick(response, 'configuration', 'rate', 'rateUnit', 'name') + _.pick( + response, + 'configuration', + 'rate', + 'rateUnit', + 'name', + 'jobCache' + ) ); } }) @@ -154,13 +168,13 @@ export default class OrganizationAnalyzersController { refreshAnalyzers() { this.AnalyzerService.scan() - .then(() => this.AnalyzerService.definitions()) + .then(() => this.AnalyzerService.definitions(true)) .then(defintions => { this.analyzerDefinitions = defintions; - this.$onInit(); + this.reload(); this.NotificationService.success('Analyzer definitions refreshed.'); }) - .catch(err => + .catch(() => this.NotificationService.error( 'Failed to refresh analyzer definitions.' ) diff --git a/www/src/app/pages/admin/organizations/components/analyzers-list.html b/www/src/app/pages/admin/organizations/components/analyzers-list.html index 1bc27f577..4747c314a 100644 --- a/www/src/app/pages/admin/organizations/components/analyzers-list.html +++ b/www/src/app/pages/admin/organizations/components/analyzers-list.html @@ -1,4 +1,27 @@
+ +
+
+

You have {{$ctrl.invalidAnalyzers.length}} invalid

+

Invalid analyzers have no definition and cannot be run on any observable. You have to remove them.

+
+
+
+
+
+

+ {{a.name}} +

+
+
+ + Disable +
+
+
+
+
+

Available analyzers ({{$ctrl.definitionsIds.length || 0}}) Refresh analyzers

diff --git a/www/src/app/pages/admin/organizations/components/config-list.controller.js b/www/src/app/pages/admin/organizations/components/config-list.controller.js index 018a49852..6f0e2297e 100644 --- a/www/src/app/pages/admin/organizations/components/config-list.controller.js +++ b/www/src/app/pages/admin/organizations/components/config-list.controller.js @@ -26,12 +26,18 @@ export default class OrganizationConfigsController { size: 'lg', resolve: { configuration: () => { + let defaultValues = { + string: null, + number: 0, + boolean: true + }; let conf = angular.copy(config); _.forEach(conf.configurationItems, item => { - conf.config[item.name] = conf.config[item.name] - ? conf.config[item.name] - : item.defaultValue || (item.multi ? [undefined] : undefined); + conf.config[item.name] = + conf.config[item.name] !== undefined + ? conf.config[item.name] + : item.defaultValue || (item.multi ? [undefined] : undefined); }); return conf; diff --git a/www/src/app/pages/admin/organizations/organizations.module.js b/www/src/app/pages/admin/organizations/organizations.module.js index 3f42609df..6b551b3e9 100644 --- a/www/src/app/pages/admin/organizations/organizations.module.js +++ b/www/src/app/pages/admin/organizations/organizations.module.js @@ -126,7 +126,10 @@ const organizationsModule = angular templateUrl: analyzerConfigFormTpl, bindings: { analyzer: '=', - definition: '=' + definition: '=', + configuration: '<', + globalConfig: '<', + baseConfig: '<' } }) .service('OrganizationService', organizationService); diff --git a/www/src/app/pages/analyzers/analyzers.service.js b/www/src/app/pages/analyzers/analyzers.service.js index 04afd3bbd..ad7278d8f 100644 --- a/www/src/app/pages/analyzers/analyzers.service.js +++ b/www/src/app/pages/analyzers/analyzers.service.js @@ -23,10 +23,10 @@ export default class AnalyzerService { return this.dataTypes; } - definitions() { + definitions(force) { let defered = this.$q.defer(); - if (this.analyzerDefinitions === null) { + if (force || this.analyzerDefinitions === null) { this.$http.get('./api/analyzerdefinition').then( response => { this.analyzerDefinitions = _.keyBy(response.data, 'id'); @@ -84,6 +84,21 @@ export default class AnalyzerService { return defer.promise; } + getBaseConfig(baseConfig) { + let defer = this.$q.defer(); + + if (baseConfig) { + this.getConfiguration(baseConfig).then( + cfg => defer.resolve(cfg), + () => defer.resolve({}) + ); + } else { + defer.resolve({}); + } + + return defer.promise; + } + getConfiguration(name) { let defer = this.$q.defer();