diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a42bac96..ac6646e61d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Change Log +## [4.0.2](https://github.com/TheHive-Project/TheHive/milestone/64) (2020-11-20) + +**Implemented enhancements:** + +- [Feature Request] Add a dedicated permission to give access to TheHiveFS [\#1655](https://github.com/TheHive-Project/TheHive/issues/1655) +- [Feature Request] Normalize editable input fields [\#1669](https://github.com/TheHive-Project/TheHive/issues/1669) + +**Fixed bugs:** + +- [Bug] Unable to list Cases [\#1598](https://github.com/TheHive-Project/TheHive/issues/1598) +- [Bug] Alert to case merge is broken in v4.0.1 [\#1648](https://github.com/TheHive-Project/TheHive/issues/1648) +- [Bug] Attachment.* filters are broken under observable search in v4.0.1 [\#1649](https://github.com/TheHive-Project/TheHive/issues/1649) +- [Bug] Result of observable update API v0 is empty [\#1652](https://github.com/TheHive-Project/TheHive/issues/1652) +- [Bug] Display issue of custom fields [\#1653](https://github.com/TheHive-Project/TheHive/issues/1653) +- [Bug] Persistent AuditSrv:undefined error on 4.0.1 [\#1656](https://github.com/TheHive-Project/TheHive/issues/1656) +- [Bug] Issues with case attachments section [\#1657](https://github.com/TheHive-Project/TheHive/issues/1657) +- [Bug] API method broken: /api/case/artifact/_search in 4.0.1 [\#1659](https://github.com/TheHive-Project/TheHive/issues/1659) +- [Bug] API method broken: /api/case/task/log/_search in 4.0.1 [\#1660](https://github.com/TheHive-Project/TheHive/issues/1660) +- [Bug] Unable to define ES index on migration [\#1661](https://github.com/TheHive-Project/TheHive/issues/1661) +- [Bug] Dashboard max aggregation on custom-integer field does not work [\#1662](https://github.com/TheHive-Project/TheHive/issues/1662) +- [Bug] Missing the fix for errorMessage [\#1666](https://github.com/TheHive-Project/TheHive/issues/1666) +- [Bug] Fix alert details dialog [\#1672](https://github.com/TheHive-Project/TheHive/issues/1672) +- [Bug] error 500 with adding an empty file in Observables of an Alert [\#1673](https://github.com/TheHive-Project/TheHive/issues/1673) +- [Bug] Fix migration of audit logs [\#1676](https://github.com/TheHive-Project/TheHive/issues/1676) + ## [4.0.1](https://github.com/TheHive-Project/TheHive/milestone/60) (2020-11-13) **Implemented enhancements:** @@ -24,6 +49,7 @@ **Fixed bugs:** +- [Bug] MISP->THEHIVE4 'ExportOnly' and 'Exceptions' ignored in application.conf file [\#1482](https://github.com/TheHive-Project/TheHive/issues/1482) - [Bug] Mobile-responsive Hamburger not visible [\#1290](https://github.com/TheHive-Project/TheHive/issues/1290) - [Bug] Unable to start TheHive after migration [\#1450](https://github.com/TheHive-Project/TheHive/issues/1450) - [Bug] Expired session should show a dialog or login page on pageload [\#1456](https://github.com/TheHive-Project/TheHive/issues/1456) @@ -34,7 +60,6 @@ - [Bug] Dashboard shared/private [\#1474](https://github.com/TheHive-Project/TheHive/issues/1474) - [Bug]Migration tool date/number/duration params don't work [\#1478](https://github.com/TheHive-Project/TheHive/issues/1478) - [Bug] AuditSrv: undefined on non-case page(s), thehive4-4.0.0-1, Ubuntu [\#1479](https://github.com/TheHive-Project/TheHive/issues/1479) -- [Bug] MISP->THEHIVE4 'ExportOnly' and 'Exceptions' ignored in application.conf file [\#1482](https://github.com/TheHive-Project/TheHive/issues/1482) - [Bug] Unable to enumerate tasks via API [\#1483](https://github.com/TheHive-Project/TheHive/issues/1483) - [Bug] Case close notification displays "#undefined" instead of case number [\#1488](https://github.com/TheHive-Project/TheHive/issues/1488) - [Bug] Task under "Waiting tasks" and "My tasks" do not display the case number [\#1489](https://github.com/TheHive-Project/TheHive/issues/1489) diff --git a/ScalliGraph b/ScalliGraph index 0dc00d560b..f6a4d2165c 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 0dc00d560b7c48f5f7a781cd4c9861a64f56cd23 +Subproject commit f6a4d2165c26826c5b28db1a513ade15dfb060f2 diff --git a/build.sbt b/build.sbt index c93640d396..e4fa911b8b 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.0.1-1" +val thehiveVersion = "4.0.2-1" val scala212 = "2.12.12" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/docker/README.md b/docker/README.md index 3aadbdb298..2bb97b37c2 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,14 +1,14 @@ ## Example of docker-compose (not for production) With this docker-compose.yml you will be able to run the following images: -- The Hive 4 +- The Hive 4.0.1-1 - Cassandra 3.11 - Cortex 3.1.0-1 - Elasticsearch 7.9.3 - Kibana 7.9.3 -- MISP 2.4.133 +- MISP 2.4.134 - Mysql 8.0.22 - Redis 6.0.9 -- Shuffle 0.7.1 +- Shuffle 0.7.6 ## Some Hint @@ -17,47 +17,46 @@ In docker-compose version is set 3.8, to run this version you need at least Dock ``` Compose file format Docker Engine release 3.8 19.03.0+ -3.7 18.06.0+ -3.6 18.02.0+ -3.5 17.12.0+ -3.4 17.09.0+ +3.7 18.06.0+ +3.6 18.02.0+ +3.5 17.12.0+ +3.4 17.09.0+ ``` If for some reason you have a previous version of Docker Engine or a previous version of Docker Compose and can't upgrade those, you can use 3.7 or 3.6 in docker-compose.yml ### Mapping volumes -If you take a look of docker-compose.yml you will see you need some local folder that needs to be mapped, so before do docker-compose up, ensure folders (and config files) exist: -- ./elasticsearch/data:/usr/share/elasticsearch/data -- ./elasticsearch/logs:/usr/share/elasticsearch/logs +If you take a look of docker-compose.yml you will see you need some local folder that needs to be mapped, so before do docker-compose up, ensure at least folders with config files exist: - ./cortex/application.conf:/etc/cortex/application.conf - ./thehive/application.conf:/etc/thehive/application.conf -- ./data:/data -- ./mysql:/var/lib/mysql Structure would look like: ``` ├── docker-compose.yml -├── elasticsearch -│ └── data -│ └── logs +├── elasticsearch_data +|── elasticsearch_logs ├── cortex │ └── application.conf -└── thehive - └── application.conf -└── data -└── mysql +|── thehive +| └── application.conf +|── data +|── mysql ``` +If you run docker-compose with sudo, ensure you have created elasticsearch_data and elasticsearch_logs folders with non root user, otherwise elasticsearch container will not start. ### ElasticSearch ElasticSearch container likes big mmap count (https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) so from shell you can change with ```sysctl -w vm.max_map_count=262144``` -Due you would run all on same system and maybe you have a limited amount of RAM, better to set some size, for ElasticSearch, in docker-compose.yml I added those: +To set this value permanently, update the vm.max_map_count setting in /etc/sysctl.conf. To verify after rebooting, run sysctl vm.max_map_count + +If you would run all containers on the same system - and maybe you have a limited amount of RAM - better to set some limit, for ElasticSearch, in docker-compose.yml I added those: ```- bootstrap.memory_lock=true``` ```- "ES_JAVA_OPTS=-Xms256m -Xmx256m"``` Adjust depending on your needs and your env. Without these settings in my environment ElasticSearch was using 1.5GB + ### Cassandra Like for ElasticSearch maybe you would run all on same system and maybe you don't have a limited amount of RAM, better to set some size, here for Cassandra, in docker-compose.yml I added those: @@ -68,7 +67,7 @@ Adjust depending on your needs and your env. Without these settings in my enviro ### Cortex-Analyzers - In order to use Analyzers in docker version, it is set the online json url instead absolute path of analyzers in the application.conf of Cortex: - https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json + https://download.thehive-project.org/analyzers.json - In order to use Analyzers in docker version it is set the application.conf thejob: ``` job { runner = [docker] @@ -142,3 +141,21 @@ curl -XPUT -uuser@thehive.local:user@thehive.local -H 'Content-type: application ``` - Now are able to play automation with The Hive, Cortex-Analyzers, MISP thanks to SHUFFLE! + +### Result +In conclusion, after execute ```sudo docker-compose up``` you will have the following services running: + + +| Service | Address | User | Password | +|----------|:-------------:|:------:|------:| +| The Hive | http://localhost:9000 | admin@thehive.local | secret +| Cortex | http://localhost:9001 | | +| Elasticsearch | http://localhost:9200 | | +| Kibana | http://localhost:5601 | | +| MISP | https://localhost:443 | admin@admin.test | admin +| Shuffle | http://localhost:3001 | | + + + +![image](https://user-images.githubusercontent.com/16938405/99674126-e8c99f80-2a75-11eb-9a8b-1603cf67d665.png) +![image](https://user-images.githubusercontent.com/16938405/99674544-7c02d500-2a76-11eb-92a5-3fbb5c3c5cc5.png) diff --git a/docker/cortex/application.conf b/docker/cortex/application.conf index 0fe0c01b63..6236c81902 100644 --- a/docker/cortex/application.conf +++ b/docker/cortex/application.conf @@ -179,7 +179,7 @@ analyzer { # - directory where analyzers are installed # - json file containing the list of analyzer descriptions urls = [ - "https://dl.bintray.com/thehive-project/cortexneurons/analyzers.json" + "https://download.thehive-project.org/analyzers.json" #"/absolute/path/of/analyzers" ] @@ -199,7 +199,7 @@ analyzer { responder { # responder location (same format as analyzer.urls) urls = [ - "https://dl.bintray.com/thehive-project/cortexneurons/responders.json" + "https://download.thehive-project.org/responders.json" #"/absolute/path/of/responders" ] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ebd066e945..1bb7b6b63e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,8 +22,8 @@ services: soft: 65536 hard: 65536 volumes: - - ./elasticsearch/data:/usr/share/elasticsearch/data - - ./elasticsearch/logs:/usr/share/elasticsearch/logs + - ./elasticsearch_data:/usr/share/elasticsearch/data + - ./elasticsearch_logs:/usr/share/elasticsearch/logs kibana: image: 'docker.elastic.co/kibana/kibana:7.9.3' container_name: kibana @@ -67,7 +67,7 @@ services: - '0.0.0.0:9000:9000' volumes: - ./thehive/application.conf:/etc/thehive/application.conf - - ./data:/data + - ./data:/opt/data command: '--no-config --no-config-secret' redis: @@ -77,6 +77,7 @@ services: db: image: mysql:latest + container_name: mysql restart: unless-stopped command: --default-authentication-plugin=mysql_native_password restart: always @@ -98,12 +99,19 @@ services: - "80:80" - "443:443" environment: - - "HOSTNAME=http://misp" + - "HOSTNAME=https://localhost" - "REDIS_FQDN=redis" - "INIT=true" # Initialze MISP, things includes, attempting to import SQL and the Files DIR - "CRON_USER_ID=1" # The MISP user ID to run cron jobs as - "DISIPV6=true" # Disable IPV6 in nginx - + misp-modules: + image: coolacid/misp-docker:modules-latest + container_name: misp-modules + environment: + - "REDIS_BACKEND=redis" + depends_on: + - redis + - db #READY FOR AUTOMATION ? frontend: diff --git a/docker/thehive/application.conf b/docker/thehive/application.conf index 793d661be0..b6ed0da698 100644 --- a/docker/thehive/application.conf +++ b/docker/thehive/application.conf @@ -20,7 +20,7 @@ db { storage { provider: localfs - localfs.directory: /opt/data + localfs.location: /opt/data } play.modules.enabled += org.thp.thehive.connector.cortex.CortexModule diff --git a/frontend/app/index.html b/frontend/app/index.html index 5460192e47..800b08a1ae 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -43,12 +43,14 @@ + + diff --git a/frontend/app/scripts/app.js b/frontend/app/scripts/app.js index 019121422f..800a72397d 100644 --- a/frontend/app/scripts/app.js +++ b/frontend/app/scripts/app.js @@ -411,11 +411,7 @@ angular.module('thehive', [ controller: 'CaseAlertsCtrl', resolve: { alerts: function($stateParams, CaseSrv) { - return CaseSrv.alerts({range: 'all'}, { - query: { - case: $stateParams.caseId - } - }).$promise; + return CaseSrv.alerts($stateParams.caseId); } }, guard: { diff --git a/frontend/app/scripts/controllers/SearchCtrl.js b/frontend/app/scripts/controllers/SearchCtrl.js index 0ba6db0c72..6eb5522550 100644 --- a/frontend/app/scripts/controllers/SearchCtrl.js +++ b/frontend/app/scripts/controllers/SearchCtrl.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveControllers') - .controller('SearchCtrl', function($scope, $q, $stateParams, $uibModal, PSearchSrv, CaseTemplateSrv, CaseTaskSrv, NotificationSrv, EntitySrv, UserSrv, QueryBuilderSrv, GlobalSearchSrv, metadata) { + .controller('SearchCtrl', function($scope, $q, $stateParams, $uibModal, PSearchSrv, AlertingSrv, CaseTemplateSrv, CaseTaskSrv, NotificationSrv, EntitySrv, UserSrv, QueryBuilderSrv, GlobalSearchSrv, metadata) { $scope.metadata = metadata; $scope.toolbar = [ // {name: 'all', label: 'All', icon: 'glyphicon glyphicon-search'}, @@ -42,14 +42,24 @@ controllerAs: 'dialog', size: 'max', resolve: { - event: event, + event: function() { + return AlertingSrv.get(event.id); + }, templates: function() { return CaseTemplateSrv.list(); }, readonly: true } - }).result.then(function(/*response*/) { + }) + .result + .then(function(/*response*/) { $scope.searchResults.update(); + }) + .catch(function(err) { + if(err && !_.isString(err)) { + NotificationSrv.error('AlertPreview', err.data, err.status); + } + }); }; diff --git a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js index c36e492711..c1b8d09075 100644 --- a/frontend/app/scripts/controllers/alert/AlertEventCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertEventCtrl.js @@ -39,11 +39,6 @@ AlertingSrv.get(eventId).then(function(data) { self.event = data; self.loading = false; - - self.dataTypes = _.countBy(self.event.artifacts, function(attr) { - return attr.dataType; - }); - }, function(response) { self.loading = false; NotificationSrv.error('AlertEventCtrl', response.data, response.status); @@ -185,6 +180,7 @@ self.copyId = function(id) { clipboard.copyText(id); + NotificationSrv.log('Alert ID has been copied to clipboard', 'success'); }; this.$onInit = function() { diff --git a/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js b/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js index f410a7e27f..b695e32698 100644 --- a/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseAlertsCtrl.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveControllers').controller('CaseAlertsCtrl', - function($scope, $state, $stateParams, $uibModal, $timeout, CaseTabsSrv, VersionSrv, alerts) { + function($scope, $state, $stateParams, $uibModal, $timeout, CaseTabsSrv, VersionSrv, NotificationSrv, alerts) { $scope.caseId = $stateParams.caseId; $scope.alerts = alerts; $scope.alertStats = []; @@ -84,9 +84,16 @@ templates: function() { //return CaseTemplateSrv.list(); return []; - }, + }, readonly: true } + }) + .result + .catch(function(err) { + if(err && !_.isString(err)) { + NotificationSrv.error('AlertPreview', err.data, err.status); + } + }); }; diff --git a/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js b/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js index 699af3d81f..030302eaae 100644 --- a/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseDetailsCtrl.js @@ -17,13 +17,14 @@ version: 'v1', loadAll: false, filter: { - '_contains': 'attachment' + '_contains': 'attachment.id' }, extraData: ['taskId'], pageSize: 100, operations: [ { '_name': 'getCase', 'idOrName': $scope.caseId }, { '_name': 'tasks' }, + { '_name': 'filter', '_ne':{'_field': 'status', '_value': 'Cancel'}}, { '_name': 'logs' }, ] }); diff --git a/frontend/app/scripts/directives/responder-actions.js b/frontend/app/scripts/directives/responder-actions.js index 3dc14a528f..d8fbce6bb2 100644 --- a/frontend/app/scripts/directives/responder-actions.js +++ b/frontend/app/scripts/directives/responder-actions.js @@ -11,7 +11,7 @@ templateUrl: 'views/directives/responder-actions.html', controller: function($scope, $uibModal) { - $scope.$watch('actions', function(list) { + $scope.$watchCollection('actions.values', function(list) { if(!list) { return; } diff --git a/frontend/app/scripts/directives/updatableBoolean.js b/frontend/app/scripts/directives/updatableBoolean.js index e6e231e814..5d84fc826f 100644 --- a/frontend/app/scripts/directives/updatableBoolean.js +++ b/frontend/app/scripts/directives/updatableBoolean.js @@ -12,7 +12,8 @@ 'active': '=?', 'placeholder': '@', 'trueText': '@?', - 'falseText': '@?' + 'falseText': '@?', + 'clearable': 'span { +.kv-label>span { color: #ffffff; display: table-cell; font-weight: 400; @@ -13,23 +13,45 @@ vertical-align: baseline; white-space: nowrap; } -.double-val-label>span:first-child { - border-bottom-left-radius: 0.25em; - border-top-left-radius: .25em; +.kv-label span.kv-label-key { + /* border-bottom-left-radius: 0.25em; + border-top-left-radius: .25em; */ border-left-color: #3c8dbc; border-left-width: 3px; border-left-style: solid; } -.double-val-label>span:last-child { +.kv-label span.kv-label-val { border-bottom-right-radius: 0.25em; border-top-right-radius: .25em; border-left: 1px dashed #3c8dbc; + + overflow: hidden; + text-overflow: ellipsis; + max-width: 200px; } -.double-val-label>span.primary { + +.kv-label.kv-label-addon span.kv-label-val { + border-bottom-right-radius: 0; + border-top-right-radius: 0; +} + +.kv-label.kv-label-addon span:last-child { + border-bottom-right-radius: 0.25em; + border-top-right-radius: .25em; +} + +.kv-label>span.primary { background-color: #3c8dbc; color: #fff; } -.double-val-label>span.default { +.kv-label>span.default { background-color: #d2d6de; color: #444; } + + +.kv-label .tooltip-inner{ + word-wrap: break-word; + text-align: left; + white-space: pre-line; +} diff --git a/frontend/app/styles/main.css b/frontend/app/styles/main.css index cfb024f4db..0425ebbcd9 100644 --- a/frontend/app/styles/main.css +++ b/frontend/app/styles/main.css @@ -160,7 +160,7 @@ pre.clearpre { /***************************/ .flexwrap { - display: flex; + display: flex !important; flex-wrap: wrap; justify-content: flex-start; align-items: flex-start; @@ -340,6 +340,11 @@ ul.observable-reports-summary li { width: 200px !important; } +.case-details dd, +.case-custom-fields dd { + margin-left: 200px !important; +} + .case-custom-fields dt { background-color: #f9f9f9; padding-left: 5px; @@ -761,3 +766,8 @@ table.tbody-stripped>tbody+tbody { table.tbody-stripped > tbody > tr > td { } + +.ellipsable { + overflow: hidden; + text-overflow: ellipsis; +} diff --git a/frontend/app/styles/updatable.css b/frontend/app/styles/updatable.css new file mode 100644 index 0000000000..13a11cdb75 --- /dev/null +++ b/frontend/app/styles/updatable.css @@ -0,0 +1,42 @@ +.updatable-input .updatable-value { + vertical-align: top; + white-space: pre-wrap; +} + +.updatable-input .updatable-input-value-wrapper { + position: relative; + display: inline-block; + border-bottom: 1px solid #fff; + padding-right: 20px; +} + +.updatable-input .updatable-input-value-wrapper:hover{ + border-bottom: 1px solid #337ab7; + cursor: pointer; +} + +.updatable-input .updatable-input-value-wrapper .updatable-input-icon { + display: none; + float: right; + margin-left: 10px; + color: #337ab7; +} + +.updatable-input .updatable-input-value-wrapper .updatable-input-icon.lg { + line-height: 18px; +} + +.updatable-input .updatable-input-value-wrapper:hover .updatable-input-icon { + display: inline-block; + position: absolute; + top: 0; + right: 0; +} + +.updatable-input.updatable-input-text .updatable-input-markdown { + cursor: auto; +} +.updatable-input.updatable-input-text .updatable-input-markdown .updatable-input-icon { + /* float:left; + margin-left: 0; */ +} diff --git a/frontend/app/views/components/common/custom-field-input.component.html b/frontend/app/views/components/common/custom-field-input.component.html index cde52043e6..ec4ad46a5d 100644 --- a/frontend/app/views/components/common/custom-field-input.component.html +++ b/frontend/app/views/components/common/custom-field-input.component.html @@ -1,33 +1,35 @@ -
+
{{$ctrl.field.name}}
+ value="$ctrl.value" clearable="true">
- +
+ - + - + - + - + - Not Editable + Not Editable +
{{$ctrl.value | shortDate}} diff --git a/frontend/app/views/components/common/custom-field-labels.component.html b/frontend/app/views/components/common/custom-field-labels.component.html index 0845ab7043..48e8c6f16e 100644 --- a/frontend/app/views/components/common/custom-field-labels.component.html +++ b/frontend/app/views/components/common/custom-field-labels.component.html @@ -3,12 +3,12 @@ None - - {{$cmp.fieldsCache[cf.name].name || cf.name}} - {{cf | customFieldValue}} + {{$cmp.fieldsCache[cf.name].name || cf.name}} + {{cf | customFieldValue}} diff --git a/frontend/app/views/components/search/filters-preview.component.html b/frontend/app/views/components/search/filters-preview.component.html index 7d28eeb9fd..778b3b648d 100644 --- a/frontend/app/views/components/search/filters-preview.component.html +++ b/frontend/app/views/components/search/filters-preview.component.html @@ -1,18 +1,18 @@ -
-
    -
  • {{$ctrl.filters.length}} - filter(s) applied: -
    • -
  • - - {{filter.field || '???'}}: - {{filter.value | filterValue}} - - - -
    • -
  • - Clear filters -
    • -
+
+ {{$ctrl.filters.length}} filter(s) applied: + + + {{filter.field || '???'}} + {{filter.value | filterValue}} + + + + + + + + Clear filters + +
diff --git a/frontend/app/views/directives/tlp.html b/frontend/app/views/directives/tlp.html index 2b4b76e5ab..a72abb610d 100644 --- a/frontend/app/views/directives/tlp.html +++ b/frontend/app/views/directives/tlp.html @@ -1,23 +1,18 @@ - -
- - WHITE - GREEN - AMBER - RED -
- - - {{namespace || 'TLP'}}:WHITE - {{namespace || 'TLP'}}:GREEN - {{namespace || 'TLP'}}:AMBER - {{namespace || 'TLP'}}:RED - - - - - - - - +
+ WHITE + GREEN + AMBER + RED +
+ + {{namespace || 'TLP'}}:WHITE + {{namespace || 'TLP'}}:GREEN + {{namespace || 'TLP'}}:AMBER + {{namespace || 'TLP'}}:RED + + + + + + diff --git a/frontend/app/views/directives/updatable-boolean.html b/frontend/app/views/directives/updatable-boolean.html index 0c9c7e6e16..4a383096de 100644 --- a/frontend/app/views/directives/updatable-boolean.html +++ b/frontend/app/views/directives/updatable-boolean.html @@ -1,15 +1,14 @@ - - {{value ? trueText || 'True' : falseText || 'False'}} - Not Specified - - - - - -       - - -
+
+ + {{value ? trueText || 'True' : falseText || 'False'}} + Not Specified + + + + + + +
@@ -19,7 +18,10 @@ - + +
- +
diff --git a/frontend/app/views/directives/updatable-date.html b/frontend/app/views/directives/updatable-date.html index 3e4a1e2964..0479e71ec3 100644 --- a/frontend/app/views/directives/updatable-date.html +++ b/frontend/app/views/directives/updatable-date.html @@ -1,29 +1,30 @@ - - {{value | shortDate}} - Not Specified - - - - - -       - - +
+ + {{value | shortDate}} + Not Specified -
-
- - + + + + + + +
+ + - Now - - + Now + + + -
- - +
+ +
diff --git a/frontend/app/views/directives/updatable-select.html b/frontend/app/views/directives/updatable-select.html index 4fc1d7a403..d7ade46e19 100644 --- a/frontend/app/views/directives/updatable-select.html +++ b/frontend/app/views/directives/updatable-select.html @@ -1,15 +1,14 @@ - - {{value}} - Not Specified - - - - - -       - - -
+
+ + {{value}} + Not Specified + + + + + + +
@@ -19,7 +18,10 @@ - + +
- +
diff --git a/frontend/app/views/directives/updatable-simple-text.html b/frontend/app/views/directives/updatable-simple-text.html index ddabf16bbc..9a6bbb2b14 100644 --- a/frontend/app/views/directives/updatable-simple-text.html +++ b/frontend/app/views/directives/updatable-simple-text.html @@ -1,17 +1,18 @@ - - {{value}} - Not Specified - - - - - -       - - -
+
+ + {{value}} + Not Specified + + + + + + +
+ + - + +
- +
diff --git a/frontend/app/views/directives/updatable-tags.html b/frontend/app/views/directives/updatable-tags.html index 46ceceb402..e835f01ed3 100644 --- a/frontend/app/views/directives/updatable-tags.html +++ b/frontend/app/views/directives/updatable-tags.html @@ -1,30 +1,31 @@ - - Not Specified +
+ + Not Specified - {{tag.text}} - - - - - + + + - - - -
- - - -
-
- - - - - - -
- + +
+ + + +
+
+ + + +
+ +
diff --git a/frontend/app/views/directives/updatable-text.html b/frontend/app/views/directives/updatable-text.html index 05b865e1fd..ef2ec25a26 100644 --- a/frontend/app/views/directives/updatable-text.html +++ b/frontend/app/views/directives/updatable-text.html @@ -1,37 +1,39 @@ - - - - - - +
+ + + Not Specified + + + -
- - - Not specified - - - - - + + + + + + +
- - -
+
-
- - - - - - +
+
+ + + +
Markdown Reference
- +
diff --git a/frontend/app/views/directives/updatable-user.html b/frontend/app/views/directives/updatable-user.html index c2844dddcd..a64a013626 100644 --- a/frontend/app/views/directives/updatable-user.html +++ b/frontend/app/views/directives/updatable-user.html @@ -1,26 +1,28 @@ - - - - - - - -       +
+ + - - - - - + + + - - + +
+ + + + +
- +
diff --git a/frontend/app/views/partials/alert/custom.fields.html b/frontend/app/views/partials/alert/custom.fields.html index 2d3800ed50..5bdda66925 100644 --- a/frontend/app/views/partials/alert/custom.fields.html +++ b/frontend/app/views/partials/alert/custom.fields.html @@ -5,8 +5,8 @@
-
-
+
+

- Alert Preview {{dialog.event.read ? 'Ignored' : 'New'}} + Alert Preview {{dialog.event.caseId ? 'Imported' : 'New'}}

@@ -20,7 +20,7 @@

ID: - {{dialog.event._id}} + {{dialog.event._id}} Date: @@ -57,6 +57,23 @@

Additional fields + + + + Layout + + +

diff --git a/frontend/app/views/partials/case/case.alerts.html b/frontend/app/views/partials/case/case.alerts.html index 6ad492473f..590976743b 100644 --- a/frontend/app/views/partials/case/case.alerts.html +++ b/frontend/app/views/partials/case/case.alerts.html @@ -29,7 +29,7 @@

- +
- - - - + + - - + + + + + + + + +
@@ -40,7 +40,7 @@ + Type @@ -48,14 +48,6 @@ - - Status - - - - - Title @@ -94,8 +86,8 @@
{{::event.sourceRef}} @@ -109,18 +101,10 @@ {{::event.type}} - {{::event.status}} -
{{::event.title}}
-
- - None - {{tag}} -
{{::event.source}} @@ -129,13 +113,29 @@ {{::event.artifacts.length || 0}}{{event.date | showDate}}{{event.date | shortDate}}
+
+ + None + {{tag}} +
+
+ +
diff --git a/frontend/app/views/partials/case/case.details.html b/frontend/app/views/partials/case/case.details.html index 8a9b2cc1ca..79ead25542 100644 --- a/frontend/app/views/partials/case/case.details.html +++ b/frontend/app/views/partials/case/case.details.html @@ -16,7 +16,7 @@

Summary

Severity
- +
diff --git a/frontend/app/views/partials/case/details/custom.fields.html b/frontend/app/views/partials/case/details/custom.fields.html index d42f4985c7..27047213df 100644 --- a/frontend/app/views/partials/case/details/custom.fields.html +++ b/frontend/app/views/partials/case/details/custom.fields.html @@ -3,7 +3,7 @@

Additional information - + Add
  • @@ -11,6 +11,23 @@

+ + + + Layout + + +

@@ -19,8 +36,8 @@

-
-
+
+
TLP
- +
diff --git a/frontend/bower.json b/frontend/bower.json index 4a58c3aed4..26f486b888 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.1-1", + "version": "4.0.2-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index e13f63a44a..d1bca2f2ce 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.0.1-1", + "version": "4.0.2-1", "license": "AGPL-3.0", "repository": { "type": "git", diff --git a/migration/src/main/scala/org/thp/thehive/migration/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/Input.scala index ddca2631d0..e6037cceeb 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Input.scala @@ -137,9 +137,11 @@ trait Input { def listAction(filter: Filter): Source[Try[(String, InputAction)], NotUsed] def countAction(filter: Filter): Future[Long] def listAction(entityId: String): Source[Try[(String, InputAction)], NotUsed] + def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed] def countAction(entityId: String): Future[Long] def listAudit(filter: Filter): Source[Try[(String, InputAudit)], NotUsed] def countAudit(filter: Filter): Future[Long] def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed] + def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed] def countAudit(entityId: String, filter: Filter): Future[Long] } diff --git a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala index b29a7f57bd..2e1b770131 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala @@ -56,7 +56,7 @@ object Migrate extends App with MigrationOps { opt[String]('i', "es-index") .valueName("") .text("TheHive3 ElasticSearch index name") - .action((i, c) => addConfig(c, "intput.search.index", i)), + .action((i, c) => addConfig(c, "input.search.index", i)), opt[String]('a', "es-keepalive") .valueName("") .text("TheHive3 ElasticSearch keepalive") diff --git a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala index aa12e5990f..dd4e70fb06 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/MigrationOps.scala @@ -7,7 +7,7 @@ import org.thp.scalligraph.{EntityId, NotFoundError, RichOptionTry} import org.thp.thehive.migration.dto.{InputAlert, InputAudit, InputCase, InputCaseTemplate} import play.api.Logger -import scala.collection.{immutable, mutable} +import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} @@ -251,10 +251,10 @@ trait MigrationOps { output.createJobObservable ) caseEntitiesIds = caseTaskIds ++ caseTaskLogIds ++ caseObservableIds ++ jobIds ++ jobObservableIds :+ caseId - actionSource = Source(caseEntitiesIds.to[immutable.Iterable]).flatMapConcat(id => input.listAction(id.inputId)) + actionSource = input.listActions(caseEntitiesIds.map(_.inputId).distinct) actionIds <- migrateWithParent("Action", caseEntitiesIds, actionSource, output.createAction) caseEntitiesAuditIds = caseEntitiesIds ++ actionIds - auditSource = Source(caseEntitiesAuditIds.to[immutable.Iterable]).flatMapConcat(id => input.listAudit(id.inputId, filter)) + auditSource = input.listAudits(caseEntitiesAuditIds.map(_.inputId).distinct, filter) _ <- migrateAudit(caseEntitiesAuditIds, auditSource, output.createAudit) } yield Some(caseId) } @@ -282,10 +282,10 @@ trait MigrationOps { output.createAlertObservable ) alertEntitiesIds = alertId +: alertObservableIds - actionSource = Source(alertEntitiesIds.to[immutable.Iterable]).flatMapConcat(id => input.listAction(id.inputId)) + actionSource = input.listActions(alertEntitiesIds.map(_.inputId).distinct) actionIds <- migrateWithParent("Action", alertEntitiesIds, actionSource, output.createAction) alertEntitiesAuditIds = alertEntitiesIds ++ actionIds - auditSource = Source(alertEntitiesAuditIds.to[immutable.Iterable]).flatMapConcat(id => input.listAudit(id.inputId, filter)) + auditSource = input.listAudits(alertEntitiesAuditIds.map(_.inputId).distinct, filter) _ <- migrateAudit(alertEntitiesAuditIds, auditSource, output.createAudit) } yield () } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala index 050e15593d..c6ce3faca7 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala @@ -639,7 +639,16 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(termQuery("relations", "action")))._2 override def listAction(entityId: String): Source[Try[(String, InputAction)], NotUsed] = - dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(Seq(termQuery("relations", "action"), idsQuery(entityId)), Nil, Nil))) + dbFind(Some("all"), Nil)(indexName => + search(indexName).query(bool(Seq(termQuery("relations", "action"), termQuery("objectId", entityId)), Nil, Nil)) + ) + ._1 + .read[(String, InputAction)] + + override def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed] = + dbFind(Some("all"), Nil)(indexName => + search(indexName).query(bool(Seq(termQuery("relations", "action"), termsQuery("objectId", entityIds)), Nil, Nil)) + ) ._1 .read[(String, InputAction)] @@ -679,11 +688,34 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe override def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed] = dbFind(Some("all"), Nil)(indexName => - search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), Nil, Nil)) + search(indexName).query( + bool( + auditFilter(filter) ++ auditIncludeFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), + Nil, + auditExcludeFilter(filter) + ) + ) + )._1.read[(String, InputAudit)] + + override def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed] = + dbFind(Some("all"), Nil)(indexName => + search(indexName).query( + bool( + auditFilter(filter) ++ auditIncludeFilter(filter) :+ termQuery("relations", "audit") :+ termsQuery("objectId", entityIds), + Nil, + auditExcludeFilter(filter) + ) + ) )._1.read[(String, InputAudit)] def countAudit(entityId: String, filter: Filter): Future[Long] = dbFind(Some("0-0"), Nil)(indexName => - search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), Nil, Nil)) + search(indexName).query( + bool( + auditFilter(filter) ++ auditIncludeFilter(filter) :+ termQuery("relations", "audit") :+ termQuery("objectId", entityId), + Nil, + auditExcludeFilter(filter) + ) + ) )._2 } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 68ed3a2167..fe388fb90a 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -32,6 +32,7 @@ import play.api.{Configuration, Environment, Logger} import scala.collection.JavaConverters._ import scala.concurrent.ExecutionContext import scala.util.{Failure, Success, Try} +import org.thp.thehive.controllers.v1.Conversion._ object Output { @@ -220,15 +221,15 @@ class Output @Inject() ( alerts.nonEmpty ) logger.info(s"""Already migrated: - | ${profiles.size} profiles\n - | ${organisations.size} organisations\n - | ${users.size} users\n - | ${impactStatuses.size} impactStatuses\n - | ${resolutionStatuses.size} resolutionStatuses\n - | ${observableTypes.size} observableTypes\n - | ${customFields.size} customFields\n - | ${caseTemplates.size} caseTemplates\n - | ${caseNumbers.size} caseNumbers\n + | ${profiles.size} profiles + | ${organisations.size} organisations + | ${users.size} users + | ${impactStatuses.size} impactStatuses + | ${resolutionStatuses.size} resolutionStatuses + | ${observableTypes.size} observableTypes + | ${customFields.size} customFields + | ${caseTemplates.size} caseTemplates + | ${caseNumbers.size} caseNumbers | ${alerts.size} alerts""".stripMargin) } @@ -579,7 +580,9 @@ class Output @Inject() ( for { task <- taskSrv.getOrFail(taskId) _ = logger.debug(s"Create log in task ${task.title}") - log <- logSrv.create(inputLog.log, task) + log <- logSrv.createEntity(inputLog.log) + _ <- logSrv.taskLogSrv.create(TaskLog(), task, log) + _ <- auditSrv.log.create(log, task, RichLog(log, Nil).toJson) _ = updateMetaData(log, inputLog.metaData) _ <- inputLog.attachments.toTry { inputAttachment => attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { attachment => @@ -717,6 +720,7 @@ class Output @Inject() ( case "Log" => logSrv.getOrFail(entityId) case "Alert" => alertSrv.getOrFail(entityId) case "Job" => jobSrv.getOrFail(entityId) + case "Action" => actionSrv.getOrFail(entityId) case _ => Failure(BadRequestError(s"objectType $entityType is not recognised")) } @@ -744,6 +748,7 @@ class Output @Inject() ( case "Alert" => "Alert" case "Log" | "Task" | "Observable" | "Case" | "Job" => "Case" case "User" => "User" + case "Action" => "Action" // FIXME case other => logger.error(s"Unknown object type: $other") other diff --git a/project/build.properties b/project/build.properties index 08e4d79332..947bdd3020 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.4.1 +sbt.version=1.4.3 diff --git a/project/plugins.sbt b/project/plugins.sbt index 0eacab22d7..171541c183 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.3") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.5") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.3.0") addSbtPlugin("org.thehive-project" % "sbt-github-changelog" % "0.3.0") diff --git a/thehive/app/org/thp/thehive/controllers/dav/Router.scala b/thehive/app/org/thp/thehive/controllers/dav/Router.scala index dbe2c6930b..b746ba3ab7 100644 --- a/thehive/app/org/thp/thehive/controllers/dav/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/dav/Router.scala @@ -6,6 +6,7 @@ import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database +import org.thp.thehive.models.Permissions import org.thp.thehive.services.AttachmentSrv import play.api.Logger import play.api.http.{HttpEntity, Status, Writeable} @@ -65,7 +66,7 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s def dav(path: String): Action[AnyContent] = entrypoint("dav") .extract("xml", FieldsParser.xml.on("xml")) - .authRoTransaction(db) { implicit request => implicit graph => + .authPermittedRoTransaction(db, Permissions.accessTheHiveFS) { implicit request => implicit graph => val pathElements = path.split('/').toList.filterNot(_.isEmpty) val baseUrl = if (request.uri.endsWith("/")) request.uri @@ -102,7 +103,7 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s def downloadFile(id: String): Action[AnyContent] = entrypoint("download attachment") - .authRoTransaction(db) { request => implicit graph => + .authPermittedRoTransaction(db, Permissions.accessTheHiveFS) { request => implicit graph => attachmentSrv.getOrFail(EntityIdOrName(id)).map { attachment => val range = request.headers.get("Range") range match { @@ -129,7 +130,7 @@ class Router @Inject() (entrypoint: Entrypoint, vfs: VFS, @Named("with-thehive-s def head(path: String): Action[AnyContent] = entrypoint("head") - .authRoTransaction(db) { implicit request => implicit graph => + .authPermittedRoTransaction(db, Permissions.accessTheHiveFS) { implicit request => implicit graph => val pathElements = path.split('/').toList vfs .get(pathElements) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index a3187ae1f3..d0ba0e18fe 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -10,7 +10,7 @@ import org.thp.scalligraph.controllers._ import org.thp.scalligraph.models.{Database, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ -import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} +import org.thp.scalligraph.traversal.{Converter, IdentityConverter, IteratorOutput, Traversal} import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityId, EntityIdOrName, EntityName, InvalidFormatAttributeError, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase} @@ -316,6 +316,10 @@ class AlertCtrl @Inject() ( attachmentSrv .create(filename, contentType, data) .flatMap(attachment => observableSrv.create(observable.toObservable, attachmentType, attachment, observable.tags, Nil)) + case Array(filename, contentType) => + attachmentSrv + .create(filename, contentType, Array.emptyByteArray) + .flatMap(attachment => observableSrv.create(observable.toObservable, attachmentType, attachment, observable.tags, Nil)) case data => Failure(InvalidFormatAttributeError("artifacts.data", "filename;contentType;base64value", Set.empty, FString(data.mkString(";")))) } @@ -407,17 +411,22 @@ class PublicAlert @Inject() ( .property("read", UMapping.boolean)(_.field.updatable) .property("follow", UMapping.boolean)(_.field.updatable) .property("status", UMapping.string)( - _.select( - _.project( + _.select { alerts => + val readAndCase = alerts.project( _.byValue(_.read) .by(_.`case`.limit(1).count) - ).domainMap { - case (false, caseCount) if caseCount == 0L => "New" - case (false, _) => "Updated" - case (true, caseCount) if caseCount == 0L => "Ignored" - case (true, _) => "Imported" - } - ).readonly + ) + readAndCase.graphMap[String, String, IdentityConverter[String]]( + jmap => + readAndCase.converter.apply(jmap) match { + case (false, caseCount) if caseCount == 0L => "New" + case (false, _) => "Updated" + case (true, caseCount) if caseCount == 0L => "Ignored" + case (true, _) => "Imported" + }, + Converter.identity[String] + ) + }.readonly ) .property("summary", UMapping.string.optional)(_.field.updatable) .property("user", UMapping.string)(_.field.updatable) diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index c48612d9af..f8fcd8aeed 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -288,10 +288,17 @@ class PublicCase @Inject() ( } yield Json.obj("impactStatus" -> impactStatus) }) .property("customFields", UMapping.jsonNative)(_.subSelect { - case (FPathElem(_, FPathElem(name, _)), caseSteps) => - caseSteps - .customFields(EntityIdOrName(name)) - .jsonValue + case (FPathElem(_, FPathElem(name, _)), caseTraversal) => + db + .roTransaction(implicit graph => customFieldSrv.get(EntityIdOrName(name)).value(_.`type`).getOrFail("CustomField")) + .map { + case CustomFieldType.boolean => caseTraversal.customFields(EntityIdOrName(name)).value(_.booleanValue).domainMap(v => JsBoolean(v)) + case CustomFieldType.date => caseTraversal.customFields(EntityIdOrName(name)).value(_.dateValue).domainMap(v => JsNumber(v.getTime)) + case CustomFieldType.float => caseTraversal.customFields(EntityIdOrName(name)).value(_.floatValue).domainMap(v => JsNumber(v)) + case CustomFieldType.integer => caseTraversal.customFields(EntityIdOrName(name)).value(_.integerValue).domainMap(v => JsNumber(v)) + case CustomFieldType.string => caseTraversal.customFields(EntityIdOrName(name)).value(_.stringValue).domainMap(v => JsString(v)) + } + .getOrElse(caseTraversal.constant2(null)) case (_, caseSteps) => caseSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) } .filter { diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala index 6973c978b2..5a1d824314 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala @@ -68,7 +68,13 @@ class CaseTemplateCtrl @Inject() ( .can(Permissions.manageCaseTemplate), propertyUpdaters ) - .map(_ => Results.NoContent) + .flatMap { + case (caseTemplates, _) => + caseTemplates + .richCaseTemplate + .getOrFail("CaseTemplate") + .map(richCaseTemplate => Results.Ok(richCaseTemplate.toJson)) + } } def delete(caseTemplateNameOrId: String): Action[AnyContent] = diff --git a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala index 16a8ceaaf5..65599ccb62 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala @@ -215,7 +215,7 @@ class DescribeCtrl @Inject() ( prop.mapping.domainTypeClass match { case c if c == classOf[Boolean] || c == classOf[JBoolean] => Seq(PropertyDescription(prop.propertyName, "boolean")) case c if c == classOf[Date] => Seq(PropertyDescription(prop.propertyName, "date")) - case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "hash")) + case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "string")) case c if classOf[Number].isAssignableFrom(c) => Seq(PropertyDescription(prop.propertyName, "number")) case c if c == classOf[String] => Seq(PropertyDescription(prop.propertyName, "string")) case _ => diff --git a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala index 583a43105f..ca1bc469a4 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala @@ -55,7 +55,13 @@ class LogCtrl @Inject() ( .can(Permissions.manageTask), propertyUpdaters ) - .map(_ => Results.NoContent) + .flatMap { + case (logs, _) => + logs + .richLog + .getOrFail("Log") + .map(richLog => Results.Ok(richLog.toJson)) + } } def delete(logId: String): Action[AnyContent] = @@ -90,6 +96,10 @@ class PublicLog @Inject() (logSrv: LogSrv, organisationSrv: OrganisationSrv) ext .property("deleted", UMapping.boolean)(_.field.updatable) .property("startDate", UMapping.date)(_.rename("date").readonly) .property("status", UMapping.string)(_.select(_.constant("Ok")).readonly) - .property("attachment", UMapping.string)(_.select(_.attachments.value(_.attachmentId)).readonly) + .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) + .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 9fadc80a6a..629711357d 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -133,7 +133,13 @@ class ObservableCtrl @Inject() ( _.get(EntityIdOrName(observableId)).can(Permissions.manageObservable), propertyUpdaters ) - .map(_ => Results.NoContent) + .flatMap { + case (observables, _) => + observables + .richObservable + .getOrFail("Observable") + .map(richObservable => Results.Ok(richObservable.toJson)) + } } def findSimilar(observableId: String): Action[AnyContent] = @@ -301,8 +307,9 @@ class PublicObservable @Inject() ( .property("dataType", UMapping.string)(_.select(_.observableType.value(_.name)).readonly) .property("data", UMapping.string.optional)(_.select(_.data.value(_.data)).readonly) .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) - .property("attachment.hashes", UMapping.hash)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala index c783dc61ea..b55bbac9f1 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala @@ -149,12 +149,14 @@ class ParentQueryInputFilter(parentFilter: InputQuery[Traversal.Unk, Traversal.U authContext: AuthContext ): Traversal.Unk = { def filter[F, T: ru.TypeTag](t: Traversal.V[F] => Traversal.V[T]): Traversal.Unk = - parentFilter( - db, - publicProperties, - ru.typeOf[Traversal.V[T]], - t(traversal.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], - authContext + traversal.filter(parent => + parentFilter( + db, + publicProperties, + ru.typeOf[Traversal.V[T]], + t(parent.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], + authContext + ) ) RichType @@ -189,12 +191,14 @@ class ChildQueryInputFilter(childType: String, childFilter: InputQuery[Traversal authContext: AuthContext ): Traversal.Unk = { def filter[F, T: ru.TypeTag](t: Traversal.V[F] => Traversal.V[T]): Traversal.Unk = - childFilter( - db, - publicProperties, - ru.typeOf[Traversal.V[T]], - t(traversal.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], - authContext + traversal.filter(child => + childFilter( + db, + publicProperties, + ru.typeOf[Traversal.V[T]], + t(child.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk], + authContext + ) ) RichType diff --git a/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala index bf4393a499..2f46fa2bb9 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/UserCtrl.scala @@ -117,7 +117,6 @@ class UserCtrl @Inject() ( .update(userSrv.get(EntityIdOrName(userId)), propertyUpdaters) // Authorisation is managed in public properties .flatMap { case (user, _) => user.richUser.getOrFail("User") } } yield Results.Ok(user.toJson) - } def setPassword(userId: String): Action[AnyContent] = diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 5c569fb0d0..ac556fca70 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -369,7 +369,7 @@ object Conversion { case (richObservable, extraData) => richObservable .into[OutputObservable] - .withFieldConst(_._type, "case_artifact") + .withFieldConst(_._type, "Observable") .withFieldComputed(_._id, _._id.toString) .withFieldComputed(_.dataType, _.`type`.name) .withFieldComputed(_.startDate, _.observable._createdAt) diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index ac393112f8..33e3fae52e 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -18,6 +18,7 @@ import play.api.cache.SyncCacheApi import play.api.inject.Injector import play.api.libs.json._ import play.api.mvc.{Action, AnyContent, Results} +import org.thp.thehive.controllers.v0.{QueryCtrl => QueryCtrlV0} import scala.concurrent.duration.Duration import scala.util.{Failure, Success, Try} @@ -72,14 +73,15 @@ class DescribeCtrl @Inject() ( def describeCortexEntity( name: String, className: String, - packageName: String = "org.thp.thehive.connector.cortex.controllers.v1" + packageName: String = "org.thp.thehive.connector.cortex.controllers.v0" ): Option[EntityDescription] = Try( EntityDescription( name, injector .instanceOf(getClass.getClassLoader.loadClass(s"$packageName.$className")) - .asInstanceOf[QueryableCtrl] + .asInstanceOf[QueryCtrlV0] + .publicData .publicProperties .list .flatMap(propertyToJson(name, _)) @@ -103,8 +105,8 @@ class DescribeCtrl @Inject() ( EntityDescription("profile", profileCtrl.publicProperties.list.flatMap(propertyToJson("profile", _))) // EntityDescription("dashboard", dashboardCtrl.publicProperties.list.flatMap(propertyToJson("dashboard", _))), // EntityDescription("page", pageCtrl.publicProperties.list.flatMap(propertyToJson("page", _))) - ) ++ describeCortexEntity("case_artifact_job", "/connector/cortex/job", "JobCtrl") ++ - describeCortexEntity("action", "/connector/cortex/action", "ActionCtrl") + ) ++ describeCortexEntity("case_artifact_job", "JobCtrl") ++ + describeCortexEntity("action", "ActionCtrl") } implicit val propertyDescriptionWrites: Writes[PropertyDescription] = @@ -207,7 +209,7 @@ class DescribeCtrl @Inject() ( prop.mapping.domainTypeClass match { case c if c == classOf[Boolean] || c == classOf[JBoolean] => Seq(PropertyDescription(prop.propertyName, "boolean")) case c if c == classOf[Date] => Seq(PropertyDescription(prop.propertyName, "date")) - case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "hash")) + case c if c == classOf[Hash] => Seq(PropertyDescription(prop.propertyName, "string")) case c if classOf[Number].isAssignableFrom(c) => Seq(PropertyDescription(prop.propertyName, "number")) case c if c == classOf[String] => Seq(PropertyDescription(prop.propertyName, "string")) case _ => diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala index 0a4a08dfe3..fcd7e2be74 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogCtrl.scala @@ -41,9 +41,9 @@ class LogCtrl @Inject() ( override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Log], IteratorOutput]( "page", FieldsParser[OutputParam], - (range, logSteps, _) => + (range, logSteps, authContext) => logSteps.richPage(range.from, range.to, range.extraData.contains("total"))( - _.richLogWithCustomRenderer(logStatsRenderer(range.extraData - "total")) + _.richLogWithCustomRenderer(logStatsRenderer(range.extraData - "total")(authContext)) ) ) override val outputQuery: Query = Query.output[RichLog, Traversal.V[Log]](_.richLog) diff --git a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala index a06de4a596..6b160c1635 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/LogRenderer.scala @@ -4,18 +4,30 @@ import java.lang.{Long => JLong} import java.util.{List => JList, Map => JMap} import org.apache.tinkerpop.gremlin.structure.Vertex +import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models.Log +import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.LogOps._ import org.thp.thehive.services.TaskOps._ import play.api.libs.json._ trait LogRenderer { - def taskParent: Traversal.V[Log] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = - _.task.richTask.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) + def caseParent(implicit + authContext: AuthContext + ): Traversal.V[Log] => Traversal[JsValue, JList[JMap[String, Any]], Converter[JsValue, JList[JMap[String, Any]]]] = + _.`case`.richCase.fold.domainMap(_.headOption.fold[JsValue](JsNull)(_.toJson)) + + def taskParent(implicit + authContext: AuthContext + ): Traversal.V[Log] => Traversal[JsValue, JMap[String, Any], Converter[JsValue, JMap[String, Any]]] = + _.task.project(_.by(_.richTask.fold).by(_.`case`.richCase.fold)).domainMap { + case (task, case0) => + task.headOption.fold[JsValue](JsNull)(_.toJson.as[JsObject] + ("case" -> case0.headOption.fold[JsValue](JsNull)(_.toJson))) + } def taskParentId: Traversal.V[Log] => Traversal[JsValue, JList[Vertex], Converter[JsValue, JList[Vertex]]] = _.task.fold.domainMap(_.headOption.fold[JsValue](JsNull)(c => JsString(c._id.toString))) @@ -23,33 +35,35 @@ trait LogRenderer { def actionCount: Traversal.V[Log] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = _.in("ActionContext").count.domainMap(JsNumber(_)) - def logStatsRenderer(extraData: Set[String]): Traversal.V[Log] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { - traversal => - def addData[G]( - name: String - )(f: Traversal.V[Log] => Traversal[JsValue, G, Converter[JsValue, G]]): Traversal[JsObject, JMap[String, Any], Converter[ - JsObject, - JMap[String, Any] - ]] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { t => - val dataTraversal = f(traversal.start) - t.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.by(dataTraversal.raw)) { jmap => - t.converter(jmap) + (name -> dataTraversal.converter(jmap.get(name).asInstanceOf[G])) - } + def logStatsRenderer(extraData: Set[String])(implicit + authContext: AuthContext + ): Traversal.V[Log] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { traversal => + def addData[G]( + name: String + )(f: Traversal.V[Log] => Traversal[JsValue, G, Converter[JsValue, G]]): Traversal[JsObject, JMap[String, Any], Converter[ + JsObject, + JMap[String, Any] + ]] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = { t => + val dataTraversal = f(traversal.start) + t.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.by(dataTraversal.raw)) { jmap => + t.converter(jmap) + (name -> dataTraversal.converter(jmap.get(name).asInstanceOf[G])) } + } - if (extraData.isEmpty) traversal.constant2[JsObject, JMap[String, Any]](JsObject.empty) - else { - val dataName = extraData.toSeq - dataName.foldLeft[Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]]]( - traversal.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.project(dataName.head, dataName.tail: _*))(_ => - JsObject.empty - ) - ) { - case (f, "task") => addData("task")(taskParent)(f) - case (f, "taskId") => addData("taskId")(taskParentId)(f) - case (f, "actionCount") => addData("actionCount")(actionCount)(f) - case (f, _) => f - } + if (extraData.isEmpty) traversal.constant2[JsObject, JMap[String, Any]](JsObject.empty) + else { + val dataName = extraData.toSeq + dataName.foldLeft[Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]]]( + traversal.onRawMap[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]](_.project(dataName.head, dataName.tail: _*))(_ => + JsObject.empty + ) + ) { + case (f, "case") => addData("case")(caseParent)(f) + case (f, "task") => addData("task")(taskParent)(f) + case (f, "taskId") => addData("taskId")(taskParentId)(f) + case (f, "actionCount") => addData("actionCount")(actionCount)(f) + case (f, _) => f } + } } } diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index fa40aae20b..a41f6a537d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -413,6 +413,7 @@ class Properties @Inject() ( .property("endDate", UMapping.date.optional)(_.field.updatable) .property("order", UMapping.int)(_.field.updatable) .property("dueDate", UMapping.date.optional)(_.field.updatable) + .property("group", UMapping.string)(_.field.updatable) .property("assignee", UMapping.string.optional)(_.select(_.assignee.value(_.login)).custom { case (_, value, vertex, _, graph, authContext) => taskSrv @@ -435,7 +436,11 @@ class Properties @Inject() ( .property("message", UMapping.string)(_.field.updatable) .property("deleted", UMapping.boolean)(_.field.updatable) .property("date", UMapping.date)(_.field.readonly) - .property("attachment", UMapping.string)(_.select(_.attachments.value(_.attachmentId)).readonly) + .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) + .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build lazy val user: PublicProperties = @@ -480,6 +485,10 @@ class Properties @Inject() ( .property("tlp", UMapping.int)(_.field.updatable) .property("dataType", UMapping.string)(_.select(_.observableType.value(_.name)).readonly) .property("data", UMapping.string.optional)(_.select(_.data.value(_.data)).readonly) - // TODO add attachment ? + .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly) + .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly) + .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) + .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) + .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) .build } diff --git a/thehive/app/org/thp/thehive/models/Permissions.scala b/thehive/app/org/thp/thehive/models/Permissions.scala index bf10a22dde..14b45cf5fc 100644 --- a/thehive/app/org/thp/thehive/models/Permissions.scala +++ b/thehive/app/org/thp/thehive/models/Permissions.scala @@ -19,7 +19,8 @@ object Permissions extends Perms { lazy val manageShare: PermissionDesc = PermissionDesc("manageShare", "Manage shares", "organisation") lazy val manageAnalyse: PermissionDesc = PermissionDesc("manageAnalyse", "Run Cortex analyzer", "organisation") lazy val managePage: PermissionDesc = PermissionDesc("managePage", "Manage pages", "organisation") - lazy val manageObservableTemplate: PermissionDesc = PermissionDesc("manageObservableTemplate", "Manage observable types ", "admin") + lazy val manageObservableTemplate: PermissionDesc = PermissionDesc("manageObservableTemplate", "Manage observable types", "admin") + lazy val accessTheHiveFS: PermissionDesc = PermissionDesc("accessTheHiveFS", "Access to TheHiveFS", "organisation") lazy val list: Set[PermissionDesc] = Set( @@ -39,7 +40,8 @@ object Permissions extends Perms { manageShare, manageAnalyse, managePage, - manageObservableTemplate + manageObservableTemplate, + accessTheHiveFS ) // These permissions are available only if the user is in admin organisation, they are removed for other organisations diff --git a/thehive/app/org/thp/thehive/models/Role.scala b/thehive/app/org/thp/thehive/models/Role.scala index 51a2bc80cd..996b9709bd 100644 --- a/thehive/app/org/thp/thehive/models/Role.scala +++ b/thehive/app/org/thp/thehive/models/Role.scala @@ -26,7 +26,8 @@ object Profile { Permissions.manageAction, Permissions.manageShare, Permissions.manageAnalyse, - Permissions.managePage + Permissions.managePage, + Permissions.accessTheHiveFS ) ) val readonly: Profile = Profile("read-only", Set.empty) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 62683434d6..eeab7f15fd 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -3,7 +3,9 @@ package org.thp.thehive.models import java.lang.reflect.Modifier import javax.inject.{Inject, Singleton} +import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.Graph +import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality import org.janusgraph.core.schema.ConsistencyModifier import org.janusgraph.graphdb.types.TypeDefinitionCategory import org.reflections.Reflections @@ -68,12 +70,20 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .noop // .addIndex("Tag", IndexType.unique, "namespace", "predicate", "value") .noop // .addIndex("Audit", IndexType.basic, "requestId", "mainAction") .rebuildIndexes - // release 4.0.0 + //=====[release 4.0.0]===== .updateGraph("Remove cases with a Deleted status", "Case") { traversal => traversal.unsafeHas("status", "Deleted").remove() Success(()) } .addProperty[Option[Boolean]]("Observable", "ignoreSimilarity") + //=====[release 4.0.1]===== + .updateGraph("Add accessTheHiveFS permission to analyst and org-admin profiles", "Profile") { traversal => + traversal + .unsafeHas("name", P.within("org-admin", "analyst")) + .onRaw(_.property(Cardinality.set: Cardinality, "permissions", "accessTheHiveFS", Nil: _*)) // Nil is for disambiguate the overloaded methods + .iterate() + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index b9877b41e2..49c91f328d 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -276,17 +276,16 @@ class AlertSrv @Inject() ( def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = auditSrv .mergeAudits { + // No audit for markAsRead and observables + // Audits for customFields, description and tags val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" - for { _ <- markAsRead(alert._id) _ <- importObservables(alert, `case`) _ <- importCustomFields(alert, `case`) - _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) - // No audit for markAsRead and observables - // Audits for customFields, description and tags - c <- caseSrv.getOrFail(`case`._id) + _ <- alertCaseSrv.create(AlertCase(), alert, `case`) + c <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") details <- Success( Json.obj( "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), @@ -418,10 +417,9 @@ object AlertOps { ) .by( _.selectValues - .unfold .project( - _.by(_.groupCount(_.byValue(_.ioc))) - .by(_.groupCount(_.by(_.typeName))) + _.by(_.unfold.groupCount(_.byValue(_.ioc))) + .by(_.unfold.groupCount(_.by(_.typeName))) ) ) ) diff --git a/thehive/app/org/thp/thehive/services/FlowActor.scala b/thehive/app/org/thp/thehive/services/FlowActor.scala index 44b6da1d26..5c3411498a 100644 --- a/thehive/app/org/thp/thehive/services/FlowActor.scala +++ b/thehive/app/org/thp/thehive/services/FlowActor.scala @@ -1,13 +1,17 @@ package org.thp.thehive.services +import java.util.Date + import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.cluster.singleton.{ClusterSingletonManager, ClusterSingletonManagerSettings, ClusterSingletonProxy, ClusterSingletonProxySettings} import com.google.inject.name.Names import com.google.inject.{Injector, Key => GuiceKey} import javax.inject.{Inject, Provider, Singleton} -import org.apache.tinkerpop.gremlin.process.traversal.Order +import org.apache.tinkerpop.gremlin.process.traversal.{Order, P} import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.EventSrv +import org.thp.scalligraph.services.config.ApplicationConfig.finiteDurationFormat +import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{EntityId, EntityIdOrName} import org.thp.thehive.GuiceAkkaExtension @@ -15,6 +19,8 @@ import org.thp.thehive.services.AuditOps._ import org.thp.thehive.services.CaseOps._ import play.api.cache.SyncCacheApi +import scala.concurrent.duration.FiniteDuration + object FlowActor { case class FlowId(organisation: EntityIdOrName, caseId: Option[EntityIdOrName]) { override def toString: String = s"$organisation;${caseId.getOrElse("-")}" @@ -25,20 +31,26 @@ object FlowActor { class FlowActor extends Actor { import FlowActor._ - lazy val injector: Injector = GuiceAkkaExtension(context.system).injector - lazy val cache: SyncCacheApi = injector.getInstance(classOf[SyncCacheApi]) - lazy val auditSrv: AuditSrv = injector.getInstance(classOf[AuditSrv]) - lazy val caseSrv: CaseSrv = injector.getInstance(classOf[CaseSrv]) - lazy val db: Database = injector.getInstance(GuiceKey.get(classOf[Database], Names.named("with-thehive-schema"))) - lazy val eventSrv: EventSrv = injector.getInstance(classOf[EventSrv]) + lazy val injector: Injector = GuiceAkkaExtension(context.system).injector + lazy val cache: SyncCacheApi = injector.getInstance(classOf[SyncCacheApi]) + lazy val auditSrv: AuditSrv = injector.getInstance(classOf[AuditSrv]) + lazy val caseSrv: CaseSrv = injector.getInstance(classOf[CaseSrv]) + lazy val db: Database = injector.getInstance(GuiceKey.get(classOf[Database], Names.named("with-thehive-schema"))) + lazy val appConfig: ApplicationConfig = injector.getInstance(classOf[ApplicationConfig]) + lazy val maxAgeConfig: ConfigItem[FiniteDuration, FiniteDuration] = + appConfig.item[FiniteDuration]("flow.maxAge", "Max age of audit logs shown in initial flow") + def fromDate: Date = new Date(System.currentTimeMillis() - maxAgeConfig.get.toMillis) + lazy val eventSrv: EventSrv = injector.getInstance(classOf[EventSrv]) override def preStart(): Unit = eventSrv.subscribe(StreamTopic(), self) override def receive: Receive = { case flowId @ FlowId(organisation, caseId) => val auditIds = cache.getOrElseUpdate(flowId.toString) { db.roTransaction { implicit graph => caseId - .fold(auditSrv.startTraversal.has(_.mainAction, true).visible(organisation))(caseSrv.get(_).audits(organisation)) + .fold(auditSrv.startTraversal.has(_.mainAction, true).has(_._createdAt, P.gt(fromDate)).visible(organisation))( + caseSrv.get(_).audits(organisation) + ) .sort(_.by("_createdAt", Order.desc)) .range(0, 10) ._id diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 7f8f68b25d..9e0aa4f5b5 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -68,11 +68,9 @@ class LogSrv @Inject() (attachmentSrv: AttachmentSrv, auditSrv: AuditSrv, taskSr )(implicit graph: Graph, authContext: AuthContext): Try[(Traversal.V[Log], JsObject)] = auditSrv.mergeAudits(super.update(traversal, propertyUpdaters)) { case (logSteps, updatedFields) => - for { - task <- logSteps.clone().task.getOrFail("Task") - log <- logSteps.getOrFail("Log") - _ <- auditSrv.log.update(log, task, updatedFields) - } yield () + logSteps.clone().project(_.by.by(_.task)).getOrFail("Log").flatMap { + case (log, task) => auditSrv.log.update(log, task, updatedFields) + } } } diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 2a156ec43f..dc47f08504 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -183,10 +183,10 @@ class ObservableSrv @Inject() ( )(implicit graph: Graph, authContext: AuthContext): Try[(Traversal.V[Observable], JsObject)] = auditSrv.mergeAudits(super.update(traversal, propertyUpdaters)) { case (observableSteps, updatedFields) => - for { - observable <- observableSteps.getOrFail("Observable") - _ <- auditSrv.observable.update(observable, updatedFields) - } yield () + observableSteps + .clone() + .getOrFail("Observable") + .flatMap(observable => auditSrv.observable.update(observable, updatedFields)) } } diff --git a/thehive/conf/reference.conf b/thehive/conf/reference.conf index 1bce841547..f9f5bf54d3 100644 --- a/thehive/conf/reference.conf +++ b/thehive/conf/reference.conf @@ -9,6 +9,8 @@ storage { localfs.directory: /data/thehive } +flow.maxAge: 1 day + auth { providers: [ {name: session} diff --git a/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala index a2bba30c28..31c50cec90 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/CaseTemplateCtrlTest.scala @@ -131,7 +131,8 @@ class CaseTemplateCtrlTest extends PlaySpecification with TestAppBuilder { ) val result = app[CaseTemplateCtrl].update("spam")(request) - status(result) must equalTo(204).updateMessage(s => s"$s\n${contentAsString(result)}") + status(result) must equalTo(200).updateMessage(s => s"$s\n${contentAsString(result)}") + contentAsJson(result).as[OutputCaseTemplate].displayName must beEqualTo("patched") val updatedOutput = app[Database].roTransaction { implicit graph => app[CaseTemplateSrv].get(EntityName("spam")).richCaseTemplate.head diff --git a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala index e7ac8f762c..8a5773b794 100644 --- a/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v1/UserCtrlTest.scala @@ -110,7 +110,8 @@ class UserCtrlTest extends PlaySpecification with TestAppBuilder { Permissions.manageObservable, Permissions.manageAlert, Permissions.manageAction, - Permissions.manageConfig + Permissions.manageConfig, + Permissions.accessTheHiveFS ), organisation = "cert" ) diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index de36479d95..258c2c38cc 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -62,7 +62,8 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { Permissions.manageAction, Permissions.manageAnalyse, Permissions.manageShare, - Permissions.managePage + Permissions.managePage, + Permissions.accessTheHiveFS ) ) richCase.tags.map(_.toString) must contain(exactly("testNamespace:testPredicate=\"t1\"", "testNamespace:testPredicate=\"t3\"")) @@ -102,7 +103,8 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { Permissions.manageAction, Permissions.manageAnalyse, Permissions.manageShare, - Permissions.managePage + Permissions.managePage, + Permissions.accessTheHiveFS ) ) richCase.tags.map(_.toString) must contain(exactly("testNamespace:testPredicate=\"t2\"", "testNamespace:testPredicate=\"t1\""))