From d57d7d815bc8beba8da3783086caf66b2a53decd Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 20 Feb 2017 10:30:37 +0100 Subject: [PATCH] #86 Add filtering capabilities to MISP event's list --- .../app/connectors/misp/MispSrv.scala | 2 +- ui/app/index.html | 2 + .../scripts/controllers/misp/MispListCtrl.js | 193 ++++++++++++++++- .../scripts/controllers/misp/MispStatsCtrl.js | 87 ++++++++ ui/app/scripts/services/FilteringSrv.js | 200 ++++++++++++++++++ ui/app/scripts/services/MispSrv.js | 67 ++++-- ui/app/views/app.html | 2 +- ui/app/views/directives/tag-list.html | 4 +- ui/app/views/partials/misp/list.html | 93 +++++--- ui/app/views/partials/misp/list/filters.html | 93 ++++++++ .../views/partials/misp/list/mini-stats.html | 61 ++++++ ui/app/views/partials/misp/list/toolbar.html | 83 ++++++++ ui/bower.json | 2 +- 13 files changed, 840 insertions(+), 49 deletions(-) create mode 100644 ui/app/scripts/controllers/misp/MispStatsCtrl.js create mode 100644 ui/app/scripts/services/FilteringSrv.js create mode 100644 ui/app/views/partials/misp/list/filters.html create mode 100644 ui/app/views/partials/misp/list/mini-stats.html create mode 100644 ui/app/views/partials/misp/list/toolbar.html diff --git a/thehive-misp/app/connectors/misp/MispSrv.scala b/thehive-misp/app/connectors/misp/MispSrv.scala index 0f0c9e5c5f..31711567fb 100644 --- a/thehive-misp/app/connectors/misp/MispSrv.scala +++ b/thehive-misp/app/connectors/misp/MispSrv.scala @@ -373,7 +373,7 @@ class MispSrv @Inject() ( def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Misp, NotUsed], Future[Long]) = { import org.elastic4play.services.QueryDSL._ - findSrv[MispModel, Misp](mispModel, and(queryDef, not("eventStatus" in ("Ignore", "Imported"))), range, sortBy) + findSrv[MispModel, Misp](mispModel, queryDef, range, sortBy) } def stats(queryDef: QueryDef, aggs: Seq[Agg]) = findSrv(mispModel, queryDef, aggs: _*) diff --git a/ui/app/index.html b/ui/app/index.html index 57baf77e67..b167a0e54f 100644 --- a/ui/app/index.html +++ b/ui/app/index.html @@ -136,6 +136,7 @@ + @@ -193,6 +194,7 @@ + diff --git a/ui/app/scripts/controllers/misp/MispListCtrl.js b/ui/app/scripts/controllers/misp/MispListCtrl.js index e8e522d61e..01558f55c5 100644 --- a/ui/app/scripts/controllers/misp/MispListCtrl.js +++ b/ui/app/scripts/controllers/misp/MispListCtrl.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveControllers') - .controller('MispListCtrl', function($scope, $q, $state, $uibModal, MispSrv, AlertSrv) { + .controller('MispListCtrl', function($scope, $q, $state, $uibModal, MispSrv, AlertSrv, FilteringSrv) { var self = this; self.list = []; @@ -9,8 +9,85 @@ self.menu = { follow: false, unfollow: false, + markAsRead: false, selectAll: false }; + self.filtering = new FilteringSrv('misp-section', { + defaults: { + showFilters: false, + showStats: false, + pageSize: 15, + sort: ['-publishDate'] + }, + defaultFilter: { + eventStatus: { + field: 'eventStatus', + label: 'Status', + value: [{ + text: 'New' + }, { + text: 'Update' + }], + filter: '(eventStatus:"New" OR eventStatus:"Update")' + } + }, + filterDefs: { + keyword: { + field: 'keyword', + type: 'string', + defaultValue: [] + }, + eventStatus: { + field: 'eventStatus', + type: 'list', + defaultValue: [], + label: 'Status' + }, + tags: { + field: 'tags', + type: 'list', + defaultValue: [], + label: 'Tags' + }, + org: { + field: 'org', + type: 'list', + defaultValue: [], + label: 'Source' + }, + info: { + field: 'info', + type: 'string', + defaultValue: '', + label: 'Title' + }, + publishDate: { + field: 'publishDate', + type: 'date', + defaultValue: { + from: null, + to: null + }, + label: 'Publish Date' + } + } + }); + self.filtering.initContext('list'); + self.searchForm = { + searchQuery: self.filtering.buildQuery() || '' + }; + + $scope.$watch('misp.list.pageSize', function (newValue) { + self.filtering.setPageSize(newValue); + }); + + this.toggleStats = function () { + this.filtering.toggleStats(); + }; + + this.toggleFilters = function () { + this.filtering.toggleFilters(); + }; self.follow = function(event) { var fn = angular.noop; @@ -120,7 +197,17 @@ }; self.load = function() { - self.list = MispSrv.list($scope, self.resetSelection); + var config = { + scope: $scope, + filter: self.searchForm.searchQuery !== '' ? { + _string: self.searchForm.searchQuery + } : '', + loadAll: false, + sort: self.filtering.context.sort, + pageSize: self.filtering.context.pageSize, + }; + + self.list = MispSrv.list(config, self.resetSelection); }; self.cancel = function() { @@ -132,6 +219,11 @@ self.menu.unfollow = temp.length === 1 && temp[0] === true; self.menu.follow = temp.length === 1 && temp[0] === false; + + + temp = _.uniq(_.pluck(self.selection, 'eventStatus')); + + self.menu.markAsRead = temp.indexOf('Ignore') === -1; }; self.select = function(event) { @@ -163,6 +255,103 @@ }; + this.filter = function () { + self.filtering.filter().then(this.applyFilters); + }; + + this.applyFilters = function () { + self.searchForm.searchQuery = self.filtering.buildQuery(); + self.search(); + }; + + this.clearFilters = function () { + self.filtering.clearFilters().then(this.applyFilters); + }; + + this.addFilter = function (field, value) { + self.filtering.addFilter(field, value).then(this.applyFilters); + }; + + this.removeFilter = function (field) { + self.filtering.removeFilter(field).then(this.applyFilters); + }; + + this.search = function () { + this.list.filter = { + _string: this.searchForm.searchQuery + }; + + this.list.update(); + }; + this.addFilterValue = function (field, value) { + var filterDef = self.filtering.filterDefs[field]; + var filter = self.filtering.activeFilters[field]; + var date; + + if (filter && filter.value) { + if (filterDef.type === 'list') { + if (_.pluck(filter.value, 'text').indexOf(value) === -1) { + filter.value.push({ + text: value + }); + } + } else if (filterDef.type === 'date') { + date = moment(value); + self.filtering.activeFilters[field] = { + value: { + from: date.hour(0).minutes(0).seconds(0).toDate(), + to: date.hour(23).minutes(59).seconds(59).toDate() + } + }; + } else { + filter.value = value; + } + } else { + if (filterDef.type === 'list') { + self.filtering.activeFilters[field] = { + value: [{ + text: value + }] + }; + } else if (filterDef.type === 'date') { + date = moment(value); + self.filtering.activeFilters[field] = { + value: { + from: date.hour(0).minutes(0).seconds(0).toDate(), + to: date.hour(23).minutes(59).seconds(59).toDate() + } + }; + } else { + self.filtering.activeFilters[field] = { + value: value + }; + } + } + + this.filter(); + }; + + this.filterByStatus = function(status) { + self.filtering.clearFilters() + .then(function(){ + self.addFilterValue('eventStatus', status); + }); + }; + + this.sortBy = function(sort) { + self.list.sort = sort; + self.list.update(); + self.filtering.setSort(sort); + }; + + this.getStatuses = function(query) { + return MispSrv.statuses(query); + }; + + this.getSources = function(query) { + return MispSrv.sources(query); + }; + self.load(); }); })(); diff --git a/ui/app/scripts/controllers/misp/MispStatsCtrl.js b/ui/app/scripts/controllers/misp/MispStatsCtrl.js new file mode 100644 index 0000000000..203244edcc --- /dev/null +++ b/ui/app/scripts/controllers/misp/MispStatsCtrl.js @@ -0,0 +1,87 @@ +/** + * Controller for About The Hive modal page + */ +(function() { + 'use strict'; + + angular.module('theHiveControllers').controller('MispStatsCtrl', + function($rootScope, $scope, $stateParams, $timeout, StatSrv, StreamStatSrv, FilteringSrv) { + var self = this; + + this.filtering = FilteringSrv; + + this.bySources = {}; + this.byStatus = {}; + this.byTags = {}; + + // Get stats by tags + StreamStatSrv({ + scope: $scope, + rootId: 'any', + query: {}, + objectType: 'connector/misp', + streamObjectType: 'misp', + field: 'tags', + sort: ['-count'], + limit: 5, + result: {}, + success: function(data){ + self.byTags = self.prepareResult(data); + } + }); + + // Get stats by type + StreamStatSrv({ + scope: $scope, + rootId: 'any', + query: {}, + objectType: 'connector/misp', + streamObjectType: 'misp', + field: 'eventStatus', + result: {}, + success: function(data){ + self.byStatus = self.prepareResult(data); + } + }); + + // Get stats by ioc + StreamStatSrv({ + scope: $scope, + rootId: 'any', + query: {}, + objectType: 'connector/misp', + streamObjectType: 'misp', + field: 'org', + sort: ['-count'], + limit: 5, + result: {}, + success: function(data){ + self.bySources = self.prepareResult(data); + } + }); + + this.prepareResult = function(rawStats) { + var total = rawStats.count; + + var keys = _.without(_.keys(rawStats), 'count'); + var columns = keys.map(function(key) { + return { + key: key, + count: rawStats[key].count + }; + }).sort(function(a, b) { + return a.count <= b.count; + }); + + return { + total: total, + details: columns + }; + }; + + this.filterBy = function(field, value) { + this.filtering.addFilter(field, value); + }; + } + ); +})(); diff --git a/ui/app/scripts/services/FilteringSrv.js b/ui/app/scripts/services/FilteringSrv.js new file mode 100644 index 0000000000..fa98d5e5f0 --- /dev/null +++ b/ui/app/scripts/services/FilteringSrv.js @@ -0,0 +1,200 @@ +(function() { + 'use strict'; + angular.module('theHiveServices') + .service('FilteringSrv', function($q, localStorageService) { + return function(sectionName, config) { + var self = this; + + this.sectionName = sectionName; + this.config = config; + this.defaults = config.defaults || {}; + this.filterDefs = config.filterDefs || {}; + this.defaultFilter = config.defaultFilter || {}; + + this.filters = {}; + this.activeFilters = {}; + this.context = { + state: null, + showFilters: false, + showStats: false, + pageSize: this.defaults.pageSize || 15, + sort: this.defaults.sort || [] + }; + + this.initContext = function(state) { + var storedContext = localStorageService.get(self.sectionName); + if (storedContext) { + self.context = storedContext; + self.filters = storedContext.filters || {}; + self.activeFilters = storedContext.activeFilters || {}; + return; + } else { + self.context = { + state: state, + showFilters: false, + showStats: false, + pageSize: self.defaults.pageSize || 15, + sort: self.defaults.sort || [] + }; + + self.filters = self.defaultFilter; + self.activeFilters = _.mapObject(self.defaultFilter || {}, function(val){ + return _.omit(val, 'field', 'filter'); + }); + + self.storeContext(); + } + }; + + this.initFilters = function() { + self.activeFilters = {}; + _.each(_.keys(self.filterDefs), function(key) { + var def = self.filterDefs[key]; + self.activeFilters[key] = { + field: def.field, + type: def.type, + value: self.hasFilter(key) ? angular.copy(self.getFilterValue(key)) : angular.copy(def.defaultValue) + }; + }); + }; + + this.isEmpty = function(value) { + return value === undefined || value === null || value.length === 0 || (angular.isObject(value) &&_.without(_.values(value), null, undefined, '').length === 0); + }; + + this.clearFilters = function() { + self.filters = {}; + self.activeFilters = {}; + self.storeContext(); + return $q.resolve({}); + }; + + this.filter = function() { + var activeFilters = self.activeFilters; + + _.each(activeFilters, function(filterValue, field /*, filters*/ ) { + var value = filterValue.value; + + if (!self.isEmpty(value)) { + self.addFilter(field, angular.copy(value)); + } else { + self.removeFilter(field); + } + }); + + self.storeContext(); + + return $q.resolve(); + }; + + self.addFilter = function(field, value) { + var query, + filterDef = self.filterDefs[field]; + + // Prepare the filter value + if (field === 'keyword') { + query = value; + } else if (angular.isArray(value) && value.length > 0) { + query = _.map(value, function(val) { + return field + ':"' + val.text + '"'; + }).join(' OR '); + query = '(' + query + ')'; + } else if (filterDef.type === 'date') { + var fromDate = value.from ? moment(value.from).hour(0).minutes(0).seconds(0).valueOf() : '*', + toDate = value.to ? moment(value.to).hour(23).minutes(59).seconds(59).valueOf() : '*'; + + query = field + ':[ ' + fromDate + ' TO ' + toDate + ' ]'; + + } else { + query = field + ':' + value; + } + + self.filters[field] = { + field: field, + label: filterDef.label || field, + value: value, + filter: query + }; + + return $q.resolve(self.filters); + }; + + this.removeFilter = function(field) { + var filter = self.activeFilters[field]; + + if(_.isObject(filter.value) && !_.isArray(filter.value)) { + _.each(filter.value, function(value, key) { + filter.value[key] = null; + }); + } + + delete self.filters[field]; + delete self.activeFilters[field]; + + self.storeContext(); + + return $q.resolve(self.filters); + }; + + this.hasFilter = function(field) { + return self.filters[field]; + }; + + this.hasFilters = function() { + return _.keys(self.filters).length > 0; + }; + + this.countFilters = function() { + return _.keys(self.filters).length; + }; + + this.countSorts = function() { + return self.context.sort.length; + }; + + this.getFilterValue = function(field) { + if (self.filters[field]) { + return self.filters[field].value; + } + }; + + this.buildQuery = function() { + if (_.keys(self.filters).length === 0) { + return; + } + + _.keys(self.filters).map(function(key) { + return self.filters[key].filter; + }).join(' AND '); + + return _.pluck(self.filters, 'filter').join(' AND '); + }; + + this.toggleStats = function() { + self.context.showStats = !self.context.showStats; + self.storeContext(); + }; + + this.toggleFilters = function() { + self.context.showFilters = !self.context.showFilters; + self.storeContext(); + }; + + this.setPageSize = function(pageSize) { + self.context.pageSize = pageSize; + self.storeContext(); + }; + + this.setSort = function(sorts) { + self.context.sort = sorts; + self.storeContext(); + }; + + this.storeContext = function() { + self.context.filters = self.filters; + self.context.activeFilters = self.activeFilters; + localStorageService.set(self.sectionName, self.context); + }; + } + }); +})(); diff --git a/ui/app/scripts/services/MispSrv.js b/ui/app/scripts/services/MispSrv.js index 14dac0cd34..2c38651928 100644 --- a/ui/app/scripts/services/MispSrv.js +++ b/ui/app/scripts/services/MispSrv.js @@ -6,12 +6,14 @@ var baseUrl = './api/connector/misp'; var factory = { - list: function(scope, callback) { + + list: function(config, callback) { return PSearchSrv(undefined, 'connector/misp', { - scope: scope, - sort: '-publishDate', - loadAll: false, - pageSize: 10, + scope: config.scope, + sort: config.sort || '-publishDate', + loadAll: config.loadAll || false, + pageSize: config.pageSize || 10, + filter: config.filter || '', onUpdate: callback || angular.noop, streamObjectType: 'misp' }); @@ -47,17 +49,9 @@ stats: function(scope) { var field = 'eventStatus', - mispStatQuery = { - _not: { - _in: { - _field: 'eventStatus', - _values: ['Ignore', 'Imported'] - } - } - }, result = {}, statConfig = { - query: mispStatQuery, + query: {}, objectType: 'connector/misp', field: field, result: result, @@ -77,6 +71,51 @@ }); return StatSrv.get(statConfig); + }, + + sources: function(query) { + var defer = $q.defer(); + + StatSrv.getPromise({ + objectType: 'connector/misp', + field: 'org', + limit: 1000 + }).then(function(response) { + var sources = []; + + sources = _.map(_.filter(_.keys(response.data), function(source) { + var regex = new RegExp(query, 'gi'); + return regex.test(source); + }), function(source) { + return {text: source}; + }); + + defer.resolve(sources); + }); + + return defer.promise; + }, + + statuses: function(query) { + var defer = $q.defer(); + + $q.resolve([ + {text: 'New'}, + {text: 'Update'}, + {text: 'Imported'}, + {text: 'Ignore'} + ]).then(function(response) { + var statuses = []; + + statuses = _.filter(response, function(status) { + var regex = new RegExp(query, 'gi'); + return regex.test(status.text); + }); + + defer.resolve(statuses); + }); + + return defer.promise; } }; diff --git a/ui/app/views/app.html b/ui/app/views/app.html index c889275e2b..cee2853f7a 100644 --- a/ui/app/views/app.html +++ b/ui/app/views/app.html @@ -45,7 +45,7 @@
  • MISP - {{mispEvents.count}} + {{(mispEvents.New.count || 0) + (mispEvents.Update.count || 0)}}
    • diff --git a/ui/app/views/directives/tag-list.html b/ui/app/views/directives/tag-list.html index 380d4441b5..d8bcbb718b 100644 --- a/ui/app/views/directives/tag-list.html +++ b/ui/app/views/directives/tag-list.html @@ -1 +1,3 @@ -{{tag}} +
    + {{tag}} +
    diff --git a/ui/app/views/partials/misp/list.html b/ui/app/views/partials/misp/list.html index 8c7d35fa77..1aebedec0d 100644 --- a/ui/app/views/partials/misp/list.html +++ b/ui/app/views/partials/misp/list.html @@ -1,21 +1,16 @@ -

    Import MISP events ({{misp.list.total}})

    +
    - + -->
    - +
    + +
    + +
    + +
    +
    +
    +

    List of MISP events ({{misp.list.total || 0}} of {{mispEvents.count}})

    +
    +
    +
      +
    • {{misp.filtering.countFilters()}} + filter(s) applied: +
      • +
    • + + {{filter.label}}: + {{filter.value | filterValue}} + + + +
      • +
    • + Clear filters +
      • +
    +
    +
    +
    + + + +
    +
    +
    - + + - + - - + @@ -62,30 +93,34 @@

    Import MISP events ({{misp.list.total}})

    + - + - - +
    Event IDEvent IDStatus Title SourceSeveritySeverity AttributesTagsDatePublish Date
    - {{::event.eventId}} + {{::event.eventId}} + + {{::event.eventStatus}} - {{::event.eventStatus}} - - {{::event.info}} - - {{::event.info}} +
    + + {{::event.info}} + + {{::event.info}} + - +
    +
    + Tags: + None + {{tag}} +
    		{{event.org}}{{event.org}} {{::event.attributeCount}} - - {{tag}} - - {{event.publishDate | showDate}}{{event.publishDate | showDate}} -
    +
    @@ -95,7 +130,7 @@

    Import MISP events ({{misp.list.total}})

    - +
    diff --git a/ui/app/views/partials/misp/list/filters.html b/ui/app/views/partials/misp/list/filters.html new file mode 100644 index 0000000000..91f2a8be8d --- /dev/null +++ b/ui/app/views/partials/misp/list/filters.html @@ -0,0 +1,93 @@ +
    +
    +

    Filters

    +
    + +
    +
    +
    + +
    + +
    +
    +
    +
    + +
    +
    +
    +
    +
    +
    +
    + +
    +
    +
    + +
    + + + +
    +
    +
    + +
    + + + +
    +
    +
    + +
    + + + +
    +
    +
    +
    +
    + +
    + +
    +
    +
    + +
    +
    +
    + +
    +
    + +
    +
    +
    +
    +
    +
    + +
    + +
    +
    + +
    +
    diff --git a/ui/app/views/partials/misp/list/mini-stats.html b/ui/app/views/partials/misp/list/mini-stats.html new file mode 100644 index 0000000000..6e9d3370a8 --- /dev/null +++ b/ui/app/views/partials/misp/list/mini-stats.html @@ -0,0 +1,61 @@ +
    +
    +

    Statistics

    +
    +
    +
    +
    Events by Status
    +
    + + + + + +
    {{item.key}} + {{item.count}} +
    +
    +
    +
    + +
    +
    +
    Top 5 Sources
    +
    + + + + + +
    {{item.key}} + {{item.count}} +
    +
    +
    +
    + +
    +
    +
    Top 5 tags
    +
    + + + + + +
    {{item.key}} + {{item.count}} +
    +
    +
    +
    +
    + diff --git a/ui/app/views/partials/misp/list/toolbar.html b/ui/app/views/partials/misp/list/toolbar.html new file mode 100644 index 0000000000..f629a7fe57 --- /dev/null +++ b/ui/app/views/partials/misp/list/toolbar.html @@ -0,0 +1,83 @@ +
    +
    + +
    +
    diff --git a/ui/bower.json b/ui/bower.json index 6e4e176cbb..5ee1ca6831 100644 --- a/ui/bower.json +++ b/ui/bower.json @@ -19,7 +19,7 @@ "bootstrap": "~3.3.7", "bootstrap-sass-official": "~3.3.7", "dropzone": "~4.3.0", - "font-awesome": "fontawesome#~4.4.0", + "font-awesome": "fontawesome#^4.7.0", "jquery": "~3.1.0", "moment": "~2.14.1", "ng-csv": "~0.3.6",