Skip to content

Commit

Permalink
Merge pull request #236 from CybercentreCanada/hotfix/group_search_total
Browse files Browse the repository at this point in the history 
Allow alert and error list API to track total hits
  • Loading branch information
@cccs-sgaron
cccs-sgaron authored Jul 21, 2021
2 parents 2bef8fa + 9d88967 commit 945b1de
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 34 deletions.
47 changes: 27 additions & 20 deletions assemblyline_ui/api/v4/alert.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,14 +230,15 @@ def list_alerts(**kwargs):
None
Arguments:
fq => Post filter queries (you can have multiple of those)
q => Query to apply to the alert list
no_delay => Do not delay alerts
offset => Offset at which we start giving alerts
rows => Numbers of alerts to return
tc_start => Time offset at which we start the time constraint
tc => Time constraint applied to the API
use_archive => List alerts from archive as well (Default: False)
fq => Post filter queries (you can have multiple of those)
q => Query to apply to the alert list
no_delay => Do not delay alerts
offset => Offset at which we start giving alerts
rows => Numbers of alerts to return
tc_start => Time offset at which we start the time constraint
tc => Time constraint applied to the API
use_archive => List alerts from archive as well (Default: False)
track_total_hits => Track the total number of item that match the query (Default: 10 000)
Data Block:
None
Expand Down Expand Up @@ -265,17 +266,18 @@ def list_alerts(**kwargs):
if tc and config.ui.read_only:
tc += config.ui.read_only_offset
timming_filter = get_timming_filter(tc_start, tc)
track_total_hits = request.args.get('track_total_hits', False)

filters = [x for x in request.args.getlist("fq") if x != ""]
if timming_filter:
filters.append(timming_filter)

try:
res = STORAGE.alert.search(
query, offset=offset, rows=rows, fl="alert_id", sort="reporting_ts desc",
query, offset=offset, rows=rows, fl="id", sort="reporting_ts desc",
access_control=user['access_control'],
filters=filters, as_obj=False, use_archive=use_archive)
res['items'] = sorted(STORAGE.alert.multiget([v['alert_id'] for v in res['items']],
filters=filters, as_obj=False, use_archive=use_archive, track_total_hits=track_total_hits)
res['items'] = sorted(STORAGE.alert.multiget([v['id'] for v in res['items']],
as_dictionary=False, as_obj=False),
key=lambda k: k['reporting_ts'], reverse=True)
return make_api_response(res)
Expand All @@ -293,13 +295,15 @@ def list_grouped_alerts(field, **kwargs):
None
Arguments:
fq => Post filter queries (you can have multiple of those)
q => Query to apply to the alert list
no_delay => Do not delay alerts
offset => Offset at which we start giving alerts
rows => Numbers of alerts to return
tc_start => Time offset at which we start the time constraint
tc => Time constraint applied to the API
fq => Post filter queries (you can have multiple of those)
q => Query to apply to the alert list
no_delay => Do not delay alerts
offset => Offset at which we start giving alerts
rows => Numbers of alerts to return
tc_start => Time offset at which we start the time constraint
tc => Time constraint applied to the API
use_archive => List alerts from archive as well (Default: False)
track_total_hits => Track the total number of item that match the query (Default: 10 000)
Data Block:
None
Expand Down Expand Up @@ -329,6 +333,8 @@ def get_dict_item(parent, cur_item):
rows = int(request.args.get('rows', 100))
query = request.args.get('q', "alert_id:*") or "alert_id:*"
tc_start = request.args.get('tc_start', None)
track_total_hits = request.args.get('track_total_hits', False)
use_archive = request.args.get('use_archive', 'false').lower() == 'true'

if not tc_start:
if "no_delay" not in request.args and config.core.alerter.delay != 0:
Expand All @@ -349,7 +355,8 @@ def get_dict_item(parent, cur_item):
try:
res = STORAGE.alert.grouped_search(field, query=query, offset=offset, rows=rows, sort="reporting_ts desc",
group_sort="reporting_ts desc", access_control=user['access_control'],
filters=filters, fl=f"alert_id,{field}", as_obj=False)
filters=filters, fl=f"id,{field}", as_obj=False,
use_archive=use_archive, track_total_hits=track_total_hits)
alert_keys = []
hash_list = []
hint_list = []
Expand All @@ -359,7 +366,7 @@ def get_dict_item(parent, cur_item):
counted_total += item['total']
group_count[item['value']] = item['total']
data = item['items'][0]
alert_keys.append(data['alert_id'])
alert_keys.append(data['id'])
if field in ['file.md5', 'file.sha1', 'file.sha256']:
hash_list.append(get_dict_item(data, field))

Expand Down
15 changes: 9 additions & 6 deletions assemblyline_ui/api/v4/error.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,12 @@ def list_errors(**_):
None
Arguments:
offset => Offset at which we start giving errors
query => Query to apply to the error list
rows => Numbers of errors to return
sort => Sort order
use_archive => List error from archive as well (Default: False)
offset => Offset at which we start giving errors
query => Query to apply to the error list
rows => Numbers of errors to return
sort => Sort order
use_archive => List error from archive as well (Default: False)
track_total_hits => Track the total number of item that match the query (Default: 10 000)
Data Block:
None
Expand All @@ -75,9 +76,11 @@ def list_errors(**_):
query = request.args.get('query', "id:*") or "id:*"
sort = request.args.get('sort', "created desc")
use_archive = request.args.get('use_archive', "false").lower() == 'true'
track_total_hits = request.args.get('track_total_hits', False)

try:
return make_api_response(STORAGE.error.search(query, offset=offset, rows=rows, as_obj=False,
sort=sort, use_archive=use_archive))
sort=sort, use_archive=use_archive,
track_total_hits=track_total_hits))
except SearchException as e:
return make_api_response("", f"The specified search query is not valid. ({e})", 400)
26 changes: 18 additions & 8 deletions assemblyline_ui/api/v4/submission.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -669,9 +669,11 @@ def list_submissions_for_group(group, **kwargs):
None
Arguments:
offset => Offset at which we start giving submissions
rows => Numbers of submissions to return
query => Query to filter to the submission list
offset => Offset at which we start giving submissions
rows => Numbers of submissions to return
query => Query to filter to the submission list
use_archive => List submissions from archive as well (Default: False)
track_total_hits => Track the total number of item that match the query (Default: 10 000)
Data Block:
None
Expand All @@ -698,6 +700,8 @@ def list_submissions_for_group(group, **kwargs):
offset = int(request.args.get('offset', 0))
rows = int(request.args.get('rows', 100))
filters = request.args.get('query', None) or None
track_total_hits = request.args.get('track_total_hits', False)
use_archive = request.args.get('use_archive', 'false').lower() == 'true'

if group == "ALL":
group_query = "id:*"
Expand All @@ -706,7 +710,8 @@ def list_submissions_for_group(group, **kwargs):
try:
return make_api_response(STORAGE.submission.search(group_query, offset=offset, rows=rows, filters=filters,
access_control=user['access_control'],
sort='times.submitted desc', as_obj=False))
sort='times.submitted desc', as_obj=False,
use_archive=use_archive, track_total_hits=track_total_hits))
except SearchException as e:
return make_api_response("", f"SearchException: {e}", 400)

Expand All @@ -721,9 +726,11 @@ def list_submissions_for_user(username, **kwargs):
None
Arguments:
offset => Offset at which we start giving submissions
rows => Numbers of submissions to return
query => Query to filter the submission list
offset => Offset at which we start giving submissions
rows => Numbers of submissions to return
query => Query to filter the submission list
use_archive => List submissions from archive as well (Default: False)
track_total_hits => Track the total number of item that match the query (Default: 10 000)
Data Block:
None
Expand All @@ -750,6 +757,8 @@ def list_submissions_for_user(username, **kwargs):
offset = int(request.args.get('offset', 0))
rows = int(request.args.get('rows', 100))
query = request.args.get('query', None) or None
track_total_hits = request.args.get('track_total_hits', False)
use_archive = request.args.get('use_archive', 'false').lower() == 'true'

account = STORAGE.user.get(username)
if not account:
Expand All @@ -758,7 +767,8 @@ def list_submissions_for_user(username, **kwargs):
try:
return make_api_response(STORAGE.submission.search(f"params.submitter:{username}", offset=offset, rows=rows,
filters=query, access_control=user['access_control'],
sort='times.submitted desc', as_obj=False))
sort='times.submitted desc', as_obj=False,
use_archive=use_archive, track_total_hits=track_total_hits))
except SearchException as e:
return make_api_response("", f"SearchException: {e}", 400)

Expand Down

0 comments on commit 945b1de

Please sign in to comment.