From 6838b0180d1f3d534923ab1de3250dad9076c2b9 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 28 Jan 2025 19:52:25 +0000 Subject: [PATCH 1/3] Added more techniques to the `refang_url` function and added their corresponding set of tests --- assemblyline_ui/helper/submission.py | 6 ++- .../helper/test/test_submission.py | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 assemblyline_ui/helper/test/test_submission.py diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 4d7c57c7..1635a465 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -190,7 +190,11 @@ def refang_url(url): ''' Refangs a url of text. Based on source of: https://pypi.org/project/defang/ ''' - new_url = re.sub(r'[\(\[](\.|dot)[\)\]]', '.', url, flags=re.IGNORECASE) + new_url = re.sub(r'[\(\[\{](\.|dot)[\)\]\}]', '.', url, flags=re.IGNORECASE) + new_url = re.sub(r'\\.', '.', new_url, flags=re.IGNORECASE) + new_url = re.sub(r'[\(\[\{]/[\)\]\}]', '/', new_url, flags=re.IGNORECASE) + new_url = re.sub(r'[\(\[\{]:[\)\]\}]', ':', new_url, flags=re.IGNORECASE) + new_url = re.sub(r'[\(\[\{]://[\)\]\}]', '://', new_url, flags=re.IGNORECASE) new_url = re.sub(r'^h[x]{1,2}p([s]?)\[?:\]?//', r'http\1://', new_url, flags=re.IGNORECASE) new_url = re.sub(r'^fxp(s?)\[?:\]?//', r'ftp\1://', new_url, flags=re.IGNORECASE) return new_url diff --git a/assemblyline_ui/helper/test/test_submission.py b/assemblyline_ui/helper/test/test_submission.py new file mode 100644 index 00000000..83b8845b --- /dev/null +++ b/assemblyline_ui/helper/test/test_submission.py @@ -0,0 +1,39 @@ +from assemblyline_ui.helper.submission import refang_url + + +# noinspection PyUnusedLocal +def test_refang_url(): + + # Testing the + assert refang_url('http://examples[.]com') == 'http://examples.com' + assert refang_url('http://examples(.)com') == 'http://examples.com' + assert refang_url('http://examples{.}com') == 'http://examples.com' + assert refang_url('http://examples[.)com') == 'http://examples.com' + assert refang_url('http://examples(.}com') == 'http://examples.com' + assert refang_url('http://examples{.]com') == 'http://examples.com' + + assert refang_url('http://examples[dot]com') == 'http://examples.com' + assert refang_url('http://examples(dot)com') == 'http://examples.com' + assert refang_url('http://examples{dot}com') == 'http://examples.com' + assert refang_url('http://examples[dot)com') == 'http://examples.com' + assert refang_url('http://examples(dot}com') == 'http://examples.com' + assert refang_url('http://examples{dot]com') == 'http://examples.com' + + assert refang_url('http://examples\\.com') == 'http://examples.com' + + assert refang_url('http://examples.com[/]path') == 'http://examples.com/path' + assert refang_url('http://examples.com[/)path') == 'http://examples.com/path' + assert refang_url('http://examples.com[/}path') == 'http://examples.com/path' + + assert refang_url('http[:]//examples.com') == 'http://examples.com' + assert refang_url('http[:)//examples.com') == 'http://examples.com' + assert refang_url('http[:}//examples.com') == 'http://examples.com' + assert refang_url('http[://]examples.com') == 'http://examples.com' + assert refang_url('http[://)examples.com') == 'http://examples.com' + assert refang_url('http[://}examples.com') == 'http://examples.com' + + assert refang_url('hxxp://examples.com') == 'http://examples.com' + assert refang_url('hxXp://examples.com') == 'http://examples.com' + assert refang_url('hXXp://examples.com') == 'http://examples.com' + + assert refang_url('hxXps[:]//test\\.example[.)com{.]uk[dot)test[/]path') == 'https://test.example.com.uk.test/path' From 3517b8b3ddfa9054f3fd46dddd1e27ac3e7141b2 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 28 Jan 2025 19:59:29 +0000 Subject: [PATCH 2/3] Changed the file name --- .../test/{test_submission.py => test_submission_helpers.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename assemblyline_ui/helper/test/{test_submission.py => test_submission_helpers.py} (100%) diff --git a/assemblyline_ui/helper/test/test_submission.py b/assemblyline_ui/helper/test/test_submission_helpers.py similarity index 100% rename from assemblyline_ui/helper/test/test_submission.py rename to assemblyline_ui/helper/test/test_submission_helpers.py From de58eb3a84c35f90dc48f16655c3c7bff55cb7d3 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Wed, 29 Jan 2025 16:51:17 +0000 Subject: [PATCH 3/3] Changed the file name of the submission helper test file --- .../{test_submission_helpers.py => test_submission_helper.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename assemblyline_ui/helper/test/{test_submission_helpers.py => test_submission_helper.py} (100%) diff --git a/assemblyline_ui/helper/test/test_submission_helpers.py b/assemblyline_ui/helper/test/test_submission_helper.py similarity index 100% rename from assemblyline_ui/helper/test/test_submission_helpers.py rename to assemblyline_ui/helper/test/test_submission_helper.py