Improvement/RefangURL (master)

assemblyline_ui/helper/submission.py

@@ -190,7 +190,11 @@ def refang_url(url):
     '''
     Refangs a url of text. Based on source of: https://pypi.org/project/defang/
     '''
-    new_url = re.sub(r'[\(\[](\.|dot)[\)\]]', '.', url, flags=re.IGNORECASE)
+    new_url = re.sub(r'[\(\[\{](\.|dot)[\)\]\}]', '.', url, flags=re.IGNORECASE)
+    new_url = re.sub(r'\\.', '.', new_url, flags=re.IGNORECASE)
+    new_url = re.sub(r'[\(\[\{]/[\)\]\}]', '/', new_url, flags=re.IGNORECASE)
+    new_url = re.sub(r'[\(\[\{]:[\)\]\}]', ':', new_url, flags=re.IGNORECASE)
+    new_url = re.sub(r'[\(\[\{]://[\)\]\}]', '://', new_url, flags=re.IGNORECASE)
     new_url = re.sub(r'^h[x]{1,2}p([s]?)\[?:\]?//', r'http\1://', new_url, flags=re.IGNORECASE)
     new_url = re.sub(r'^fxp(s?)\[?:\]?//', r'ftp\1://', new_url, flags=re.IGNORECASE)
     return new_url

assemblyline_ui/helper/test/test_submission_helpers.py

@@ -0,0 +1,39 @@
+from assemblyline_ui.helper.submission import refang_url
+
+
+# noinspection PyUnusedLocal
+def test_refang_url():
+
+    # Testing the
+    assert refang_url('http://examples[.]com') == 'http://examples.com'
+    assert refang_url('http://examples(.)com') == 'http://examples.com'
+    assert refang_url('http://examples{.}com') == 'http://examples.com'
+    assert refang_url('http://examples[.)com') == 'http://examples.com'
+    assert refang_url('http://examples(.}com') == 'http://examples.com'
+    assert refang_url('http://examples{.]com') == 'http://examples.com'
+
+    assert refang_url('http://examples[dot]com') == 'http://examples.com'
+    assert refang_url('http://examples(dot)com') == 'http://examples.com'
+    assert refang_url('http://examples{dot}com') == 'http://examples.com'
+    assert refang_url('http://examples[dot)com') == 'http://examples.com'
+    assert refang_url('http://examples(dot}com') == 'http://examples.com'
+    assert refang_url('http://examples{dot]com') == 'http://examples.com'
+
+    assert refang_url('http://examples\\.com') == 'http://examples.com'
+
+    assert refang_url('http://examples.com[/]path') == 'http://examples.com/path'
+    assert refang_url('http://examples.com[/)path') == 'http://examples.com/path'
+    assert refang_url('http://examples.com[/}path') == 'http://examples.com/path'
+
+    assert refang_url('http[:]//examples.com') == 'http://examples.com'
+    assert refang_url('http[:)//examples.com') == 'http://examples.com'
+    assert refang_url('http[:}//examples.com') == 'http://examples.com'
+    assert refang_url('http[://]examples.com') == 'http://examples.com'
+    assert refang_url('http[://)examples.com') == 'http://examples.com'
+    assert refang_url('http[://}examples.com') == 'http://examples.com'
+
+    assert refang_url('hxxp://examples.com') == 'http://examples.com'
+    assert refang_url('hxXp://examples.com') == 'http://examples.com'
+    assert refang_url('hXXp://examples.com') == 'http://examples.com'
+
+    assert refang_url('hxXps[:]//test\\.example[.)com{.]uk[dot)test[/]path') == 'https://test.example.com.uk.test/path'