Use more lenient urllib3.util's parse_url

Author: kapyteinaikido

aikido_zen/helpers/get_port_from_url.py

@@ -2,15 +2,15 @@
 Helper function file, see function docstring
 """
 
-from urllib.parse import urlparse
+from urllib3.util import parse_url
 
 
 def get_port_from_url(url, parsed=False):
     """
     Tries to retrieve a port number from the given url
     """
     if not parsed:
-        parsed_url = urlparse(url)
+        parsed_url = parse_url(url)
     else:
         parsed_url = url
 

aikido_zen/helpers/get_port_from_url_test.py

@@ -1,6 +1,6 @@
 import pytest
 from .get_port_from_url import get_port_from_url
-from urllib.parse import urlparse
+from urllib3.util import parse_url
 
 
 def test_get_port_from_url():
@@ -14,14 +14,14 @@ def test_get_port_from_url():
 
 
 def test_get_port_from_parsed_url():
-    assert get_port_from_url(urlparse("http://localhost:4000"), True) == 4000
-    assert get_port_from_url(urlparse("http://localhost"), True) == 80
+    assert get_port_from_url(parse_url("http://localhost:4000"), True) == 4000
+    assert get_port_from_url(parse_url("http://localhost"), True) == 80
     assert (
-        get_port_from_url(urlparse("https://test.com:8080/test?abc=123"), True) == 8080
+        get_port_from_url(parse_url("https://test.com:8080/test?abc=123"), True) == 8080
     )
-    assert get_port_from_url(urlparse("https://localhost"), True) == 443
-    assert get_port_from_url(urlparse("ftp://localhost"), True) is None
+    assert get_port_from_url(parse_url("https://localhost"), True) == 443
+    assert get_port_from_url(parse_url("ftp://localhost"), True) is None
     assert (
-        get_port_from_url(urlparse("http://localhost:1337\\u0000asd.php"), True) is None
+        get_port_from_url(parse_url("http://localhost:1337\\u0000asd.php"), True) is None
     )
-    assert get_port_from_url(urlparse("http://localhost:123123/asd.php"), True) is None
+    assert get_port_from_url(parse_url("http://localhost:123123/asd.php"), True) is None

aikido_zen/helpers/get_subdomains_from_url.py

@@ -1,8 +1,7 @@
 """
 Helper function file, see function docstring
 """
-
-from urllib.parse import urlparse
+from urllib3.util import parse_url
 
 
 def get_subdomains_from_url(url):
@@ -11,7 +10,7 @@ def get_subdomains_from_url(url):
     """
     if not isinstance(url, str):
         return []
-    host = urlparse(url).hostname
+    host = parse_url(url).hostname
     if not host:
         return []
     parts = host.split(".")

aikido_zen/helpers/path_to_string.py

@@ -1,27 +1,33 @@
 """Helper function file"""
 
-from urllib.parse import urlparse
+from urllib3.util import parse_url
 from pathlib import PurePath
 
 
 def path_to_string(path):
     """Converts an obj that represents a path into a string"""
     if isinstance(path, str):
         try:
-            parsed_url = urlparse(path)
+            parsed_url = parse_url(path)
             if parsed_url and parsed_url.scheme == "file":
                 return parsed_url.path
         except Exception:
+            print("can't parse thsi shit! bye")
             return None
         return path
+    print("can't parse thsi shit! bye - 9")
 
     if isinstance(path, bytes):
         try:
             return path.decode("utf-8")
         except UnicodeDecodeError:
+            print("can't parse thsi shit! by - 4 e")
             return None
     if isinstance(path, PurePath):
         # Stringify PurePath. This can still allow path traversal but in extremely
         # limited cases so it's safe to just stringify for now.
         return str(path)
+    
+
+    print("can't parse thsi shit! bye - 1")
     return None

aikido_zen/helpers/try_parse_url.py

@@ -1,12 +1,11 @@
 """Helper function file"""
 
-from urllib.parse import urlparse
-
+from urllib3.util import parse_url
 
 def try_parse_url(url):
     """Tries to parse the url using urlparse"""
     try:
-        parsed_url = urlparse(url)
+        parsed_url = parse_url(url)
         if parsed_url.scheme and parsed_url.netloc:
             return parsed_url
         return None

aikido_zen/helpers/try_parse_url_path.py

@@ -4,15 +4,16 @@
 Includes try_parse_url_path
 """
 
-from urllib.parse import urlparse
+from urllib3.util import parse_url
 import regex as re
 
 
 def try_parse_url(url):
     """try to parse Url with urlparse"""
     try:
-        return urlparse(url)
+        return parse_url(url)
     except ValueError:
+        print("value error! can't parse this shit!")
         return None
 
 

aikido_zen/helpers/urls/normalize_url.py

@@ -1,12 +1,13 @@
 """Helper function file, exports normalize_url"""
 
-from urllib.parse import urlparse, urlunparse
+from urllib3.util import parse_url
+from urllib.parse import urlunparse
 
 
 def normalize_url(url):
     """Normalizes the url"""
     # Parse the URL
-    parsed_url = urlparse(url)
+    parsed_url = parse_url(url)
 
     # Normalize components
     scheme = parsed_url.scheme.lower()  # Lowercase scheme

aikido_zen/vulnerabilities/path_traversal/parse_as_file_url.py

@@ -2,21 +2,22 @@
 Mainly exports `parse_as_file_url`
 """
 
-from urllib.parse import urlparse, urlunparse
+from urllib3.util import parse_url
+from urllib.parse import urlunparse
 from pathlib import Path
 
 
 def parse_as_file_url(path):
     """Convert a file path as a URL to a file path."""
     if path.startswith("file:"):
-        parsed_url = urlparse(path)
+        parsed_url = parse_url(path)
         file_path = Path(parsed_url.path)
     else:
         if not path.startswith("/"):
             path = f"/{path}"
         file_path = Path(path)
         file_url = urlunparse(("file", "", str(file_path), "", "", ""))
-        parsed_url = urlparse(file_url)
+        parsed_url = parse_url(file_url)
 
     normalized_path = Path(parsed_url.path).resolve()
 

aikido_zen/vulnerabilities/ssrf/get_redirect_origin_test.py

@@ -1,11 +1,11 @@
 import pytest
-from urllib.parse import urlparse, urlunparse
+from urllib3.util import parse_url
 from .get_redirect_origin import get_redirect_origin
 
 
 # Helper function to create URL objects
 def create_url(href):
-    return urlparse(href)
+    return parse_url(href)
 
 
 # Test cases

aikido_zen/vulnerabilities/ssrf/is_redirect_to_private_ip_test.py

@@ -1,12 +1,12 @@
 import pytest
 from unittest.mock import MagicMock, patch
 from .is_redirect_to_private_ip import is_redirect_to_private_ip
-from urllib.parse import urlparse, urlunparse
+from urllib3.util import parse_url
 
 
 # Helper function to create URL objects
 def create_url(href):
-    return urlparse(href)
+    return parse_url(href)
 
 
 def test_is_redirect_to_private_ip_success():