Harden network requests, downloads, and path handling

- Add core/network_utils.py with URL validation, public-IP enforcement,
  redirect validation, and safe request_public_url/validate_public_http_url helpers
- Replace direct requests calls in downloader/type_utils/sources with
  request_public_url so redirects are validated before follow-up
- Replace netloc string checks with host_matches_domain for CivitAI,
  CivArchive, and HuggingFace host matching
- Add HuggingFace Xet download backend with progress reporting and
  cancellation support via hf_xet/huggingface_hub
- Harden aria2 installer: reject symlinks/special files in archives,
  restrict custom aria2c paths to the managed install, and update UI copy
- Normalize paths with realpath and enforce configured model roots in
  local hash/metadata routes, plus remove caller-selected metadata_path
  injection in sidecar writing
- Sanitize download/network errors to strip signed query strings before
  logging or showing them to the user

core/network_utils.py

@@ -0,0 +1,163 @@
+"""Network security helpers for outbound HTTP requests."""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+from typing import Any, Dict, Optional, Tuple
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+
+REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
+SENSITIVE_REDIRECT_HEADERS = {"authorization", "cookie", "proxy-authorization"}
+
+
+class UnsafeUrlError(ValueError):
+    """Raised when an outbound URL could access a non-public network target."""
+
+
+def host_matches_domain(host: Any, *domains: str) -> bool:
+    """Return True for an exact domain or one of its real DNS subdomains."""
+    normalized_host = str(host or "").strip().lower().rstrip(".")
+    if not normalized_host:
+        return False
+    for domain in domains:
+        normalized_domain = str(domain or "").strip().lower().rstrip(".")
+        if normalized_domain and (
+            normalized_host == normalized_domain
+            or normalized_host.endswith(f".{normalized_domain}")
+        ):
+            return True
+    return False
+
+
+def _is_public_ip(address: str) -> bool:
+    try:
+        parsed = ipaddress.ip_address(str(address).split("%", 1)[0])
+    except ValueError:
+        return False
+    if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped:
+        parsed = parsed.ipv4_mapped
+    return bool(parsed.is_global)
+
+
+def validate_public_http_url(url: Any) -> str:
+    """Validate an HTTP(S) URL and reject local, private, or reserved targets."""
+    text = str(url or "").strip()
+    if not text or len(text) > 8192:
+        raise UnsafeUrlError("A valid download URL is required")
+
+    try:
+        parsed = urlparse(text)
+        port = parsed.port
+    except ValueError as exc:
+        raise UnsafeUrlError("The download URL is malformed") from exc
+
+    if parsed.scheme.lower() not in {"http", "https"}:
+        raise UnsafeUrlError("Only http and https download URLs are supported")
+    if parsed.username is not None or parsed.password is not None:
+        raise UnsafeUrlError("Credentials embedded in download URLs are not allowed")
+
+    host = str(parsed.hostname or "").strip().lower().rstrip(".")
+    if not host:
+        raise UnsafeUrlError("The download URL must include a hostname")
+    if host_matches_domain(host, "localhost"):
+        raise UnsafeUrlError("Localhost download URLs are not allowed")
+
+    try:
+        direct_ip = ipaddress.ip_address(host.split("%", 1)[0])
+    except ValueError:
+        direct_ip = None
+
+    if direct_ip is not None:
+        if not _is_public_ip(str(direct_ip)):
+            raise UnsafeUrlError("Private, local, and reserved download addresses are not allowed")
+        return text
+
+    try:
+        ascii_host = host.encode("idna").decode("ascii")
+        addresses = {
+            item[4][0]
+            for item in socket.getaddrinfo(
+                ascii_host,
+                port or (443 if parsed.scheme.lower() == "https" else 80),
+                type=socket.SOCK_STREAM,
+            )
+            if item and len(item) > 4 and item[4]
+        }
+    except (OSError, UnicodeError) as exc:
+        raise UnsafeUrlError("The download hostname could not be resolved") from exc
+
+    if not addresses:
+        raise UnsafeUrlError("The download hostname did not resolve to an address")
+    if any(not _is_public_ip(address) for address in addresses):
+        raise UnsafeUrlError("Private, local, and reserved download addresses are not allowed")
+    return text
+
+
+def _url_origin(url: str) -> Tuple[str, str, int]:
+    parsed = urlparse(url)
+    scheme = parsed.scheme.lower()
+    port = parsed.port or (443 if scheme == "https" else 80)
+    return scheme, str(parsed.hostname or "").lower().rstrip("."), port
+
+
+def _redirect_headers(headers: Dict[str, str], source_url: str, target_url: str) -> Dict[str, str]:
+    if _url_origin(source_url) == _url_origin(target_url):
+        return dict(headers)
+    return {
+        key: value
+        for key, value in headers.items()
+        if str(key).lower() not in SENSITIVE_REDIRECT_HEADERS
+    }
+
+
+def request_public_url(
+    method: str,
+    url: Any,
+    *,
+    headers: Optional[Dict[str, str]] = None,
+    timeout: Any = 30,
+    stream: bool = True,
+    max_redirects: int = 5,
+) -> Tuple[requests.Response, str, Dict[str, str]]:
+    """Perform a GET/HEAD request while validating every redirect target."""
+    current_url = validate_public_http_url(url)
+    current_headers = dict(headers or {})
+    request_method = str(method or "GET").upper()
+    if request_method not in {"GET", "HEAD"}:
+        raise ValueError("request_public_url supports only GET and HEAD")
+
+    for redirect_count in range(max_redirects + 1):
+        request_func = requests.head if request_method == "HEAD" else requests.get
+        response = request_func(
+            current_url,
+            headers=current_headers,
+            allow_redirects=False,
+            stream=stream,
+            timeout=timeout,
+        )
+        if response.status_code not in REDIRECT_STATUS_CODES:
+            return response, current_url, current_headers
+
+        location = response.headers.get("Location") or response.headers.get("location")
+        if not location:
+            return response, current_url, current_headers
+        if redirect_count >= max_redirects:
+            response.close()
+            raise UnsafeUrlError("The download URL has too many redirects")
+
+        candidate_url = urljoin(current_url, str(location).strip())
+        try:
+            next_url = validate_public_http_url(candidate_url)
+        except UnsafeUrlError:
+            response.close()
+            raise
+        next_headers = _redirect_headers(current_headers, current_url, next_url)
+        response.close()
+        current_url = next_url
+        current_headers = next_headers
+
+    raise UnsafeUrlError("The download URL has too many redirects")