diff --git a/py/services/download_manager.py b/py/services/download_manager.py index 1b86369f..484fc449 100644 --- a/py/services/download_manager.py +++ b/py/services/download_manager.py @@ -42,7 +42,7 @@ from ..utils.paid_access import ( ) from ..utils.file_utils import calculate_sha256, calculate_autov3 from ..utils.preview_selection import resolve_mature_threshold, select_preview_media -from ..utils.utils import calculate_filename_for_model, fit_stem_to_path_budget, sanitize_folder_name +from ..utils.utils import calculate_filename_for_model, decode_percent_encoded_filename, fit_stem_to_path_budget, sanitize_folder_name from ..utils.exif_utils import ExifUtils from ..utils.metadata_manager import MetadataManager from ..utils.sidecar_paths import get_metadata_path, get_preview_dir @@ -2241,6 +2241,19 @@ class DownloadManager: file_name = file_info.get("name", "") if not file_name: return {"success": False, "error": "No filename found in file info"} + # CivitAI stores non-ASCII file names percent-encoded; the mini + # endpoint (and CivArchive) return that raw stored name, which + # must be decoded before it becomes the on-disk name — otherwise + # the literal "%E7%8A%AC..." string lands on the filesystem. + decoded_file_name = decode_percent_encoded_filename(file_name) + if decoded_file_name != file_name: + logger.info( + "[download] Decoded percent-encoded filename '%s' to '%s'", + file_name, + decoded_file_name, + ) + file_name = decoded_file_name + file_info = {**file_info, "name": file_name} # Raw stored filenames carry no length cap, and the fixed stem # cap only applies to template renames; trim to the real path # budget (model path and sidecar mirror alike) so the target diff --git a/py/utils/utils.py b/py/utils/utils.py index 0f5800e9..2f02badd 100644 --- a/py/utils/utils.py +++ b/py/utils/utils.py @@ -4,6 +4,7 @@ import logging import os import re from typing import Any, Dict, List, Optional +from urllib.parse import unquote_to_bytes from ..services.service_registry import ServiceRegistry from ..config import config from ..services.settings_manager import get_settings_manager @@ -532,6 +533,32 @@ def _truncate_to_byte_budget(text: str, max_bytes: int) -> str: return encoded[:max_bytes].decode("utf-8", errors="ignore") +def decode_percent_encoded_filename(name: str) -> str: + """Decode a percent-encoded stored filename back to the real name. + + CivitAI's backend stores file names with non-ASCII characters + percent-encoded (URL-quoted): the mini endpoint's ``fileName`` is that + raw stored name (e.g. ``%E7%8A%AC...safetensors``), while the REST + ``files[].name`` is the decoded display name. Saving the raw string + literally would land ``%E7%8A%AC...`` on disk instead of the real + characters. + + A literal ``%`` in an uploader's name is itself stored as ``%25``, so a + decoded result that stays pure ASCII (e.g. ``100%25cool`` -> + ``100%cool``) is treated as a real name and returned unchanged — only + decodes that yield valid UTF-8 with non-ASCII characters are applied. + """ + if "%" not in name: + return name + try: + decoded = unquote_to_bytes(name).decode("utf-8", errors="strict") + except (UnicodeDecodeError, ValueError): + return name + if decoded == name or not any(ord(char) > 127 for char in decoded): + return name + return decoded + + def fit_stem_to_path_budget(dir_path: str, stem: str, extension: str = "") -> str: """Truncate ``stem`` so the file and its companions fit the path budget. diff --git a/tests/services/test_download_manager_basic.py b/tests/services/test_download_manager_basic.py index ac34cb3f..b3b87896 100644 --- a/tests/services/test_download_manager_basic.py +++ b/tests/services/test_download_manager_basic.py @@ -2293,6 +2293,116 @@ async def test_download_falls_back_to_rest_name_when_mini_fails( assert captured["file_path"].endswith("rewritten_v10.safetensors") +@pytest.mark.asyncio +async def test_download_decodes_percent_encoded_raw_file_name( + monkeypatch, scanners, metadata_provider, tmp_path +): + """CivitAI stores non-ASCII file names percent-encoded; the mini + endpoint's raw fileName must be decoded before it lands on disk.""" + manager = DownloadManager() + encoded_name = ( + "%E7%8A%AC%E5%A4%9C%E5%8F%89%E6%97%A5%E6%9A%AE2000s_anime_analog_painting_Style.safetensors" + ) + metadata_provider.payload = { + "id": 42, + "model": {"type": "LORA", "tags": ["fantasy"]}, + "baseModel": "Illustrious", + "creator": {"username": "Author"}, + "files": [ + { + "id": 1001, + "type": "Model", + "primary": True, + "name": "犬夜叉日暮2000s_anime_analog_painting_Style.safetensors", + "downloadUrl": "https://example.invalid/file.safetensors", + } + ], + } + metadata_provider.get_version_file_mini = AsyncMock( + return_value={"fileName": encoded_name} + ) + + captured = {} + + async def fake_execute_download(self, **kwargs): + captured["file_path"] = kwargs["metadata"].file_path + return {"success": True} + + monkeypatch.setattr( + DownloadManager, "_execute_download", fake_execute_download, raising=False + ) + + result = await manager.download_from_civitai( + model_version_id=42, + save_dir=str(tmp_path), + use_default_paths=True, + progress_callback=None, + source=None, + ) + + assert result["success"] is True + assert captured["file_path"].endswith( + "犬夜叉日暮2000s_anime_analog_painting_Style.safetensors" + ) + + +@pytest.mark.asyncio +async def test_download_decodes_percent_encoded_name_for_civarchive_source( + monkeypatch, scanners, metadata_provider, tmp_path +): + """CivArchive serves raw stored names too — the decode applies even + though the mini lookup is skipped.""" + manager = DownloadManager() + encoded_name = ( + "%E7%8A%AC%E5%A4%9C%E5%8F%89%E6%97%A5%E6%9A%AE2000s_anime_analog_painting_Style.safetensors" + ) + metadata_provider.payload = { + "id": 99, + "model": {"type": "LORA", "tags": ["fantasy"]}, + "baseModel": "Illustrious", + "creator": {"username": "Author"}, + "files": [ + { + "id": 1001, + "type": "Model", + "primary": True, + "name": encoded_name, + "downloadUrl": "https://example.invalid/file.safetensors", + } + ], + } + metadata_provider.get_version_file_mini = AsyncMock() + + monkeypatch.setattr( + download_manager, + "get_metadata_provider", + AsyncMock(return_value=metadata_provider), + ) + + captured = {} + + async def fake_execute_download(self, **kwargs): + captured["file_path"] = kwargs["metadata"].file_path + return {"success": True} + + monkeypatch.setattr( + DownloadManager, "_execute_download", fake_execute_download, raising=False + ) + + result = await manager.download_from_civitai( + model_version_id=99, + save_dir=str(tmp_path), + use_default_paths=True, + progress_callback=None, + source="civarchive", + ) + + assert result["success"] is True + assert captured["file_path"].endswith( + "犬夜叉日暮2000s_anime_analog_painting_Style.safetensors" + ) + + @pytest.mark.asyncio async def test_download_skips_mini_lookup_for_civarchive_source( monkeypatch, scanners, metadata_provider, tmp_path diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 3b8d6448..e9300a4d 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -16,6 +16,7 @@ from py.utils.utils import ( calculate_filename_for_model, calculate_recipe_fingerprint, calculate_relative_path_for_model, + decode_percent_encoded_filename, fit_stem_to_path_budget, get_lora_info, get_lora_info_absolute, @@ -744,3 +745,51 @@ def test_calculate_filename_without_target_dir_keeps_fixed_cap(isolated_settings result = calculate_filename_for_model(model_data, "lora") assert len(result) == MAX_FILENAME_STEM_LENGTH + + +# Real raw stored fileName served by CivitAI's mini endpoint for model +# version 1714548: non-ASCII names are percent-encoded server-side. +PERCENT_ENCODED_STORED_NAME = ( + "%E7%8A%AC%E5%A4%9C%E5%8F%89%E6%97%A5%E6%9A%AE%E3%81%8B%E3%81%94" + "%E3%82%812000%E5%B9%B4%E4%BB%A3%E3%82%A2%E3%83%8B%E3%83%A1%E3%82%A2" + "%E3%83%8A%E3%83%AD%E3%82%B0%E4%BD%9C%E7%94%BB%E9%A2%A8illustrious" + "InuyashaHigurashi_Kagome2000s_anime_analog_painting_Styleillustrious" + ".safetensors" +) + + +def test_decode_percent_encoded_filename_decodes_non_ascii_names(): + result = decode_percent_encoded_filename(PERCENT_ENCODED_STORED_NAME) + + assert result == ( + "犬夜叉日暮かごめ2000年代アニメアナログ作画風illustrious" + "InuyashaHigurashi_Kagome2000s_anime_analog_painting_Styleillustrious" + ".safetensors" + ) + + +def test_decode_percent_encoded_filename_passes_plain_names_through(): + assert decode_percent_encoded_filename("model.safetensors") == "model.safetensors" + + +def test_decode_percent_encoded_filename_keeps_ascii_only_escapes_literal(): + # A literal "%" is stored as "%25", so decoding would yield pure ASCII: + # treat the string as a real name and leave it alone. + assert ( + decode_percent_encoded_filename("100%25cool.safetensors") + == "100%25cool.safetensors" + ) + + +def test_decode_percent_encoded_filename_keeps_invalid_escapes_literal(): + assert ( + decode_percent_encoded_filename("100%zzcool.safetensors") + == "100%zzcool.safetensors" + ) + + +def test_decode_percent_encoded_filename_keeps_truncated_escapes_literal(): + # A mid-triplet cut (e.g. from a previous length-truncation bug) does not + # decode to valid UTF-8 and must pass through untouched. + truncated = PERCENT_ENCODED_STORED_NAME[:100] + assert decode_percent_encoded_filename(truncated) == truncated