nodes/willmiao-ComfyUI-Lora-Manager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(workflows): isolate artifact sidecars and hashes

Browse Source
This commit is contained in:
Will Miao
2026-11-07 10:40:00 +08:00
parent afa82bc720
commit 0761f296d1
19 changed files with 428 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/metadata_ops/__init__.py
+3 -5
View File
@@ -172,17 +172,15 @@ async def download_preview(
"""
from ..services.downloader import get_downloader
from ..utils.exif_utils import ExifUtils
from ..utils.sidecar_paths import get_preview_dir
from ..utils.sidecar_paths import get_preview_path
if not url or not url.strip():
return None
base_name = os.path.splitext(os.path.basename(model_path))[0]
preview_dir = get_preview_dir(model_path)
output_path = get_preview_path(model_path, ".webp")
# Centralized mirrors may not exist yet (unlike the model's own directory
# in alongside mode).
os.makedirs(preview_dir, exist_ok=True)
output_path = os.path.join(preview_dir, base_name + ".webp")
os.makedirs(os.path.dirname(output_path), exist_ok=True)
downloader = await get_downloader()
py/routes/workflow_routes.py
+3 -7
View File
@@ -165,7 +165,7 @@ def _write_workflow_preview(file_path: str, image_bytes: bytes) -> Optional[str]
try:
from ..utils.constants import CARD_PREVIEW_WIDTH
from ..utils.exif_utils import ExifUtils
from ..utils.sidecar_paths import get_preview_dir
from ..utils.sidecar_paths import get_preview_path
optimized, _extension = ExifUtils.optimize_image(
image_data=image_bytes,
@@ -177,12 +177,8 @@ def _write_workflow_preview(file_path: str, image_bytes: bytes) -> Optional[str]
if not optimized:
return None
base_name = os.path.splitext(os.path.basename(file_path))[0]
preview_dir = get_preview_dir(file_path)
os.makedirs(preview_dir, exist_ok=True)
preview_path = normalize_path(
os.path.join(preview_dir, f"{base_name}.preview.webp")
)
preview_path = normalize_path(get_preview_path(file_path, ".webp"))
os.makedirs(os.path.dirname(preview_path), exist_ok=True)
with open(preview_path, "wb") as handle:
handle.write(optimized)
return preview_path
py/services/download_manager.py
+5 -18
View File
@@ -48,7 +48,7 @@ from ..utils.preview_selection import resolve_mature_threshold, select_preview_m
from ..utils.utils import calculate_filename_for_model, decode_percent_encoded_filename, fit_stem_to_path_budget, sanitize_folder_name
from ..utils.exif_utils import ExifUtils
from ..utils.metadata_manager import MetadataManager
from ..utils.sidecar_paths import get_metadata_path, get_preview_dir
from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_preview_path
from .service_registry import ServiceRegistry
from .download_routing import (
is_diffusion_model_download,
@@ -2920,10 +2920,7 @@ class DownloadManager:
if media_type == "video":
preview_ext = _extension_from_url(preview_url, ".mp4")
preview_path = os.path.join(
get_preview_dir(save_path),
os.path.splitext(os.path.basename(save_path))[0] + preview_ext,
)
preview_path = get_preview_path(save_path, preview_ext)
rewritten_url, rewritten = rewrite_preview_url(
preview_url, media_type="video"
)
@@ -2966,10 +2963,7 @@ class DownloadManager:
content, headers, fallback_url=preview_url
)
if preview_ext in SUPPORTED_MEDIA_EXTENSIONS["images"]:
preview_path = os.path.join(
get_preview_dir(save_path),
os.path.splitext(os.path.basename(save_path))[0] + preview_ext,
)
preview_path = get_preview_path(save_path, preview_ext)
with open(preview_path, "wb") as preview_file:
preview_file.write(content)
preview_downloaded = True
@@ -2994,10 +2988,7 @@ class DownloadManager:
temp_file_handle.write(
content if isinstance(content, bytes) else content.encode("utf-8")
)
preview_path = os.path.join(
get_preview_dir(save_path),
os.path.splitext(os.path.basename(save_path))[0] + ".webp",
)
preview_path = get_preview_path(save_path, ".webp")
optimized_data, _ = ExifUtils.optimize_image(
image_data=temp_path,
@@ -3516,11 +3507,7 @@ class DownloadManager:
extension = os.path.splitext(preview_path)[1] or ".webp"
targets = [
os.path.join(
get_preview_dir(entry.file_path),
os.path.splitext(os.path.basename(entry.file_path))[0] + extension,
)
for entry in entries
get_preview_path(entry.file_path, extension) for entry in entries
]
if not targets:
py/services/metadata_sync_service.py
+2 -2
View File
@@ -15,7 +15,7 @@ from ..utils.file_utils import calculate_sha256, find_preview_file, normalize_pa
from ..utils.metadata_manager import MetadataManager
from ..utils.model_utils import determine_base_model
from ..utils.models import autov3_from_civitai_files
from ..utils.sidecar_paths import get_metadata_path, get_preview_dir
from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_sidecar_base_name
from .connectivity_guard import OFFLINE_FRIENDLY_MESSAGE, is_expected_offline_error
from .errors import MetadataPersistError, RateLimitError
from .model_metadata_provider import _LOCAL_PROVIDER_LABELS
@@ -738,7 +738,7 @@ class MetadataSyncService:
payload["model_name"] = base_name
if not payload.get("preview_url"):
payload["preview_url"] = normalize_path(
find_preview_file(base_name, get_preview_dir(file_path))
find_preview_file(get_sidecar_base_name(file_path), get_preview_dir(file_path))
)
if not payload.get("sha256"):
py/services/model_lifecycle_service.py
+43 -18
View File
@@ -13,8 +13,17 @@ from ..services.service_registry import ServiceRegistry
from ..services.pending_delete_service import get_pending_delete_service
from ..utils.constants import PREVIEW_EXTENSIONS
from ..utils.metadata_manager import MetadataManager
from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_sidecar_dir
from ..utils.workflow_extraction import WORKFLOW_DOCUMENT_SUFFIX
from ..utils.sidecar_paths import (
get_metadata_path,
get_preview_dir,
get_preview_path,
get_sidecar_base_name,
get_sidecar_dir,
)
from ..utils.workflow_extraction import (
WORKFLOW_DOCUMENT_SUFFIX,
get_workflow_document_path,
)
logger = logging.getLogger(__name__)
@@ -56,10 +65,9 @@ async def delete_model_artifacts(
model_path = os.path.join(target_dir, main_file)
sidecar_dir = get_sidecar_dir(model_path)
patterns = [os.path.basename(get_metadata_path(model_path))]
# Derived workflow document (workflow artifacts only; absent elsewhere).
patterns.append(f"{file_name}{WORKFLOW_DOCUMENT_SUFFIX}")
patterns.append(os.path.basename(get_workflow_document_path(model_path)))
for ext in PREVIEW_EXTENSIONS:
patterns.append(f"{file_name}{ext}")
patterns.append(os.path.basename(get_preview_path(model_path, ext)))
deleted: List[str] = []
main_path = model_path.replace(os.sep, "/")
@@ -476,19 +484,21 @@ class ModelLifecycleService:
(target_dir, f"{old_file_name}{old_extension}"),
(sidecar_dir, metadata_filename),
(sidecar_dir, f"{metadata_filename}.bak"),
# Derived workflow document (workflow artifacts only).
(sidecar_dir, f"{old_file_name}{WORKFLOW_DOCUMENT_SUFFIX}"),
(
sidecar_dir,
os.path.basename(get_workflow_document_path(file_path)),
),
]
for ext in PREVIEW_EXTENSIONS:
patterns.append((sidecar_dir, f"{old_file_name}{ext}"))
patterns.append(
(sidecar_dir, os.path.basename(get_preview_path(file_path, ext)))
)
existing_files: List[tuple[str, str]] = []
seen_paths: set[str] = set()
for pattern_dir, pattern in patterns:
path = os.path.join(pattern_dir, pattern)
# An image workflow artifact doubles as its own preview: the same
# path then appears under both the main-file and preview patterns
# and must only be renamed once.
# Companion patterns may overlap the main artifact; rename each path once.
dedupe_key = os.path.normcase(os.path.abspath(path))
if dedupe_key in seen_paths:
continue
@@ -508,7 +518,11 @@ class ModelLifecycleService:
new_preview: Optional[str] = None
renamed_files, new_metadata_path = await asyncio.to_thread(
self._rename_companion_files, existing_files, new_file_name
self._rename_companion_files,
existing_files,
new_file_name,
get_sidecar_base_name(file_path),
get_sidecar_base_name(new_file_path),
)
if metadata and new_metadata_path:
@@ -521,9 +535,13 @@ class ModelLifecycleService:
if metadata.get("preview_url"):
old_preview = str(metadata["preview_url"])
ext = self._get_multipart_ext(old_preview)
new_preview = os.path.join(
get_preview_dir(new_file_path), f"{new_file_name}{ext}"
).replace(os.sep, "/")
renamed_preview = get_preview_path(new_file_path, ext).replace(
os.sep, "/"
)
if os.path.exists(renamed_preview) or not os.path.exists(old_preview):
new_preview = renamed_preview
else:
new_preview = old_preview
metadata["preview_url"] = new_preview
await self._metadata_manager.save_metadata(new_file_path, metadata)
@@ -573,6 +591,8 @@ class ModelLifecycleService:
self,
existing_files: List[tuple[str, str]],
new_file_name: str,
old_sidecar_base: str,
new_sidecar_base: str,
) -> tuple[List[str], Optional[str]]:
"""Rename all companion files, off the event loop thread.
@@ -585,14 +605,19 @@ class ModelLifecycleService:
new_metadata_path: Optional[str] = None
for old_path, pattern in existing_files:
ext = self._get_multipart_ext(pattern)
if pattern.startswith(old_sidecar_base + "."):
ext = pattern[len(old_sidecar_base):]
new_pattern = new_sidecar_base + ext
else:
ext = self._get_multipart_ext(pattern)
new_pattern = new_file_name + ext
new_path = os.path.join(
os.path.dirname(old_path), f"{new_file_name}{ext}"
os.path.dirname(old_path), new_pattern
).replace(os.sep, "/")
os.rename(old_path, new_path)
renamed_files.append(new_path)
if ext == ".metadata.json":
if ext.endswith(".metadata.json"):
new_metadata_path = new_path
return renamed_files, new_metadata_path
py/services/model_scanner.py
+47 -29
View File
@@ -19,6 +19,8 @@ from ..utils.metadata_manager import MetadataManager
from ..utils.sidecar_paths import (
get_metadata_path,
get_preview_dir,
get_preview_path,
get_sidecar_base_name,
get_sidecar_dir,
is_centralized,
resolve_centralized_dir_for_dir,
@@ -840,6 +842,7 @@ class ModelScanner:
# override this so neither the scan-time pass nor the one-time backfill
# writes a meaningless state into their entries.
supports_quantization: bool = True
verify_hashes_on_scan: bool = False
_instances = {} # Dictionary to store instances by class
_locks = {} # Dictionary to store locks by class
@@ -2454,15 +2457,26 @@ class ModelScanner:
total_folder_repaired,
)
# Find missing files (in cache but not in filesystem). Only paths
# inside this scan's scope are candidates, and paths the walk could
# not read are kept and reported instead of being removed.
# Find missing files (in cache but not in filesystem). Scoped scans
# only remove inside their scope. A full-library refresh also drops
# rows left under roots that were removed from the configuration,
# while configured-but-unreadable roots remain protected below.
missing_files: Set[str] = set()
full_library_reconcile = scope is None or (
scope.roots is None and scope.folder is None
)
for path in cached_paths - found_paths:
if not any(
in_scope = any(
_path_matches_prefix(path, prefix)
for _root, prefix in scope_prefixes
):
)
if not in_scope:
protected_by_configured_root = any(
_path_matches_prefix(path, _scope_prefix(root, None))
for root in configured_only
)
if full_library_reconcile and not protected_by_configured_root:
missing_files.add(path)
continue
unreachable_prefix = unreachable.match(path)
if unreachable_prefix is not None:
@@ -3555,7 +3569,6 @@ class ModelScanner:
file_info = next((f for f in version_info.get('files', []) if f.get('primary')), None)
if file_info:
local_stem = os.path.splitext(os.path.basename(file_path))[0]
# from_civitai_info expects an API-shaped file entry and
# strips one extension itself, so hand it the real
# basename: passing the already extension-free stem made
@@ -3564,7 +3577,7 @@ class ModelScanner:
file_info['name'] = os.path.basename(file_path)
metadata = cast(Any, self.model_class).from_civitai_info(version_info, file_info, file_path)
metadata.preview_url = find_preview_file(local_stem, get_preview_dir(file_path))
metadata.preview_url = find_preview_file(get_sidecar_base_name(file_path), get_preview_dir(file_path))
await MetadataManager.save_metadata(file_path, metadata)
logger.info(f"Created metadata from .civitai.info for {file_path} (Reason: .civitai.info was found but .metadata.json was missing)")
except Exception as e:
@@ -3617,30 +3630,38 @@ class ModelScanner:
model_data = self._build_cache_entry(metadata, folder=normalized_folder)
# Compute SHA256 hash when metadata provided none (e.g., CivitAI API response has empty hashes).
# Respect hash_status='pending' (set by CheckpointScanner for large models) to defer
# hash calculation until on-demand — avoids reading entire checkpoint files at startup.
# Most libraries trust a stored hash to avoid reading large weight files.
# Workflow artifacts opt into verification on every full scan because
# their older shared sidecars could assign one artifact's hash to another.
# Respect hash_status='pending' for scanners that defer large-file hashing.
hash_status = model_data.get('hash_status', '')
stored_sha = (model_data.get('sha256') or '').strip()
# The empty-string placeholder is not an identity: models poisoned by a
# hash of zero bytes (issue #1162) are re-hashed once the file has
# content, so an existing library heals on its next full scan.
needs_sha = not stored_sha or is_empty_placeholder_hash(stored_sha)
needs_sha = (
self.verify_hashes_on_scan
or not stored_sha
or is_empty_placeholder_hash(stored_sha)
)
if needs_sha and hash_status != 'pending' and file_path:
try:
if stored_sha:
if stored_sha and is_empty_placeholder_hash(stored_sha):
logger.info(
"Recomputing SHA256 for %s (stored hash is the empty-file placeholder)",
file_path,
)
else:
elif not stored_sha:
logger.info(f"Computing SHA256 hash for {file_path} (was empty from metadata)")
sha256 = await calculate_sha256(file_path)
if sha256 and not is_empty_placeholder_hash(sha256):
model_data['sha256'] = sha256.lower()
normalized_sha = sha256.lower()
hash_changed = stored_sha.lower() != normalized_sha
model_data['sha256'] = normalized_sha
if isinstance(metadata, BaseModelMetadata):
metadata.sha256 = sha256.lower()
await MetadataManager.save_metadata(file_path, metadata)
metadata.sha256 = normalized_sha
if hash_changed:
await MetadataManager.save_metadata(file_path, metadata)
except Exception as e:
logger.error(f"Failed to compute SHA256 for {file_path}: {e}")
@@ -4035,7 +4056,6 @@ class ModelScanner:
)
target_file = os.path.join(target_path, final_filename).replace(os.sep, '/')
final_base_name = os.path.splitext(final_filename)[0]
# Log if filename was changed due to conflict
if final_filename != f"{base_name}{file_ext}":
@@ -4056,7 +4076,7 @@ class ModelScanner:
shutil.move(move_source, move_target)
# Move all associated files with the same base name
# Move this artifact's associated files.
source_metadata = None
moved_metadata_path = None
@@ -4069,17 +4089,19 @@ class ModelScanner:
if os.path.normpath(source_sidecar_dir) != os.path.normpath(source_dir):
associated_dirs.append((source_sidecar_dir, target_sidecar_dir))
# Find all files with the same base name in the source directories
# Match only this artifact's sidecars. Workflow artifacts include
# their original extension in the prefix so same-stem formats stay independent.
files_to_move = []
metadata_filename = os.path.basename(get_metadata_path(source_path))
source_sidecar_base = get_sidecar_base_name(source_path)
target_sidecar_base = get_sidecar_base_name(target_file)
for assoc_source_dir, assoc_target_dir in associated_dirs:
try:
for file in os.listdir(assoc_source_dir):
if file.startswith(base_name + ".") and file != os.path.basename(source_path):
if file.startswith(source_sidecar_base + ".") and file != os.path.basename(source_path):
source_file_path = os.path.join(assoc_source_dir, file)
# Generate new filename with the same base name as the model file
file_suffix = file[len(base_name):] # Get the part after base_name (e.g., ".metadata.json", ".preview.png")
new_associated_filename = f"{final_base_name}{file_suffix}"
file_suffix = file[len(source_sidecar_base):]
new_associated_filename = f"{target_sidecar_base}{file_suffix}"
target_associated_path = os.path.join(assoc_target_dir, new_associated_filename)
# Store metadata file path for special handling
@@ -4208,11 +4230,8 @@ class ModelScanner:
metadata['file_name'] = os.path.splitext(os.path.basename(model_path))[0]
if 'preview_url' in metadata and metadata['preview_url']:
preview_dir = get_preview_dir(model_path)
# Update preview filename to match the new base name
new_base_name = os.path.splitext(os.path.basename(model_path))[0]
preview_ext = get_preview_extension(metadata['preview_url'])
new_preview_path = os.path.join(preview_dir, f"{new_base_name}{preview_ext}")
new_preview_path = get_preview_path(model_path, preview_ext)
metadata['preview_url'] = new_preview_path.replace(os.sep, '/')
await MetadataManager.save_metadata(metadata_path, metadata)
@@ -4797,8 +4816,7 @@ class ModelScanner:
return None
dir_path = get_preview_dir(file_path)
base_name = os.path.splitext(os.path.basename(file_path))[0]
preview_path = find_preview_file(base_name, dir_path)
preview_path = find_preview_file(get_sidecar_base_name(file_path), dir_path)
if preview_path:
return config.get_preview_static_url(preview_path)
py/services/pending_delete_service.py
+5 -7
View File
@@ -38,8 +38,8 @@ from typing import (
)
from ..utils.constants import PREVIEW_EXTENSIONS
from ..utils.sidecar_paths import get_metadata_path, get_sidecar_dir
from ..utils.workflow_extraction import WORKFLOW_DOCUMENT_SUFFIX
from ..utils.sidecar_paths import get_metadata_path, get_preview_path, get_sidecar_dir
from ..utils.workflow_extraction import get_workflow_document_path
from ..utils import settings_paths
logger = logging.getLogger(__name__)
@@ -707,16 +707,14 @@ class PendingDeleteService:
# alongside mode, the centralized mirror otherwise).
sidecar_dir = get_sidecar_dir(model_path)
patterns = [os.path.basename(get_metadata_path(model_path))]
# Derived workflow document (workflow artifacts only; absent elsewhere).
patterns.append(f"{file_name}{WORKFLOW_DOCUMENT_SUFFIX}")
patterns.append(os.path.basename(get_workflow_document_path(model_path)))
for ext in PREVIEW_EXTENSIONS:
patterns.append(f"{file_name}{ext}")
patterns.append(os.path.basename(get_preview_path(model_path, ext)))
seen = {os.path.normcase(main_path)}
for pattern in patterns:
path = os.path.abspath(os.path.join(sidecar_dir, pattern))
# An image workflow artifact doubles as its own preview: it then
# matches both the main file and a preview pattern — stage it once.
# Companion patterns may overlap the main artifact; stage each path once.
if os.path.normcase(path) in seen:
continue
if os.path.exists(path):
py/services/preview_asset_service.py
+19 -8
View File
@@ -15,7 +15,11 @@ from ..utils.constants import (
from ..utils.civitai_utils import rewrite_preview_url
from ..utils.file_utils import detect_media_extension, reconcile_media_extension
from ..utils.preview_selection import resolve_mature_threshold, select_preview_media
from ..utils.sidecar_paths import get_metadata_path, get_preview_dir
from ..utils.sidecar_paths import (
get_metadata_path,
get_preview_dir,
get_preview_path,
)
from .settings_manager import get_settings_manager
logger = logging.getLogger(__name__)
@@ -67,8 +71,16 @@ class PreviewAssetService:
if not first_preview:
return
model_path_value = local_metadata.get("file_path")
model_path = str(model_path_value) if model_path_value else ""
base_name = os.path.splitext(os.path.splitext(os.path.basename(metadata_path))[0])[0]
preview_dir = os.path.dirname(metadata_path)
preview_dir = get_preview_dir(model_path) if model_path else os.path.dirname(metadata_path)
def build_preview_path(extension: str) -> str:
if model_path:
return get_preview_path(model_path, extension)
return os.path.join(preview_dir, base_name + extension)
# Centralized mirrors may not exist yet (unlike the model's own
# directory in alongside mode).
os.makedirs(preview_dir, exist_ok=True)
@@ -92,7 +104,7 @@ class PreviewAssetService:
if is_video:
extension = extension_from_url(preview_url, ".mp4")
preview_path = os.path.join(preview_dir, base_name + extension)
preview_path = build_preview_path(extension)
rewritten_url, rewritten = rewrite_preview_url(preview_url, media_type="video")
attempt_urls = []
@@ -126,7 +138,7 @@ class PreviewAssetService:
content, headers, fallback_url=preview_url
)
if extension in SUPPORTED_MEDIA_EXTENSIONS["images"]:
preview_path = os.path.join(preview_dir, base_name + extension)
preview_path = build_preview_path(extension)
try:
with open(preview_path, "wb") as handle:
handle.write(content)
@@ -138,7 +150,7 @@ class PreviewAssetService:
return
extension = ".webp"
preview_path = os.path.join(preview_dir, base_name + extension)
preview_path = build_preview_path(extension)
success, content, _headers = await downloader.download_to_memory(
preview_url, use_auth=False
)
@@ -180,7 +192,6 @@ class PreviewAssetService:
) -> Dict[str, object]:
"""Replace an existing preview asset for a model."""
base_name = os.path.splitext(os.path.basename(model_path))[0]
folder = get_preview_dir(model_path)
# Centralized mirrors may not exist yet (unlike the model's own
# directory in alongside mode).
@@ -191,7 +202,7 @@ class PreviewAssetService:
)
for ext in PREVIEW_EXTENSIONS:
existing_preview = os.path.join(folder, base_name + ext)
existing_preview = get_preview_path(model_path, ext)
if os.path.exists(existing_preview):
try:
os.remove(existing_preview)
@@ -200,7 +211,7 @@ class PreviewAssetService:
"Failed to delete existing preview %s: %s", existing_preview, exc
)
preview_path = os.path.join(folder, base_name + extension).replace(os.sep, "/")
preview_path = get_preview_path(model_path, extension).replace(os.sep, "/")
with open(preview_path, "wb") as handle:
handle.write(optimized_data)
py/services/use_cases/sidecar_migration_use_case.py
+2 -1
View File
@@ -55,6 +55,7 @@ from ...utils.sidecar_paths import (
ROOT_MAP_FILENAME,
STORAGE_MODE_CENTRALIZED,
get_configured_sidecar_root,
get_sidecar_base_name,
get_sidecar_root,
get_storage_mode,
relocate_root_map,
@@ -573,7 +574,7 @@ class SidecarMigrationUseCase:
result["skipped"] = 1
return result
stem = os.path.splitext(os.path.basename(model_path))[0]
stem = get_sidecar_base_name(model_path)
sidecar_name = stem + METADATA_SUFFIX
moved_previews: List[str] = []
py/services/workflow_scanner.py
+81 -2
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import json
import logging
import os
from datetime import datetime
@@ -16,7 +17,12 @@ from ..utils.file_utils import (
)
from ..utils.metadata_manager import MetadataManager
from ..utils.models import WorkflowMetadata
from ..utils.sidecar_paths import get_preview_dir
from ..utils.sidecar_paths import (
get_legacy_metadata_path,
get_metadata_path,
get_preview_dir,
get_sidecar_base_name,
)
from ..utils.workflow_analysis import analyze_workflow_file
from ..utils.workflow_extraction import (
IMAGE_CONTAINER_EXTENSIONS,
@@ -57,6 +63,7 @@ class WorkflowScanner(ModelScanner):
# Workflows are graph documents, not weights: quantization has no meaning
# for them, so the inherited scan-time pass and backfill are skipped.
supports_quantization = False
verify_hashes_on_scan = True
def __init__(self):
super().__init__(
@@ -92,6 +99,74 @@ class WorkflowScanner(ModelScanner):
def _is_image_artifact(file_path: str) -> bool:
return os.path.splitext(file_path)[1].lower() in IMAGE_CONTAINER_EXTENSIONS
@staticmethod
def _path_key(file_path: str) -> str:
return os.path.normcase(os.path.abspath(os.path.normpath(file_path)))
def _sidecar_references_preview(
self, metadata_path: str, preview_path: str, owner_path: str
) -> bool:
try:
with open(metadata_path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
except (OSError, json.JSONDecodeError):
return False
referenced = payload.get("preview_url") if isinstance(payload, dict) else None
return bool(
referenced
and self._path_key(str(referenced)) == self._path_key(preview_path)
and self._path_key(owner_path) != self._path_key(preview_path)
)
def _is_known_preview(self, file_path: str) -> bool:
if not self._is_image_artifact(file_path):
return False
cache = getattr(self, "_cache", None)
for item in getattr(cache, "raw_data", None) or []:
owner_path = item.get("file_path")
preview_url = item.get("preview_url")
if (
owner_path
and preview_url
and self._path_key(owner_path) != self._path_key(file_path)
and self._path_key(preview_url) == self._path_key(file_path)
):
return True
stem = os.path.splitext(os.path.basename(file_path))[0]
artifact_dir = os.path.dirname(file_path)
for extension in WORKFLOW_ARTIFACT_EXTENSIONS:
owner_path = os.path.join(artifact_dir, stem + extension)
metadata_paths = {
get_metadata_path(owner_path),
get_legacy_metadata_path(owner_path),
}
for metadata_path in metadata_paths:
if os.path.isfile(metadata_path) and self._sidecar_references_preview(
metadata_path, file_path, owner_path
):
return True
return False
async def _process_model_file(
self,
file_path: str,
root_path: str,
*,
hash_index: Optional[ModelHashIndex] = None,
excluded_models: Optional[List[str]] = None,
):
if self._is_known_preview(file_path):
logger.debug("Skipping workflow preview artifact: %s", file_path)
return None
return await super()._process_model_file(
file_path,
root_path,
hash_index=hash_index,
excluded_models=excluded_models,
)
def adjust_metadata(self, metadata, file_path, root_path):
"""Fill workflow analysis fields when they are missing.
@@ -156,7 +231,11 @@ class WorkflowScanner(ModelScanner):
return None
base_name = os.path.splitext(os.path.basename(file_path))[0]
preview_url = find_preview_file(base_name, get_preview_dir(file_path))
preview_url = find_preview_file(
get_sidecar_base_name(file_path), get_preview_dir(file_path)
)
if not preview_url:
preview_url = find_preview_file(base_name, get_preview_dir(file_path))
if not preview_url and self._is_image_artifact(file_path):
# An image artifact doubles as its own preview.
preview_url = file_path
py/utils/constants.py
+2
View File
@@ -28,6 +28,8 @@ PREVIEW_EXTENSIONS = [
".preview.jpeg",
".preview.jpg",
".preview.mp4",
".preview.gif",
".preview.webm",
".png",
".jpeg",
".jpg",
py/utils/metadata_manager.py
+14 -5
View File
@@ -9,7 +9,13 @@ from .models import BaseModelMetadata, CheckpointMetadata, EmbeddingMetadata, Lo
from .file_utils import normalize_path, find_preview_file, calculate_sha256, calculate_autov3
from .gguf_utils import detect_quantization
from .lora_metadata import extract_lora_metadata, extract_checkpoint_metadata
from .sidecar_paths import get_metadata_path, get_preview_dir, resolve_metadata_path
from .sidecar_paths import (
get_legacy_metadata_path,
get_metadata_path,
get_preview_dir,
get_sidecar_base_name,
resolve_metadata_path,
)
logger = logging.getLogger(__name__)
@@ -35,10 +41,13 @@ class MetadataManager:
- should_skip: True if corrupted metadata file exists and model should be skipped
"""
metadata_path = get_metadata_path(file_path)
# Check if metadata file exists
if not os.path.exists(metadata_path):
return None, False
legacy_path = get_legacy_metadata_path(file_path)
if legacy_path != metadata_path and os.path.exists(legacy_path):
metadata_path = legacy_path
else:
return None, False
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
@@ -404,7 +413,7 @@ class MetadataManager:
# Update preview if it doesn't exist OR if model and preview are in different directories
if not os.path.exists(preview_url) or file_dir != preview_dir:
base_name = os.path.splitext(os.path.basename(file_path))[0]
base_name = get_sidecar_base_name(file_path)
new_preview_url = find_preview_file(base_name, file_dir)
if new_preview_url:
metadata.preview_url = normalize_path(new_preview_url)
py/utils/sidecar_paths.py
+59 -1
View File
@@ -1029,10 +1029,68 @@ def get_sidecar_dir(model_path: str) -> str:
return os.path.dirname(os.path.abspath(model_path))
def is_workflow_artifact_path(model_path: str) -> bool:
"""Return whether *model_path* belongs to the configured workflow library."""
if os.path.splitext(model_path)[1].lower() not in {
".json",
".zip",
".png",
".webp",
}:
return False
try:
from ..config import config
roots = config.workflows_roots or []
except Exception:
return False
normalized = os.path.normcase(os.path.abspath(os.path.normpath(model_path)))
for root in roots:
if not root:
continue
normalized_root = os.path.normcase(os.path.abspath(os.path.normpath(root)))
if normalized == normalized_root or normalized.startswith(
normalized_root + os.sep
):
return True
return False
def get_sidecar_base_name(model_path: str) -> str:
"""Return the filename prefix that uniquely identifies an artifact's assets."""
if is_workflow_artifact_path(model_path):
return os.path.basename(model_path)
return os.path.splitext(os.path.basename(model_path))[0]
def get_legacy_metadata_path(model_path: str) -> str:
"""Return the pre-extension-qualified metadata path used by older versions."""
base_name = os.path.splitext(os.path.basename(model_path))[0] + METADATA_SUFFIX
return os.path.join(get_sidecar_dir(model_path), base_name)
def get_metadata_path(model_path: str) -> str:
"""Return the ``.metadata.json`` sidecar path for a model file."""
base_name = os.path.splitext(os.path.basename(model_path))[0] + METADATA_SUFFIX
base_name = get_sidecar_base_name(model_path) + METADATA_SUFFIX
return os.path.join(get_sidecar_dir(model_path), base_name)
def get_preview_path(model_path: str, extension: str) -> str:
"""Return a preview path that cannot collide with a workflow artifact."""
if not extension.startswith("."):
extension = "." + extension
if is_workflow_artifact_path(model_path):
if not extension.startswith(".preview."):
extension = ".preview" + extension
base_name = os.path.basename(model_path) + extension
else:
base_name = os.path.splitext(os.path.basename(model_path))[0] + extension
return os.path.join(get_sidecar_dir(model_path), base_name)
py/utils/workflow_extraction.py
+8 -6
View File
@@ -7,7 +7,7 @@ extracted workflow document instead:
- ``.json`` artifacts ARE the document; they are validated and read in place.
- ``.zip`` / ``.png`` / ``.webp`` containers get the document materialized
once into an LM-owned derived sidecar ``<name>.workflow.json`` inside the
once into an LM-owned derived ``.workflow.json`` sidecar inside the
artifact's sidecar directory. Scanners exclude the suffix from the library
walk; lifecycle operations (move/rename/delete) carry it like any other
sidecar.
@@ -21,7 +21,7 @@ import os
import zipfile
from typing import Any, Dict, List, Optional
from .sidecar_paths import get_sidecar_dir
from .sidecar_paths import get_sidecar_base_name, get_sidecar_dir
from .workflow_analysis import (
MAX_WORKFLOW_JSON_BYTES,
load_workflow_file,
@@ -46,14 +46,16 @@ _MAX_ZIP_JSON_CANDIDATES = 64
def get_workflow_document_path(artifact_path: str) -> str:
"""Return the path of the extracted workflow document for an artifact.
``.json`` artifacts are their own document; containers map to the
derived ``<name>.workflow.json`` in the artifact's sidecar directory
(alongside the artifact or in the centralized mirror).
``.json`` artifacts are their own document; containers map to a derived
``.workflow.json`` sidecar in the artifact's sidecar directory (alongside
the artifact or in the centralized mirror). Workflow-library containers
include their original extension in the sidecar name to keep same-stem
artifacts independent.
"""
if os.path.splitext(artifact_path)[1].lower() == ".json":
return artifact_path
base_name = os.path.splitext(os.path.basename(artifact_path))[0]
base_name = get_sidecar_base_name(artifact_path)
return os.path.join(
get_sidecar_dir(artifact_path), base_name + WORKFLOW_DOCUMENT_SUFFIX
)
tests/routes/test_workflow_routes.py
+1 -1
View File
@@ -1464,7 +1464,7 @@ async def test_create_workflow_attaches_preview_for_matching_prompt(
assert response.status == 200
assert payload["preview"] is True
assert (root / "With Preview.preview.webp").exists()
assert (root / "With Preview.json.preview.webp").exists()
async def test_create_workflow_skips_preview_for_other_prompt(
tests/services/test_model_lifecycle_service.py
+30
View File
@@ -1199,3 +1199,33 @@ async def test_rename_image_workflow_artifact_renames_main_file_once(tmp_path: P
assert not model_path.exists()
renamed = result["renamed_files"]
assert len(renamed) == len(set(renamed))
@pytest.mark.asyncio
async def test_delete_workflow_artifact_preserves_same_stem_sibling(
tmp_path: Path, monkeypatch
):
from py import config as config_module
from py.services.model_lifecycle_service import delete_model_artifacts
monkeypatch.setattr(config_module.config, "workflows_roots", [tmp_path.as_posix()])
zip_path = tmp_path / "same.zip"
webp_path = tmp_path / "same.webp"
zip_path.write_bytes(b"zip")
webp_path.write_bytes(b"webp")
for base in ("same.zip", "same.webp"):
(tmp_path / f"{base}.metadata.json").write_text("{}", encoding="utf-8")
(tmp_path / f"{base}.workflow.json").write_text("{}", encoding="utf-8")
(tmp_path / f"{base}.preview.webp").write_bytes(b"preview")
await delete_model_artifacts(str(tmp_path), "same", main_extension=".zip")
assert not zip_path.exists()
assert not (tmp_path / "same.zip.metadata.json").exists()
assert not (tmp_path / "same.zip.workflow.json").exists()
assert not (tmp_path / "same.zip.preview.webp").exists()
assert webp_path.read_bytes() == b"webp"
assert (tmp_path / "same.webp.metadata.json").is_file()
assert (tmp_path / "same.webp.workflow.json").is_file()
assert (tmp_path / "same.webp.preview.webp").is_file()
tests/services/test_model_scanner.py
+22
View File
@@ -864,6 +864,28 @@ async def test_reconcile_cache_adds_new_files_and_updates_hash_index(tmp_path: P
assert cache.folders == [""]
@pytest.mark.asyncio
async def test_reconcile_cache_removes_entries_under_removed_roots(tmp_path: Path):
root = tmp_path / "current"
root.mkdir()
(root / "one.txt").write_text("one", encoding="utf-8")
scanner = DummyScanner(root)
await scanner._initialize_cache()
stale_path = _normalize_path(tmp_path / "removed" / "old.txt")
stale_entry = dict(scanner._cache.raw_data[0])
stale_entry["file_path"] = stale_path
stale_entry["sha256"] = "hash-stale"
scanner._cache.raw_data.append(stale_entry)
scanner._hash_index.add_entry("hash-stale", stale_path)
summary = await scanner._reconcile_cache()
assert stale_path not in {item["file_path"] for item in scanner._cache.raw_data}
assert scanner._hash_index.get_path("hash-stale") is None
assert summary["removed"] == 1
@pytest.mark.asyncio
async def test_reconcile_cache_applies_adjust_cached_entry(tmp_path: Path):
existing = tmp_path / "one.txt"
tests/services/test_workflow_auto_organize.py
+6 -6
View File
@@ -87,12 +87,12 @@ async def test_auto_organize_moves_workflow_with_its_sidecars(workflow_library:
json.dumps({"1": {"class_type": "CheckpointLoaderSimple", "inputs": {}}}),
encoding="utf-8",
)
(old_dir / "portrait.workflow.json").write_text(
(old_dir / "portrait.json.workflow.json").write_text(
json.dumps({"1": {"class_type": "CheckpointLoaderSimple"}}),
encoding="utf-8",
)
(old_dir / "portrait.preview.webp").write_bytes(b"preview")
(old_dir / "portrait.metadata.json").write_text(
(old_dir / "portrait.json.preview.webp").write_bytes(b"preview")
(old_dir / "portrait.json.metadata.json").write_text(
json.dumps(
{
"file_path": _normalize(workflow_path),
@@ -121,11 +121,11 @@ async def test_auto_organize_moves_workflow_with_its_sidecars(workflow_library:
assert result.operation_type == "bulk"
assert result.success_count == 1
assert (target_dir / "portrait.json").is_file()
assert (target_dir / "portrait.workflow.json").is_file()
assert (target_dir / "portrait.preview.webp").read_bytes() == b"preview"
assert (target_dir / "portrait.json.workflow.json").is_file()
assert (target_dir / "portrait.json.preview.webp").read_bytes() == b"preview"
# The metadata sidecar follows the artifact and records the new path.
moved_metadata = json.loads(
(target_dir / "portrait.metadata.json").read_text(encoding="utf-8")
(target_dir / "portrait.json.metadata.json").read_text(encoding="utf-8")
)
assert moved_metadata["file_path"] == _normalize(target_dir / "portrait.json")
tests/services/test_workflow_scanner.py
+76 -2
View File
@@ -8,6 +8,7 @@ import pytest
from py import config as config_module
from py.services.model_scanner import ModelScanner
from py.services.workflow_scanner import WorkflowScanner
from py.utils.metadata_manager import MetadataManager
from py.utils.models import WorkflowMetadata
@@ -162,7 +163,7 @@ async def test_create_default_metadata_from_zip_artifact(workflow_root, enable_w
assert metadata.sha256
# The derived workflow document is materialized as a sidecar.
derived = workflow_root / "pack.workflow.json"
derived = workflow_root / "pack.zip.workflow.json"
assert derived.exists()
assert json.loads(derived.read_text(encoding="utf-8")) == workflow
@@ -239,7 +240,7 @@ def test_adjust_metadata_fills_missing_analysis(workflow_root, enable_workflows)
assert adjusted.workflow_format == "api"
assert adjusted.node_count == 1
assert adjusted.model_refs == ["base.safetensors"]
assert (workflow_root / "downloaded.workflow.json").exists()
assert (workflow_root / "downloaded.zip.workflow.json").exists()
def test_adjust_metadata_keeps_populated_analysis(workflow_root, enable_workflows):
@@ -449,3 +450,76 @@ async def test_register_local_artifact_allows_remote_resolution(
assert metadata is not None
assert metadata.base_model == "SD 1.5"
assert captured["include_remote"] is True
async def test_same_stem_artifacts_recover_independent_hashes_from_legacy_sidecar(
workflow_root, enable_workflows
):
import hashlib
import zipfile
from py.utils.sidecar_paths import get_metadata_path
workflow = {"1": {"class_type": "KSampler", "inputs": {}}}
json_path = workflow_root / "same.json"
zip_path = workflow_root / "same.zip"
json_path.write_text(json.dumps(workflow), encoding="utf-8")
with zipfile.ZipFile(zip_path, "w") as archive:
archive.writestr("workflow.json", json.dumps(workflow))
(workflow_root / "same.metadata.json").write_text(
json.dumps(
{
"file_name": "same",
"model_name": "same",
"file_path": _normalize(json_path),
"size": 1,
"modified": 0,
"sha256": "polluted-shared-hash",
"base_model": "Unknown",
"preview_url": "",
"workflow_format": "",
}
),
encoding="utf-8",
)
from py.services.model_hash_index import ModelHashIndex
scanner = _make_scanner()
scanner._hash_index = ModelHashIndex()
scanner._excluded_models = []
json_entry = await scanner._process_model_file(str(json_path), str(workflow_root))
zip_entry = await scanner._process_model_file(str(zip_path), str(workflow_root))
assert json_entry["sha256"] == hashlib.sha256(json_path.read_bytes()).hexdigest()
assert zip_entry["sha256"] == hashlib.sha256(zip_path.read_bytes()).hexdigest()
assert json_entry["sha256"] != zip_entry["sha256"]
assert (workflow_root / "same.json.metadata.json").is_file()
assert (workflow_root / "same.zip.metadata.json").is_file()
assert get_metadata_path(str(json_path)) != get_metadata_path(str(zip_path))
assert (workflow_root / "same.zip.workflow.json").is_file()
async def test_scan_skips_legacy_preview_referenced_by_sibling_metadata(
workflow_root, enable_workflows
):
import zipfile
PIL = pytest.importorskip("PIL.Image")
from py.utils.exif_utils import ExifUtils
workflow = {"nodes": [{"id": 1, "type": "KSampler"}], "links": []}
zip_path = workflow_root / "same.zip"
preview_path = workflow_root / "same.webp"
with zipfile.ZipFile(zip_path, "w") as archive:
archive.writestr("workflow.json", json.dumps(workflow))
PIL.new("RGB", (8, 8)).save(preview_path, format="WEBP")
ExifUtils.embed_workflow(str(preview_path), json.dumps(workflow))
scanner = _make_scanner()
metadata = await scanner._create_default_metadata(str(zip_path))
metadata.preview_url = _normalize(preview_path)
await MetadataManager.save_metadata(str(zip_path), metadata)
assert await scanner._process_model_file(str(preview_path), str(workflow_root)) is None