diff --git a/py/metadata_ops/__init__.py b/py/metadata_ops/__init__.py index 15b0f1d9..0c03f0b6 100644 --- a/py/metadata_ops/__init__.py +++ b/py/metadata_ops/__init__.py @@ -172,17 +172,15 @@ async def download_preview( """ from ..services.downloader import get_downloader from ..utils.exif_utils import ExifUtils - from ..utils.sidecar_paths import get_preview_dir + from ..utils.sidecar_paths import get_preview_path if not url or not url.strip(): return None - base_name = os.path.splitext(os.path.basename(model_path))[0] - preview_dir = get_preview_dir(model_path) + output_path = get_preview_path(model_path, ".webp") # Centralized mirrors may not exist yet (unlike the model's own directory # in alongside mode). - os.makedirs(preview_dir, exist_ok=True) - output_path = os.path.join(preview_dir, base_name + ".webp") + os.makedirs(os.path.dirname(output_path), exist_ok=True) downloader = await get_downloader() diff --git a/py/routes/workflow_routes.py b/py/routes/workflow_routes.py index 0f1d4acf..dbe1e0fe 100644 --- a/py/routes/workflow_routes.py +++ b/py/routes/workflow_routes.py @@ -165,7 +165,7 @@ def _write_workflow_preview(file_path: str, image_bytes: bytes) -> Optional[str] try: from ..utils.constants import CARD_PREVIEW_WIDTH from ..utils.exif_utils import ExifUtils - from ..utils.sidecar_paths import get_preview_dir + from ..utils.sidecar_paths import get_preview_path optimized, _extension = ExifUtils.optimize_image( image_data=image_bytes, @@ -177,12 +177,8 @@ def _write_workflow_preview(file_path: str, image_bytes: bytes) -> Optional[str] if not optimized: return None - base_name = os.path.splitext(os.path.basename(file_path))[0] - preview_dir = get_preview_dir(file_path) - os.makedirs(preview_dir, exist_ok=True) - preview_path = normalize_path( - os.path.join(preview_dir, f"{base_name}.preview.webp") - ) + preview_path = normalize_path(get_preview_path(file_path, ".webp")) + os.makedirs(os.path.dirname(preview_path), exist_ok=True) with open(preview_path, "wb") as handle: handle.write(optimized) return preview_path diff --git a/py/services/download_manager.py b/py/services/download_manager.py index c2d595bc..deb1b1b2 100644 --- a/py/services/download_manager.py +++ b/py/services/download_manager.py @@ -48,7 +48,7 @@ from ..utils.preview_selection import resolve_mature_threshold, select_preview_m from ..utils.utils import calculate_filename_for_model, decode_percent_encoded_filename, fit_stem_to_path_budget, sanitize_folder_name from ..utils.exif_utils import ExifUtils from ..utils.metadata_manager import MetadataManager -from ..utils.sidecar_paths import get_metadata_path, get_preview_dir +from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_preview_path from .service_registry import ServiceRegistry from .download_routing import ( is_diffusion_model_download, @@ -2920,10 +2920,7 @@ class DownloadManager: if media_type == "video": preview_ext = _extension_from_url(preview_url, ".mp4") - preview_path = os.path.join( - get_preview_dir(save_path), - os.path.splitext(os.path.basename(save_path))[0] + preview_ext, - ) + preview_path = get_preview_path(save_path, preview_ext) rewritten_url, rewritten = rewrite_preview_url( preview_url, media_type="video" ) @@ -2966,10 +2963,7 @@ class DownloadManager: content, headers, fallback_url=preview_url ) if preview_ext in SUPPORTED_MEDIA_EXTENSIONS["images"]: - preview_path = os.path.join( - get_preview_dir(save_path), - os.path.splitext(os.path.basename(save_path))[0] + preview_ext, - ) + preview_path = get_preview_path(save_path, preview_ext) with open(preview_path, "wb") as preview_file: preview_file.write(content) preview_downloaded = True @@ -2994,10 +2988,7 @@ class DownloadManager: temp_file_handle.write( content if isinstance(content, bytes) else content.encode("utf-8") ) - preview_path = os.path.join( - get_preview_dir(save_path), - os.path.splitext(os.path.basename(save_path))[0] + ".webp", - ) + preview_path = get_preview_path(save_path, ".webp") optimized_data, _ = ExifUtils.optimize_image( image_data=temp_path, @@ -3516,11 +3507,7 @@ class DownloadManager: extension = os.path.splitext(preview_path)[1] or ".webp" targets = [ - os.path.join( - get_preview_dir(entry.file_path), - os.path.splitext(os.path.basename(entry.file_path))[0] + extension, - ) - for entry in entries + get_preview_path(entry.file_path, extension) for entry in entries ] if not targets: diff --git a/py/services/metadata_sync_service.py b/py/services/metadata_sync_service.py index ec2d571d..3fcbd231 100644 --- a/py/services/metadata_sync_service.py +++ b/py/services/metadata_sync_service.py @@ -15,7 +15,7 @@ from ..utils.file_utils import calculate_sha256, find_preview_file, normalize_pa from ..utils.metadata_manager import MetadataManager from ..utils.model_utils import determine_base_model from ..utils.models import autov3_from_civitai_files -from ..utils.sidecar_paths import get_metadata_path, get_preview_dir +from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_sidecar_base_name from .connectivity_guard import OFFLINE_FRIENDLY_MESSAGE, is_expected_offline_error from .errors import MetadataPersistError, RateLimitError from .model_metadata_provider import _LOCAL_PROVIDER_LABELS @@ -738,7 +738,7 @@ class MetadataSyncService: payload["model_name"] = base_name if not payload.get("preview_url"): payload["preview_url"] = normalize_path( - find_preview_file(base_name, get_preview_dir(file_path)) + find_preview_file(get_sidecar_base_name(file_path), get_preview_dir(file_path)) ) if not payload.get("sha256"): diff --git a/py/services/model_lifecycle_service.py b/py/services/model_lifecycle_service.py index e170fb24..14b7d02a 100644 --- a/py/services/model_lifecycle_service.py +++ b/py/services/model_lifecycle_service.py @@ -13,8 +13,17 @@ from ..services.service_registry import ServiceRegistry from ..services.pending_delete_service import get_pending_delete_service from ..utils.constants import PREVIEW_EXTENSIONS from ..utils.metadata_manager import MetadataManager -from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_sidecar_dir -from ..utils.workflow_extraction import WORKFLOW_DOCUMENT_SUFFIX +from ..utils.sidecar_paths import ( + get_metadata_path, + get_preview_dir, + get_preview_path, + get_sidecar_base_name, + get_sidecar_dir, +) +from ..utils.workflow_extraction import ( + WORKFLOW_DOCUMENT_SUFFIX, + get_workflow_document_path, +) logger = logging.getLogger(__name__) @@ -56,10 +65,9 @@ async def delete_model_artifacts( model_path = os.path.join(target_dir, main_file) sidecar_dir = get_sidecar_dir(model_path) patterns = [os.path.basename(get_metadata_path(model_path))] - # Derived workflow document (workflow artifacts only; absent elsewhere). - patterns.append(f"{file_name}{WORKFLOW_DOCUMENT_SUFFIX}") + patterns.append(os.path.basename(get_workflow_document_path(model_path))) for ext in PREVIEW_EXTENSIONS: - patterns.append(f"{file_name}{ext}") + patterns.append(os.path.basename(get_preview_path(model_path, ext))) deleted: List[str] = [] main_path = model_path.replace(os.sep, "/") @@ -476,19 +484,21 @@ class ModelLifecycleService: (target_dir, f"{old_file_name}{old_extension}"), (sidecar_dir, metadata_filename), (sidecar_dir, f"{metadata_filename}.bak"), - # Derived workflow document (workflow artifacts only). - (sidecar_dir, f"{old_file_name}{WORKFLOW_DOCUMENT_SUFFIX}"), + ( + sidecar_dir, + os.path.basename(get_workflow_document_path(file_path)), + ), ] for ext in PREVIEW_EXTENSIONS: - patterns.append((sidecar_dir, f"{old_file_name}{ext}")) + patterns.append( + (sidecar_dir, os.path.basename(get_preview_path(file_path, ext))) + ) existing_files: List[tuple[str, str]] = [] seen_paths: set[str] = set() for pattern_dir, pattern in patterns: path = os.path.join(pattern_dir, pattern) - # An image workflow artifact doubles as its own preview: the same - # path then appears under both the main-file and preview patterns - # and must only be renamed once. + # Companion patterns may overlap the main artifact; rename each path once. dedupe_key = os.path.normcase(os.path.abspath(path)) if dedupe_key in seen_paths: continue @@ -508,7 +518,11 @@ class ModelLifecycleService: new_preview: Optional[str] = None renamed_files, new_metadata_path = await asyncio.to_thread( - self._rename_companion_files, existing_files, new_file_name + self._rename_companion_files, + existing_files, + new_file_name, + get_sidecar_base_name(file_path), + get_sidecar_base_name(new_file_path), ) if metadata and new_metadata_path: @@ -521,9 +535,13 @@ class ModelLifecycleService: if metadata.get("preview_url"): old_preview = str(metadata["preview_url"]) ext = self._get_multipart_ext(old_preview) - new_preview = os.path.join( - get_preview_dir(new_file_path), f"{new_file_name}{ext}" - ).replace(os.sep, "/") + renamed_preview = get_preview_path(new_file_path, ext).replace( + os.sep, "/" + ) + if os.path.exists(renamed_preview) or not os.path.exists(old_preview): + new_preview = renamed_preview + else: + new_preview = old_preview metadata["preview_url"] = new_preview await self._metadata_manager.save_metadata(new_file_path, metadata) @@ -573,6 +591,8 @@ class ModelLifecycleService: self, existing_files: List[tuple[str, str]], new_file_name: str, + old_sidecar_base: str, + new_sidecar_base: str, ) -> tuple[List[str], Optional[str]]: """Rename all companion files, off the event loop thread. @@ -585,14 +605,19 @@ class ModelLifecycleService: new_metadata_path: Optional[str] = None for old_path, pattern in existing_files: - ext = self._get_multipart_ext(pattern) + if pattern.startswith(old_sidecar_base + "."): + ext = pattern[len(old_sidecar_base):] + new_pattern = new_sidecar_base + ext + else: + ext = self._get_multipart_ext(pattern) + new_pattern = new_file_name + ext new_path = os.path.join( - os.path.dirname(old_path), f"{new_file_name}{ext}" + os.path.dirname(old_path), new_pattern ).replace(os.sep, "/") os.rename(old_path, new_path) renamed_files.append(new_path) - if ext == ".metadata.json": + if ext.endswith(".metadata.json"): new_metadata_path = new_path return renamed_files, new_metadata_path diff --git a/py/services/model_scanner.py b/py/services/model_scanner.py index 37d6ed88..9db8014c 100644 --- a/py/services/model_scanner.py +++ b/py/services/model_scanner.py @@ -19,6 +19,8 @@ from ..utils.metadata_manager import MetadataManager from ..utils.sidecar_paths import ( get_metadata_path, get_preview_dir, + get_preview_path, + get_sidecar_base_name, get_sidecar_dir, is_centralized, resolve_centralized_dir_for_dir, @@ -840,6 +842,7 @@ class ModelScanner: # override this so neither the scan-time pass nor the one-time backfill # writes a meaningless state into their entries. supports_quantization: bool = True + verify_hashes_on_scan: bool = False _instances = {} # Dictionary to store instances by class _locks = {} # Dictionary to store locks by class @@ -2454,15 +2457,26 @@ class ModelScanner: total_folder_repaired, ) - # Find missing files (in cache but not in filesystem). Only paths - # inside this scan's scope are candidates, and paths the walk could - # not read are kept and reported instead of being removed. + # Find missing files (in cache but not in filesystem). Scoped scans + # only remove inside their scope. A full-library refresh also drops + # rows left under roots that were removed from the configuration, + # while configured-but-unreadable roots remain protected below. missing_files: Set[str] = set() + full_library_reconcile = scope is None or ( + scope.roots is None and scope.folder is None + ) for path in cached_paths - found_paths: - if not any( + in_scope = any( _path_matches_prefix(path, prefix) for _root, prefix in scope_prefixes - ): + ) + if not in_scope: + protected_by_configured_root = any( + _path_matches_prefix(path, _scope_prefix(root, None)) + for root in configured_only + ) + if full_library_reconcile and not protected_by_configured_root: + missing_files.add(path) continue unreachable_prefix = unreachable.match(path) if unreachable_prefix is not None: @@ -3555,7 +3569,6 @@ class ModelScanner: file_info = next((f for f in version_info.get('files', []) if f.get('primary')), None) if file_info: - local_stem = os.path.splitext(os.path.basename(file_path))[0] # from_civitai_info expects an API-shaped file entry and # strips one extension itself, so hand it the real # basename: passing the already extension-free stem made @@ -3564,7 +3577,7 @@ class ModelScanner: file_info['name'] = os.path.basename(file_path) metadata = cast(Any, self.model_class).from_civitai_info(version_info, file_info, file_path) - metadata.preview_url = find_preview_file(local_stem, get_preview_dir(file_path)) + metadata.preview_url = find_preview_file(get_sidecar_base_name(file_path), get_preview_dir(file_path)) await MetadataManager.save_metadata(file_path, metadata) logger.info(f"Created metadata from .civitai.info for {file_path} (Reason: .civitai.info was found but .metadata.json was missing)") except Exception as e: @@ -3617,30 +3630,38 @@ class ModelScanner: model_data = self._build_cache_entry(metadata, folder=normalized_folder) - # Compute SHA256 hash when metadata provided none (e.g., CivitAI API response has empty hashes). - # Respect hash_status='pending' (set by CheckpointScanner for large models) to defer - # hash calculation until on-demand — avoids reading entire checkpoint files at startup. + # Most libraries trust a stored hash to avoid reading large weight files. + # Workflow artifacts opt into verification on every full scan because + # their older shared sidecars could assign one artifact's hash to another. + # Respect hash_status='pending' for scanners that defer large-file hashing. hash_status = model_data.get('hash_status', '') stored_sha = (model_data.get('sha256') or '').strip() # The empty-string placeholder is not an identity: models poisoned by a # hash of zero bytes (issue #1162) are re-hashed once the file has # content, so an existing library heals on its next full scan. - needs_sha = not stored_sha or is_empty_placeholder_hash(stored_sha) + needs_sha = ( + self.verify_hashes_on_scan + or not stored_sha + or is_empty_placeholder_hash(stored_sha) + ) if needs_sha and hash_status != 'pending' and file_path: try: - if stored_sha: + if stored_sha and is_empty_placeholder_hash(stored_sha): logger.info( "Recomputing SHA256 for %s (stored hash is the empty-file placeholder)", file_path, ) - else: + elif not stored_sha: logger.info(f"Computing SHA256 hash for {file_path} (was empty from metadata)") sha256 = await calculate_sha256(file_path) if sha256 and not is_empty_placeholder_hash(sha256): - model_data['sha256'] = sha256.lower() + normalized_sha = sha256.lower() + hash_changed = stored_sha.lower() != normalized_sha + model_data['sha256'] = normalized_sha if isinstance(metadata, BaseModelMetadata): - metadata.sha256 = sha256.lower() - await MetadataManager.save_metadata(file_path, metadata) + metadata.sha256 = normalized_sha + if hash_changed: + await MetadataManager.save_metadata(file_path, metadata) except Exception as e: logger.error(f"Failed to compute SHA256 for {file_path}: {e}") @@ -4035,7 +4056,6 @@ class ModelScanner: ) target_file = os.path.join(target_path, final_filename).replace(os.sep, '/') - final_base_name = os.path.splitext(final_filename)[0] # Log if filename was changed due to conflict if final_filename != f"{base_name}{file_ext}": @@ -4056,7 +4076,7 @@ class ModelScanner: shutil.move(move_source, move_target) - # Move all associated files with the same base name + # Move this artifact's associated files. source_metadata = None moved_metadata_path = None @@ -4069,17 +4089,19 @@ class ModelScanner: if os.path.normpath(source_sidecar_dir) != os.path.normpath(source_dir): associated_dirs.append((source_sidecar_dir, target_sidecar_dir)) - # Find all files with the same base name in the source directories + # Match only this artifact's sidecars. Workflow artifacts include + # their original extension in the prefix so same-stem formats stay independent. files_to_move = [] metadata_filename = os.path.basename(get_metadata_path(source_path)) + source_sidecar_base = get_sidecar_base_name(source_path) + target_sidecar_base = get_sidecar_base_name(target_file) for assoc_source_dir, assoc_target_dir in associated_dirs: try: for file in os.listdir(assoc_source_dir): - if file.startswith(base_name + ".") and file != os.path.basename(source_path): + if file.startswith(source_sidecar_base + ".") and file != os.path.basename(source_path): source_file_path = os.path.join(assoc_source_dir, file) - # Generate new filename with the same base name as the model file - file_suffix = file[len(base_name):] # Get the part after base_name (e.g., ".metadata.json", ".preview.png") - new_associated_filename = f"{final_base_name}{file_suffix}" + file_suffix = file[len(source_sidecar_base):] + new_associated_filename = f"{target_sidecar_base}{file_suffix}" target_associated_path = os.path.join(assoc_target_dir, new_associated_filename) # Store metadata file path for special handling @@ -4208,11 +4230,8 @@ class ModelScanner: metadata['file_name'] = os.path.splitext(os.path.basename(model_path))[0] if 'preview_url' in metadata and metadata['preview_url']: - preview_dir = get_preview_dir(model_path) - # Update preview filename to match the new base name - new_base_name = os.path.splitext(os.path.basename(model_path))[0] preview_ext = get_preview_extension(metadata['preview_url']) - new_preview_path = os.path.join(preview_dir, f"{new_base_name}{preview_ext}") + new_preview_path = get_preview_path(model_path, preview_ext) metadata['preview_url'] = new_preview_path.replace(os.sep, '/') await MetadataManager.save_metadata(metadata_path, metadata) @@ -4797,8 +4816,7 @@ class ModelScanner: return None dir_path = get_preview_dir(file_path) - base_name = os.path.splitext(os.path.basename(file_path))[0] - preview_path = find_preview_file(base_name, dir_path) + preview_path = find_preview_file(get_sidecar_base_name(file_path), dir_path) if preview_path: return config.get_preview_static_url(preview_path) diff --git a/py/services/pending_delete_service.py b/py/services/pending_delete_service.py index b9c914b8..58fabae4 100644 --- a/py/services/pending_delete_service.py +++ b/py/services/pending_delete_service.py @@ -38,8 +38,8 @@ from typing import ( ) from ..utils.constants import PREVIEW_EXTENSIONS -from ..utils.sidecar_paths import get_metadata_path, get_sidecar_dir -from ..utils.workflow_extraction import WORKFLOW_DOCUMENT_SUFFIX +from ..utils.sidecar_paths import get_metadata_path, get_preview_path, get_sidecar_dir +from ..utils.workflow_extraction import get_workflow_document_path from ..utils import settings_paths logger = logging.getLogger(__name__) @@ -707,16 +707,14 @@ class PendingDeleteService: # alongside mode, the centralized mirror otherwise). sidecar_dir = get_sidecar_dir(model_path) patterns = [os.path.basename(get_metadata_path(model_path))] - # Derived workflow document (workflow artifacts only; absent elsewhere). - patterns.append(f"{file_name}{WORKFLOW_DOCUMENT_SUFFIX}") + patterns.append(os.path.basename(get_workflow_document_path(model_path))) for ext in PREVIEW_EXTENSIONS: - patterns.append(f"{file_name}{ext}") + patterns.append(os.path.basename(get_preview_path(model_path, ext))) seen = {os.path.normcase(main_path)} for pattern in patterns: path = os.path.abspath(os.path.join(sidecar_dir, pattern)) - # An image workflow artifact doubles as its own preview: it then - # matches both the main file and a preview pattern — stage it once. + # Companion patterns may overlap the main artifact; stage each path once. if os.path.normcase(path) in seen: continue if os.path.exists(path): diff --git a/py/services/preview_asset_service.py b/py/services/preview_asset_service.py index 109f1f7d..cb32b966 100644 --- a/py/services/preview_asset_service.py +++ b/py/services/preview_asset_service.py @@ -15,7 +15,11 @@ from ..utils.constants import ( from ..utils.civitai_utils import rewrite_preview_url from ..utils.file_utils import detect_media_extension, reconcile_media_extension from ..utils.preview_selection import resolve_mature_threshold, select_preview_media -from ..utils.sidecar_paths import get_metadata_path, get_preview_dir +from ..utils.sidecar_paths import ( + get_metadata_path, + get_preview_dir, + get_preview_path, +) from .settings_manager import get_settings_manager logger = logging.getLogger(__name__) @@ -67,8 +71,16 @@ class PreviewAssetService: if not first_preview: return + model_path_value = local_metadata.get("file_path") + model_path = str(model_path_value) if model_path_value else "" base_name = os.path.splitext(os.path.splitext(os.path.basename(metadata_path))[0])[0] - preview_dir = os.path.dirname(metadata_path) + preview_dir = get_preview_dir(model_path) if model_path else os.path.dirname(metadata_path) + + def build_preview_path(extension: str) -> str: + if model_path: + return get_preview_path(model_path, extension) + return os.path.join(preview_dir, base_name + extension) + # Centralized mirrors may not exist yet (unlike the model's own # directory in alongside mode). os.makedirs(preview_dir, exist_ok=True) @@ -92,7 +104,7 @@ class PreviewAssetService: if is_video: extension = extension_from_url(preview_url, ".mp4") - preview_path = os.path.join(preview_dir, base_name + extension) + preview_path = build_preview_path(extension) rewritten_url, rewritten = rewrite_preview_url(preview_url, media_type="video") attempt_urls = [] @@ -126,7 +138,7 @@ class PreviewAssetService: content, headers, fallback_url=preview_url ) if extension in SUPPORTED_MEDIA_EXTENSIONS["images"]: - preview_path = os.path.join(preview_dir, base_name + extension) + preview_path = build_preview_path(extension) try: with open(preview_path, "wb") as handle: handle.write(content) @@ -138,7 +150,7 @@ class PreviewAssetService: return extension = ".webp" - preview_path = os.path.join(preview_dir, base_name + extension) + preview_path = build_preview_path(extension) success, content, _headers = await downloader.download_to_memory( preview_url, use_auth=False ) @@ -180,7 +192,6 @@ class PreviewAssetService: ) -> Dict[str, object]: """Replace an existing preview asset for a model.""" - base_name = os.path.splitext(os.path.basename(model_path))[0] folder = get_preview_dir(model_path) # Centralized mirrors may not exist yet (unlike the model's own # directory in alongside mode). @@ -191,7 +202,7 @@ class PreviewAssetService: ) for ext in PREVIEW_EXTENSIONS: - existing_preview = os.path.join(folder, base_name + ext) + existing_preview = get_preview_path(model_path, ext) if os.path.exists(existing_preview): try: os.remove(existing_preview) @@ -200,7 +211,7 @@ class PreviewAssetService: "Failed to delete existing preview %s: %s", existing_preview, exc ) - preview_path = os.path.join(folder, base_name + extension).replace(os.sep, "/") + preview_path = get_preview_path(model_path, extension).replace(os.sep, "/") with open(preview_path, "wb") as handle: handle.write(optimized_data) diff --git a/py/services/use_cases/sidecar_migration_use_case.py b/py/services/use_cases/sidecar_migration_use_case.py index dc896476..63f4818e 100644 --- a/py/services/use_cases/sidecar_migration_use_case.py +++ b/py/services/use_cases/sidecar_migration_use_case.py @@ -55,6 +55,7 @@ from ...utils.sidecar_paths import ( ROOT_MAP_FILENAME, STORAGE_MODE_CENTRALIZED, get_configured_sidecar_root, + get_sidecar_base_name, get_sidecar_root, get_storage_mode, relocate_root_map, @@ -573,7 +574,7 @@ class SidecarMigrationUseCase: result["skipped"] = 1 return result - stem = os.path.splitext(os.path.basename(model_path))[0] + stem = get_sidecar_base_name(model_path) sidecar_name = stem + METADATA_SUFFIX moved_previews: List[str] = [] diff --git a/py/services/workflow_scanner.py b/py/services/workflow_scanner.py index 09dd8076..db17a6ef 100644 --- a/py/services/workflow_scanner.py +++ b/py/services/workflow_scanner.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import logging import os from datetime import datetime @@ -16,7 +17,12 @@ from ..utils.file_utils import ( ) from ..utils.metadata_manager import MetadataManager from ..utils.models import WorkflowMetadata -from ..utils.sidecar_paths import get_preview_dir +from ..utils.sidecar_paths import ( + get_legacy_metadata_path, + get_metadata_path, + get_preview_dir, + get_sidecar_base_name, +) from ..utils.workflow_analysis import analyze_workflow_file from ..utils.workflow_extraction import ( IMAGE_CONTAINER_EXTENSIONS, @@ -57,6 +63,7 @@ class WorkflowScanner(ModelScanner): # Workflows are graph documents, not weights: quantization has no meaning # for them, so the inherited scan-time pass and backfill are skipped. supports_quantization = False + verify_hashes_on_scan = True def __init__(self): super().__init__( @@ -92,6 +99,74 @@ class WorkflowScanner(ModelScanner): def _is_image_artifact(file_path: str) -> bool: return os.path.splitext(file_path)[1].lower() in IMAGE_CONTAINER_EXTENSIONS + @staticmethod + def _path_key(file_path: str) -> str: + return os.path.normcase(os.path.abspath(os.path.normpath(file_path))) + + def _sidecar_references_preview( + self, metadata_path: str, preview_path: str, owner_path: str + ) -> bool: + try: + with open(metadata_path, "r", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, json.JSONDecodeError): + return False + referenced = payload.get("preview_url") if isinstance(payload, dict) else None + return bool( + referenced + and self._path_key(str(referenced)) == self._path_key(preview_path) + and self._path_key(owner_path) != self._path_key(preview_path) + ) + + def _is_known_preview(self, file_path: str) -> bool: + if not self._is_image_artifact(file_path): + return False + + cache = getattr(self, "_cache", None) + for item in getattr(cache, "raw_data", None) or []: + owner_path = item.get("file_path") + preview_url = item.get("preview_url") + if ( + owner_path + and preview_url + and self._path_key(owner_path) != self._path_key(file_path) + and self._path_key(preview_url) == self._path_key(file_path) + ): + return True + + stem = os.path.splitext(os.path.basename(file_path))[0] + artifact_dir = os.path.dirname(file_path) + for extension in WORKFLOW_ARTIFACT_EXTENSIONS: + owner_path = os.path.join(artifact_dir, stem + extension) + metadata_paths = { + get_metadata_path(owner_path), + get_legacy_metadata_path(owner_path), + } + for metadata_path in metadata_paths: + if os.path.isfile(metadata_path) and self._sidecar_references_preview( + metadata_path, file_path, owner_path + ): + return True + return False + + async def _process_model_file( + self, + file_path: str, + root_path: str, + *, + hash_index: Optional[ModelHashIndex] = None, + excluded_models: Optional[List[str]] = None, + ): + if self._is_known_preview(file_path): + logger.debug("Skipping workflow preview artifact: %s", file_path) + return None + return await super()._process_model_file( + file_path, + root_path, + hash_index=hash_index, + excluded_models=excluded_models, + ) + def adjust_metadata(self, metadata, file_path, root_path): """Fill workflow analysis fields when they are missing. @@ -156,7 +231,11 @@ class WorkflowScanner(ModelScanner): return None base_name = os.path.splitext(os.path.basename(file_path))[0] - preview_url = find_preview_file(base_name, get_preview_dir(file_path)) + preview_url = find_preview_file( + get_sidecar_base_name(file_path), get_preview_dir(file_path) + ) + if not preview_url: + preview_url = find_preview_file(base_name, get_preview_dir(file_path)) if not preview_url and self._is_image_artifact(file_path): # An image artifact doubles as its own preview. preview_url = file_path diff --git a/py/utils/constants.py b/py/utils/constants.py index 87d41f0f..c5298f8b 100644 --- a/py/utils/constants.py +++ b/py/utils/constants.py @@ -28,6 +28,8 @@ PREVIEW_EXTENSIONS = [ ".preview.jpeg", ".preview.jpg", ".preview.mp4", + ".preview.gif", + ".preview.webm", ".png", ".jpeg", ".jpg", diff --git a/py/utils/metadata_manager.py b/py/utils/metadata_manager.py index b5a7ef91..f22ae96b 100644 --- a/py/utils/metadata_manager.py +++ b/py/utils/metadata_manager.py @@ -9,7 +9,13 @@ from .models import BaseModelMetadata, CheckpointMetadata, EmbeddingMetadata, Lo from .file_utils import normalize_path, find_preview_file, calculate_sha256, calculate_autov3 from .gguf_utils import detect_quantization from .lora_metadata import extract_lora_metadata, extract_checkpoint_metadata -from .sidecar_paths import get_metadata_path, get_preview_dir, resolve_metadata_path +from .sidecar_paths import ( + get_legacy_metadata_path, + get_metadata_path, + get_preview_dir, + get_sidecar_base_name, + resolve_metadata_path, +) logger = logging.getLogger(__name__) @@ -35,10 +41,13 @@ class MetadataManager: - should_skip: True if corrupted metadata file exists and model should be skipped """ metadata_path = get_metadata_path(file_path) - - # Check if metadata file exists + if not os.path.exists(metadata_path): - return None, False + legacy_path = get_legacy_metadata_path(file_path) + if legacy_path != metadata_path and os.path.exists(legacy_path): + metadata_path = legacy_path + else: + return None, False try: with open(metadata_path, 'r', encoding='utf-8') as f: @@ -404,7 +413,7 @@ class MetadataManager: # Update preview if it doesn't exist OR if model and preview are in different directories if not os.path.exists(preview_url) or file_dir != preview_dir: - base_name = os.path.splitext(os.path.basename(file_path))[0] + base_name = get_sidecar_base_name(file_path) new_preview_url = find_preview_file(base_name, file_dir) if new_preview_url: metadata.preview_url = normalize_path(new_preview_url) diff --git a/py/utils/sidecar_paths.py b/py/utils/sidecar_paths.py index 5a69c8dc..35a1603a 100644 --- a/py/utils/sidecar_paths.py +++ b/py/utils/sidecar_paths.py @@ -1029,10 +1029,68 @@ def get_sidecar_dir(model_path: str) -> str: return os.path.dirname(os.path.abspath(model_path)) +def is_workflow_artifact_path(model_path: str) -> bool: + """Return whether *model_path* belongs to the configured workflow library.""" + + if os.path.splitext(model_path)[1].lower() not in { + ".json", + ".zip", + ".png", + ".webp", + }: + return False + try: + from ..config import config + + roots = config.workflows_roots or [] + except Exception: + return False + + normalized = os.path.normcase(os.path.abspath(os.path.normpath(model_path))) + for root in roots: + if not root: + continue + normalized_root = os.path.normcase(os.path.abspath(os.path.normpath(root))) + if normalized == normalized_root or normalized.startswith( + normalized_root + os.sep + ): + return True + return False + + +def get_sidecar_base_name(model_path: str) -> str: + """Return the filename prefix that uniquely identifies an artifact's assets.""" + + if is_workflow_artifact_path(model_path): + return os.path.basename(model_path) + return os.path.splitext(os.path.basename(model_path))[0] + + +def get_legacy_metadata_path(model_path: str) -> str: + """Return the pre-extension-qualified metadata path used by older versions.""" + + base_name = os.path.splitext(os.path.basename(model_path))[0] + METADATA_SUFFIX + return os.path.join(get_sidecar_dir(model_path), base_name) + + def get_metadata_path(model_path: str) -> str: """Return the ``.metadata.json`` sidecar path for a model file.""" - base_name = os.path.splitext(os.path.basename(model_path))[0] + METADATA_SUFFIX + base_name = get_sidecar_base_name(model_path) + METADATA_SUFFIX + return os.path.join(get_sidecar_dir(model_path), base_name) + + +def get_preview_path(model_path: str, extension: str) -> str: + """Return a preview path that cannot collide with a workflow artifact.""" + + if not extension.startswith("."): + extension = "." + extension + if is_workflow_artifact_path(model_path): + if not extension.startswith(".preview."): + extension = ".preview" + extension + base_name = os.path.basename(model_path) + extension + else: + base_name = os.path.splitext(os.path.basename(model_path))[0] + extension return os.path.join(get_sidecar_dir(model_path), base_name) diff --git a/py/utils/workflow_extraction.py b/py/utils/workflow_extraction.py index 90ad9f36..62b04088 100644 --- a/py/utils/workflow_extraction.py +++ b/py/utils/workflow_extraction.py @@ -7,7 +7,7 @@ extracted workflow document instead: - ``.json`` artifacts ARE the document; they are validated and read in place. - ``.zip`` / ``.png`` / ``.webp`` containers get the document materialized - once into an LM-owned derived sidecar ``.workflow.json`` inside the + once into an LM-owned derived ``.workflow.json`` sidecar inside the artifact's sidecar directory. Scanners exclude the suffix from the library walk; lifecycle operations (move/rename/delete) carry it like any other sidecar. @@ -21,7 +21,7 @@ import os import zipfile from typing import Any, Dict, List, Optional -from .sidecar_paths import get_sidecar_dir +from .sidecar_paths import get_sidecar_base_name, get_sidecar_dir from .workflow_analysis import ( MAX_WORKFLOW_JSON_BYTES, load_workflow_file, @@ -46,14 +46,16 @@ _MAX_ZIP_JSON_CANDIDATES = 64 def get_workflow_document_path(artifact_path: str) -> str: """Return the path of the extracted workflow document for an artifact. - ``.json`` artifacts are their own document; containers map to the - derived ``.workflow.json`` in the artifact's sidecar directory - (alongside the artifact or in the centralized mirror). + ``.json`` artifacts are their own document; containers map to a derived + ``.workflow.json`` sidecar in the artifact's sidecar directory (alongside + the artifact or in the centralized mirror). Workflow-library containers + include their original extension in the sidecar name to keep same-stem + artifacts independent. """ if os.path.splitext(artifact_path)[1].lower() == ".json": return artifact_path - base_name = os.path.splitext(os.path.basename(artifact_path))[0] + base_name = get_sidecar_base_name(artifact_path) return os.path.join( get_sidecar_dir(artifact_path), base_name + WORKFLOW_DOCUMENT_SUFFIX ) diff --git a/tests/routes/test_workflow_routes.py b/tests/routes/test_workflow_routes.py index cbf81ea1..4c6e5f07 100644 --- a/tests/routes/test_workflow_routes.py +++ b/tests/routes/test_workflow_routes.py @@ -1464,7 +1464,7 @@ async def test_create_workflow_attaches_preview_for_matching_prompt( assert response.status == 200 assert payload["preview"] is True - assert (root / "With Preview.preview.webp").exists() + assert (root / "With Preview.json.preview.webp").exists() async def test_create_workflow_skips_preview_for_other_prompt( diff --git a/tests/services/test_model_lifecycle_service.py b/tests/services/test_model_lifecycle_service.py index bf89de0d..eb2f2235 100644 --- a/tests/services/test_model_lifecycle_service.py +++ b/tests/services/test_model_lifecycle_service.py @@ -1199,3 +1199,33 @@ async def test_rename_image_workflow_artifact_renames_main_file_once(tmp_path: P assert not model_path.exists() renamed = result["renamed_files"] assert len(renamed) == len(set(renamed)) + + +@pytest.mark.asyncio +async def test_delete_workflow_artifact_preserves_same_stem_sibling( + tmp_path: Path, monkeypatch +): + from py import config as config_module + from py.services.model_lifecycle_service import delete_model_artifacts + + monkeypatch.setattr(config_module.config, "workflows_roots", [tmp_path.as_posix()]) + zip_path = tmp_path / "same.zip" + webp_path = tmp_path / "same.webp" + zip_path.write_bytes(b"zip") + webp_path.write_bytes(b"webp") + + for base in ("same.zip", "same.webp"): + (tmp_path / f"{base}.metadata.json").write_text("{}", encoding="utf-8") + (tmp_path / f"{base}.workflow.json").write_text("{}", encoding="utf-8") + (tmp_path / f"{base}.preview.webp").write_bytes(b"preview") + + await delete_model_artifacts(str(tmp_path), "same", main_extension=".zip") + + assert not zip_path.exists() + assert not (tmp_path / "same.zip.metadata.json").exists() + assert not (tmp_path / "same.zip.workflow.json").exists() + assert not (tmp_path / "same.zip.preview.webp").exists() + assert webp_path.read_bytes() == b"webp" + assert (tmp_path / "same.webp.metadata.json").is_file() + assert (tmp_path / "same.webp.workflow.json").is_file() + assert (tmp_path / "same.webp.preview.webp").is_file() diff --git a/tests/services/test_model_scanner.py b/tests/services/test_model_scanner.py index a2e19b1a..91c76d85 100644 --- a/tests/services/test_model_scanner.py +++ b/tests/services/test_model_scanner.py @@ -864,6 +864,28 @@ async def test_reconcile_cache_adds_new_files_and_updates_hash_index(tmp_path: P assert cache.folders == [""] +@pytest.mark.asyncio +async def test_reconcile_cache_removes_entries_under_removed_roots(tmp_path: Path): + root = tmp_path / "current" + root.mkdir() + (root / "one.txt").write_text("one", encoding="utf-8") + scanner = DummyScanner(root) + await scanner._initialize_cache() + + stale_path = _normalize_path(tmp_path / "removed" / "old.txt") + stale_entry = dict(scanner._cache.raw_data[0]) + stale_entry["file_path"] = stale_path + stale_entry["sha256"] = "hash-stale" + scanner._cache.raw_data.append(stale_entry) + scanner._hash_index.add_entry("hash-stale", stale_path) + + summary = await scanner._reconcile_cache() + + assert stale_path not in {item["file_path"] for item in scanner._cache.raw_data} + assert scanner._hash_index.get_path("hash-stale") is None + assert summary["removed"] == 1 + + @pytest.mark.asyncio async def test_reconcile_cache_applies_adjust_cached_entry(tmp_path: Path): existing = tmp_path / "one.txt" diff --git a/tests/services/test_workflow_auto_organize.py b/tests/services/test_workflow_auto_organize.py index 3a9e5c46..2aec3f73 100644 --- a/tests/services/test_workflow_auto_organize.py +++ b/tests/services/test_workflow_auto_organize.py @@ -87,12 +87,12 @@ async def test_auto_organize_moves_workflow_with_its_sidecars(workflow_library: json.dumps({"1": {"class_type": "CheckpointLoaderSimple", "inputs": {}}}), encoding="utf-8", ) - (old_dir / "portrait.workflow.json").write_text( + (old_dir / "portrait.json.workflow.json").write_text( json.dumps({"1": {"class_type": "CheckpointLoaderSimple"}}), encoding="utf-8", ) - (old_dir / "portrait.preview.webp").write_bytes(b"preview") - (old_dir / "portrait.metadata.json").write_text( + (old_dir / "portrait.json.preview.webp").write_bytes(b"preview") + (old_dir / "portrait.json.metadata.json").write_text( json.dumps( { "file_path": _normalize(workflow_path), @@ -121,11 +121,11 @@ async def test_auto_organize_moves_workflow_with_its_sidecars(workflow_library: assert result.operation_type == "bulk" assert result.success_count == 1 assert (target_dir / "portrait.json").is_file() - assert (target_dir / "portrait.workflow.json").is_file() - assert (target_dir / "portrait.preview.webp").read_bytes() == b"preview" + assert (target_dir / "portrait.json.workflow.json").is_file() + assert (target_dir / "portrait.json.preview.webp").read_bytes() == b"preview" # The metadata sidecar follows the artifact and records the new path. moved_metadata = json.loads( - (target_dir / "portrait.metadata.json").read_text(encoding="utf-8") + (target_dir / "portrait.json.metadata.json").read_text(encoding="utf-8") ) assert moved_metadata["file_path"] == _normalize(target_dir / "portrait.json") diff --git a/tests/services/test_workflow_scanner.py b/tests/services/test_workflow_scanner.py index d5226914..ea050d8c 100644 --- a/tests/services/test_workflow_scanner.py +++ b/tests/services/test_workflow_scanner.py @@ -8,6 +8,7 @@ import pytest from py import config as config_module from py.services.model_scanner import ModelScanner from py.services.workflow_scanner import WorkflowScanner +from py.utils.metadata_manager import MetadataManager from py.utils.models import WorkflowMetadata @@ -162,7 +163,7 @@ async def test_create_default_metadata_from_zip_artifact(workflow_root, enable_w assert metadata.sha256 # The derived workflow document is materialized as a sidecar. - derived = workflow_root / "pack.workflow.json" + derived = workflow_root / "pack.zip.workflow.json" assert derived.exists() assert json.loads(derived.read_text(encoding="utf-8")) == workflow @@ -239,7 +240,7 @@ def test_adjust_metadata_fills_missing_analysis(workflow_root, enable_workflows) assert adjusted.workflow_format == "api" assert adjusted.node_count == 1 assert adjusted.model_refs == ["base.safetensors"] - assert (workflow_root / "downloaded.workflow.json").exists() + assert (workflow_root / "downloaded.zip.workflow.json").exists() def test_adjust_metadata_keeps_populated_analysis(workflow_root, enable_workflows): @@ -449,3 +450,76 @@ async def test_register_local_artifact_allows_remote_resolution( assert metadata is not None assert metadata.base_model == "SD 1.5" assert captured["include_remote"] is True + + +async def test_same_stem_artifacts_recover_independent_hashes_from_legacy_sidecar( + workflow_root, enable_workflows +): + import hashlib + import zipfile + + from py.utils.sidecar_paths import get_metadata_path + + workflow = {"1": {"class_type": "KSampler", "inputs": {}}} + json_path = workflow_root / "same.json" + zip_path = workflow_root / "same.zip" + json_path.write_text(json.dumps(workflow), encoding="utf-8") + with zipfile.ZipFile(zip_path, "w") as archive: + archive.writestr("workflow.json", json.dumps(workflow)) + + (workflow_root / "same.metadata.json").write_text( + json.dumps( + { + "file_name": "same", + "model_name": "same", + "file_path": _normalize(json_path), + "size": 1, + "modified": 0, + "sha256": "polluted-shared-hash", + "base_model": "Unknown", + "preview_url": "", + "workflow_format": "", + } + ), + encoding="utf-8", + ) + + from py.services.model_hash_index import ModelHashIndex + + scanner = _make_scanner() + scanner._hash_index = ModelHashIndex() + scanner._excluded_models = [] + json_entry = await scanner._process_model_file(str(json_path), str(workflow_root)) + zip_entry = await scanner._process_model_file(str(zip_path), str(workflow_root)) + + assert json_entry["sha256"] == hashlib.sha256(json_path.read_bytes()).hexdigest() + assert zip_entry["sha256"] == hashlib.sha256(zip_path.read_bytes()).hexdigest() + assert json_entry["sha256"] != zip_entry["sha256"] + assert (workflow_root / "same.json.metadata.json").is_file() + assert (workflow_root / "same.zip.metadata.json").is_file() + assert get_metadata_path(str(json_path)) != get_metadata_path(str(zip_path)) + assert (workflow_root / "same.zip.workflow.json").is_file() + + +async def test_scan_skips_legacy_preview_referenced_by_sibling_metadata( + workflow_root, enable_workflows +): + import zipfile + + PIL = pytest.importorskip("PIL.Image") + from py.utils.exif_utils import ExifUtils + + workflow = {"nodes": [{"id": 1, "type": "KSampler"}], "links": []} + zip_path = workflow_root / "same.zip" + preview_path = workflow_root / "same.webp" + with zipfile.ZipFile(zip_path, "w") as archive: + archive.writestr("workflow.json", json.dumps(workflow)) + PIL.new("RGB", (8, 8)).save(preview_path, format="WEBP") + ExifUtils.embed_workflow(str(preview_path), json.dumps(workflow)) + + scanner = _make_scanner() + metadata = await scanner._create_default_metadata(str(zip_path)) + metadata.preview_url = _normalize(preview_path) + await MetadataManager.save_metadata(str(zip_path), metadata) + + assert await scanner._process_model_file(str(preview_path), str(workflow_root)) is None