"""The staged LoRA reference import (#221, #249 System S10, #355). The import reads LoraManager's ``*.metadata.json`` sidecars and turns each LoRA into one prompt of the ``lora-manager`` category, tagged with its name and trigger words and linked to its images. It runs in three stages, and only the last writes to the database: 1. **discover**: read LoraManager's settings the way LoraManager does (``py/lora_manager_layout.py``: the settings file from the environment folder, the portable plugin file or the user config folder; the LoRA roots, the storage mode, the sidecar and example-images roots) and find the sidecars (``discover``). In ``alongside`` mode a sidecar beside its model is kept when ``.safetensors`` exists; in ``centralized`` mode every sidecar under the sidecar root is kept when its ``file_path`` names a model inside a LoRA root. ``fingerprint`` covers every scanned sidecar, kept or skipped (path, size, modification time, kept or not, #362) and the layout; ``preview`` adds the counts a review shows. Nothing is written and nothing is requested. 2. **stage**: read each sidecar and build its record. Its LoraManager preview and example images are linked where they are (``local_media``); only a LoRA without any local image downloads its CivitAI example images into the image cache first (best effort, counted). A cancelled or failed stage leaves the database untouched; downloaded images stay in the cache. 3. **replace**: the sidecars are found and fingerprinted once more (a file changed, removed or added while staging aborts the import), then ``PromptDatabase.replace_category_prompts`` swaps the old collection for the staged records in one transaction (or changes nothing). Both the System job (``POST /lora/import/apply``) and the legacy ``POST /lora/scan`` run these stages (``py/api/lora_integration.py``). """ import hashlib from dataclasses import dataclass, field from pathlib import Path from typing import AbstractSet, Callable, Dict, List, Optional from . import lora_utils from .lora_manager_layout import ( Discovery, discover_sidecars, layout_summary, local_media, ) try: from ..utils.hashing import generate_prompt_hash except ImportError: # pragma: no cover - flat test imports from utils.hashing import generate_prompt_hash LORA_CATEGORY = "lora-manager" JOB_TITLE = "LoRA reference import" # The job's outcome groups, in its counter-tile order. GROUPS = ("imported", "linked", "skipped", "downloaded", "download_errors", "errors") class ImportCancelled(Exception): """The import was cancelled while staging; nothing was written.""" def discover(lora_manager_path: Optional[str]) -> Discovery: """The sidecars of every LoRA whose model exists inside a LoRA root. ``lora_manager_path`` is resolved as the integration resolves it (an install under ``custom_nodes``; empty means auto-detection). Raises whatever listing a folder raised: a discovery that cannot see every folder must not replace the collection with part of it. """ return discover_sidecars(lora_utils.lora_manager_layout(lora_manager_path)) def fingerprint(discovery: Discovery) -> str: """What a review saw: the layout, then every scanned sidecar, kept or skipped, with its path, size, modification time and whether it was kept (#362). A LoraManager save that changes unrelated settings leaves it as it was.""" layout = discovery.layout digest = hashlib.sha256() for value in ( layout.settings_path, layout.source, layout.error, layout.mode, "\0".join(layout.lora_roots), layout.sidecar_root, layout.example_images_root, layout.library, ): digest.update(f"{value}\n".encode("utf-8", "surrogatepass")) for name in discovery.unreadable: digest.update(f"unreadable\0{name}\n".encode("utf-8", "surrogatepass")) kept = {str(sidecar.metadata_path) for sidecar in discovery.sidecars} for path in discovery.scanned_files: flag = 1 if str(path) in kept else 0 try: stat = path.stat() entry = f"{path}\0{stat.st_size}\0{stat.st_mtime_ns}\0{flag}\n" except OSError: entry = f"{path}\0missing\0{flag}\n" digest.update(entry.encode("utf-8", "surrogatepass")) return "sha256:" + digest.hexdigest() def preview( lora_manager_path: Optional[str], cache_dir: Path, media_extensions: AbstractSet[str], ) -> Dict: """The review's counts (blocking; reads only, requests nothing). ``{lora_files, with_metadata, skipped, with_local_media, planned_downloads, fingerprint, names, layout}``: the metadata files seen, the kept LoRAs whose sidecar parses, the rest, the LoRAs with a local image, the CivitAI images an import would download for the LoRAs without one (not yet in the cache), the first LoRA names and what the layout says (``layout_summary``). """ discovery = discover(lora_manager_path) layout = discovery.layout with_metadata = 0 with_local_media = 0 planned = 0 names = [] for sidecar in discovery.sidecars: metadata = lora_utils.read_lora_metadata(sidecar.metadata_path) if not metadata: continue with_metadata += 1 if local_media(sidecar, metadata, layout, media_extensions): with_local_media += 1 else: planned += lora_utils.planned_civitai_downloads( metadata, sidecar.metadata_path, cache_dir ) if len(names) < 50: names.append(lora_utils.get_model_name_from_metadata(metadata)) return { "lora_files": discovery.scanned, "with_metadata": with_metadata, "skipped": discovery.scanned - with_metadata, "with_local_media": with_local_media, "planned_downloads": planned, "fingerprint": fingerprint(discovery), "names": names, "layout": layout_summary(layout, Path.home()), } def build_record(metadata: Dict, path: Path, images: List[str]) -> Dict: """The prompt a LoRA becomes: its example prompt (else its name), tags, images.""" model_name = lora_utils.get_model_name_from_metadata(metadata) text = lora_utils.get_example_prompt_from_metadata(metadata) or model_name tags = ["lora-manager", f"lora:{model_name}"] tags.extend(lora_utils.get_trigger_words_from_metadata(metadata)) return { "name": model_name, "text": text, "hash": generate_prompt_hash(text), "tags": list(dict.fromkeys(tags)), "images": images, } @dataclass class StagedImport: """The records an import will write, and what staging counted.""" total: int records: List[Dict] = field(default_factory=list) skipped: List[str] = field(default_factory=list) stats: Dict[str, int] = field( default_factory=lambda: {"downloaded": 0, "download_errors": 0} ) def stage( discovery: Discovery, cache_dir: Path, api_key: str = "", *, media_extensions: AbstractSet[str], on_record: Optional[Callable[[int, Dict, "StagedImport"], None]] = None, cancelled: Callable[[], bool] = lambda: False, logger=None, ) -> StagedImport: """Read each sidecar and build its record with its images. Blocking. Local images (``local_media``) come first; a LoRA without any downloads its CivitAI example images. A sidecar that cannot be read, or whose record fails to build, is skipped by its name, as are the centralized sidecars discovery could not parse. ``cancelled()`` is asked before each sidecar (raises ImportCancelled); ``on_record(index, record_or_None, staged)`` follows each one. """ sidecars = discovery.sidecars staged = StagedImport(total=len(sidecars)) staged.skipped.extend(discovery.unreadable) for index, sidecar in enumerate(sidecars): if cancelled(): raise ImportCancelled() record = None path = sidecar.metadata_path metadata = lora_utils.read_lora_metadata(path) if not metadata: staged.skipped.append(path.name) else: try: local = local_media( sidecar, metadata, discovery.layout, media_extensions ) remote = [] if not local: remote = lora_utils.download_civitai_images( metadata, path, cache_dir, api_key, stats=staged.stats ) images = list(dict.fromkeys([*local, *remote])) record = build_record(metadata, path, images) staged.records.append(record) except Exception as error: # one LoRA never stops the import name = lora_utils.get_model_name_from_metadata(metadata) if logger is not None: logger.warning(f"Failed to prepare LoRA {name}: {error}") staged.skipped.append(name) if on_record is not None: on_record(index, record, staged) return staged