From b9f364e3e4ea66f0c77bb3d7a1b36470d9d86fae Mon Sep 17 00:00:00 2001 From: Dariusz L Date: Sun, 5 Jul 2026 21:52:57 +0200 Subject: [PATCH] Add Local Metadata Builder UI and backend API This change introduces a complete Local Metadata Builder feature in the resolver options panel. It allows users to scan local model files, create or update .metadata.json sidecars, fill missing SHA256 hashes from safetensors headers or file hashing, and review run history with cancel/error handling. Changes include: - backend API: /model_resolver/metadata-build/{capabilities,start,progress/{id},cancel} - backend worker: core/metadata_builder.py with parallel metadata generation - frontend section: Local Metadata Builder tab in Options - frontend polling UI: progress, worker breakdown, history table, status/notifications --- __init__.py | 131 +++ core/metadata_builder.py | 1134 +++++++++++++++++++++++++ web/css/resolver-main.css | 276 ++++++ web/resolver/views/options_methods.js | 689 ++++++++++++++- 4 files changed, 2229 insertions(+), 1 deletion(-) create mode 100644 core/metadata_builder.py diff --git a/__init__.py b/__init__.py index 2644f9b..25810f4 100644 --- a/__init__.py +++ b/__init__.py @@ -140,6 +140,7 @@ class ModelResolverExtension: self.loaded_progress = JobProgressTracker("Loading loaded models...") self.search_tracker = JobProgressTracker("Searching...") self.hash_tracker = JobProgressTracker("Preparing hash calculation...") + self.metadata_builder_progress = JobProgressTracker("Building local metadata...") self.search_result_timestamps = {} def initialize(self): @@ -188,6 +189,10 @@ class ModelResolverExtension: ) from .core.path_templates import infer_download_path_templates from .core.metadata_audit import audit_metadata_sizes + from .core.metadata_builder import ( + build_missing_local_metadata, + get_metadata_build_capabilities, + ) from .core.scanner import get_model_files, invalidate_model_files_cache, find_local_file_path from .core.path_utils import ( extract_safetensors_header_sha256, @@ -1142,6 +1147,132 @@ class ModelResolverExtension: ) return web.json_response(result) + @routes.get("/model_resolver/metadata-build/capabilities") + @json_api_endpoint("metadata-build-capabilities") + async def metadata_build_capabilities_route(request): + """Return local CPU and worker limits for the metadata builder.""" + return web.json_response(get_metadata_build_capabilities()) + + @routes.post("/model_resolver/metadata-build/start") + @json_api_endpoint("metadata-build-start") + async def metadata_build_start_route(request): + """Create missing local metadata sidecars and fill missing SHA256 values.""" + import uuid + + payload = {} + try: + if request.can_read_body: + payload = await request.json() + except Exception: + payload = {} + if not isinstance(payload, dict): + payload = {} + + force_rescan = to_bool(payload.get("force_rescan"), True) + worker_count = to_int(payload.get("worker_count"), 0) + self.metadata_builder_progress.cleanup() + progress_id = f"metadata_build_{uuid.uuid4().hex}" + self.metadata_builder_progress.update( + progress_id, + status="queued", + stage="queued", + message="Preparing local metadata build...", + percent=0, + current=0, + total=0, + requested_worker_count=worker_count, + ) + + def update_metadata_build_progress(progress_payload): + if not isinstance(progress_payload, dict): + return + data = dict(progress_payload) + status = data.pop("status", None) + stage = data.get("stage") or "" + if not status: + if stage == "done": + status = "done" + elif stage == "cancelled": + status = "cancelled" + else: + status = "running" + self.metadata_builder_progress.update( + progress_id, + status=status, + **data, + ) + + def is_metadata_build_cancelled(): + return self.metadata_builder_progress.is_cancelled(progress_id) + + def run_metadata_build_task(): + try: + result = build_missing_local_metadata( + force_rescan=force_rescan, + worker_count=worker_count, + progress_callback=update_metadata_build_progress, + is_cancelled=is_metadata_build_cancelled, + ) + if result.get("cancelled") or is_metadata_build_cancelled(): + self.metadata_builder_progress.update( + progress_id, + status="cancelled", + stage="cancelled", + message="Metadata build cancelled", + percent=100, + result=result, + **result, + ) + return + + self.metadata_builder_progress.update( + progress_id, + status="done", + stage="done", + message="Local metadata build completed.", + percent=100, + result=result, + **result, + ) + except Exception as exc: + self.logger.exception(f"Metadata build failed: {exc}") + self.metadata_builder_progress.update( + progress_id, + status="error", + stage="error", + message=str(exc) or "Metadata build failed", + percent=100, + error=str(exc) or "Metadata build failed", + ) + + threading.Thread(target=run_metadata_build_task, daemon=True).start() + return web.json_response( + { + "success": True, + "progress_id": progress_id, + } + ) + + @routes.get("/model_resolver/metadata-build/progress/{progress_id}") + @json_api_endpoint("metadata-build-progress") + async def metadata_build_progress_route(request): + """Return progress for local metadata sidecar creation.""" + return get_progress_response( + self.metadata_builder_progress, + request, + not_found_status=404, + ) + + @routes.post("/model_resolver/metadata-build/cancel/{progress_id}") + @json_api_endpoint("metadata-build-cancel") + async def metadata_build_cancel_route(request): + """Cancel local metadata sidecar creation.""" + return cancel_progress_response( + self.metadata_builder_progress, + request, + cancel_message="Stopping metadata build...", + ) + @routes.post("/model_resolver/loaded") @json_api_endpoint("get_loaded_models") async def get_loaded_models(request): diff --git a/core/metadata_builder.py b/core/metadata_builder.py new file mode 100644 index 0000000..abd745e --- /dev/null +++ b/core/metadata_builder.py @@ -0,0 +1,1134 @@ +""" +Local metadata sidecar creation and repair helpers. + +This module intentionally uses only local model files and safetensors headers. +It does not call remote providers. +""" + +import os +import threading +import time +from collections.abc import Iterable +from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed +from typing import Any, Callable, Dict, List, Optional, Tuple + +from .log_system import create_module_logger +from .path_utils import ( + HashCalculationCancelled, + calculate_file_sha256, + extract_safetensors_header_metadata, + get_filename_from_path, + get_path_identity, + read_json_safe, + read_safetensors_header, + write_json_atomic, +) +from .type_utils import ( + MODEL_EXTENSIONS, + extract_sha256_from_metadata, + format_size_bytes, + normalize_category_to_model_type, + normalize_sha256, +) + +log = create_module_logger(__name__) + +ProgressCallback = Callable[[Dict[str, Any]], None] +CancelCallback = Callable[[], bool] + +LOCAL_HEADER_MAX_KEYS = 300 +LOCAL_HEADER_MAX_LIST_ITEMS = 64 +LOCAL_HEADER_MAX_CHARS = 20000 +MIN_METADATA_BUILD_WORKERS = 1 +MAX_METADATA_BUILD_WORKERS = 64 +DEFAULT_METADATA_BUILD_WORKER_LIMIT = 4 + + +def _metadata_sidecar_paths(model_path: str) -> List[str]: + directory = os.path.dirname(model_path) + filename = get_filename_from_path(model_path) + base_name = os.path.splitext(filename)[0] + candidates = [ + os.path.join(directory, f"{base_name}.metadata.json"), + os.path.join(directory, f"{filename}.metadata.json"), + ] + + result: List[str] = [] + seen = set() + for candidate in candidates: + try: + key = os.path.normcase(os.path.abspath(candidate)) + except (OSError, ValueError): + key = os.path.normcase(candidate) + if key in seen: + continue + seen.add(key) + result.append(candidate) + return result + + +def get_metadata_sidecar_path(model_path: str) -> str: + """Return the canonical local metadata sidecar path for a model file.""" + return _metadata_sidecar_paths(model_path)[0] + + +def _select_metadata_path(model_path: str) -> Tuple[str, bool]: + for candidate in _metadata_sidecar_paths(model_path): + if os.path.isfile(candidate): + return candidate, True + return get_metadata_sidecar_path(model_path), False + + +def _is_model_file_path(path: str) -> bool: + if not path or not os.path.isfile(path): + return False + + filename = get_filename_from_path(path).lower() + if filename.endswith((".metadata.json", ".civitai.info")): + return False + + return os.path.splitext(filename)[1].lower() in MODEL_EXTENSIONS + + +def _model_identity_key(model: Dict[str, Any]) -> str: + model_path = str(model.get("path") or "").strip() + if not model_path: + return "" + try: + return get_path_identity(model_path) + except (OSError, ValueError): + try: + return os.path.normcase(os.path.abspath(model_path)) + except (OSError, ValueError): + return os.path.normcase(model_path) + + +def _dedupe_models(models: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + seen = set() + for model in models or []: + if not isinstance(model, dict): + continue + identity = _model_identity_key(model) + if not identity or identity in seen: + continue + seen.add(identity) + result.append(model) + return result + + +def _coerce_positive_int(value: Any) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return 0 + return number if number > 0 else 0 + + +def _resolve_worker_count(total_models: int, requested_workers: Optional[int] = None) -> Tuple[int, int]: + cpu_count = os.cpu_count() or 1 + requested = _coerce_positive_int(requested_workers) + if requested: + workers = requested + else: + workers = min(cpu_count, DEFAULT_METADATA_BUILD_WORKER_LIMIT) + workers = max(MIN_METADATA_BUILD_WORKERS, min(MAX_METADATA_BUILD_WORKERS, workers)) + if total_models > 0: + workers = min(total_models, workers) + return workers, cpu_count + + +def get_metadata_build_capabilities() -> Dict[str, Any]: + """Return local concurrency limits for the metadata builder UI.""" + cpu_count = os.cpu_count() or 1 + return { + "success": True, + "cpu_count": cpu_count, + "default_worker_count": max( + MIN_METADATA_BUILD_WORKERS, + min(cpu_count, DEFAULT_METADATA_BUILD_WORKER_LIMIT), + ), + "min_worker_count": MIN_METADATA_BUILD_WORKERS, + "max_worker_count": MAX_METADATA_BUILD_WORKERS, + } + + +def _limited_local_value(value: Any, *, depth: int = 0) -> Any: + if depth > 5: + return str(value)[:LOCAL_HEADER_MAX_CHARS] + if isinstance(value, str): + text = value.strip() + return text[:LOCAL_HEADER_MAX_CHARS].rstrip() if len(text) > LOCAL_HEADER_MAX_CHARS else text + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, dict): + limited: Dict[str, Any] = {} + for index, key in enumerate(sorted(value.keys(), key=lambda item: str(item).lower())): + if index >= LOCAL_HEADER_MAX_KEYS: + limited["_truncated"] = True + break + limited[str(key)] = _limited_local_value(value.get(key), depth=depth + 1) + return limited + if isinstance(value, (list, tuple, set)): + items = list(value) + limited_list = [ + _limited_local_value(item, depth=depth + 1) + for item in items[:LOCAL_HEADER_MAX_LIST_ITEMS] + ] + if len(items) > LOCAL_HEADER_MAX_LIST_ITEMS: + limited_list.append({"_truncated": True}) + return limited_list + return str(value)[:LOCAL_HEADER_MAX_CHARS] + + +def extract_local_header_snapshot(model_path: str) -> Dict[str, Any]: + """Return a compact snapshot of safetensors __metadata__ values.""" + header_json = read_safetensors_header(model_path) + if not isinstance(header_json, dict): + return {} + + metadata = header_json.get("__metadata__") + if not isinstance(metadata, dict) or not metadata: + return {} + + keys = sorted(str(key) for key in metadata.keys()) + limited_metadata = {} + for key in keys[:LOCAL_HEADER_MAX_KEYS]: + limited_metadata[key] = _limited_local_value(metadata.get(key)) + + snapshot: Dict[str, Any] = { + "source": "safetensors_header", + "metadata_keys": keys[:LOCAL_HEADER_MAX_KEYS], + "metadata": limited_metadata, + } + if len(keys) > LOCAL_HEADER_MAX_KEYS: + snapshot["truncated"] = True + snapshot["total_metadata_keys"] = len(keys) + return snapshot + + +def _is_empty_value(value: Any) -> bool: + return value in (None, "", [], {}) + + +def _set_if_missing(target: Dict[str, Any], key: str, value: Any) -> bool: + if _is_empty_value(value) or not _is_empty_value(target.get(key)): + return False + target[key] = value + return True + + +def _merge_unique_strings(*values: Any) -> List[str]: + result: List[str] = [] + seen = set() + + def collect(value: Any) -> None: + if value in (None, ""): + return + if isinstance(value, str): + items = [item.strip() for item in value.split(",")] + elif isinstance(value, Iterable) and not isinstance(value, (dict, bytes, bytearray)): + items = list(value) + else: + items = [value] + for item in items: + text = str(item or "").strip() + if not text: + continue + key = text.lower() + if key in seen: + continue + seen.add(key) + result.append(text) + + for value in values: + collect(value) + return result + + +def _normalise_metadata_file_path(path_value: str) -> str: + return str(path_value or "").replace(os.sep, "/") + + +def _model_type_for_category(category: str) -> str: + model_type = normalize_category_to_model_type(category) + return model_type if model_type and model_type != "unknown" else "" + + +def _build_local_metadata_payload( + *, + existing: Dict[str, Any], + model: Dict[str, Any], + model_path: str, + metadata_path: str, + file_size: int, + header_metadata: Dict[str, Any], + header_snapshot: Dict[str, Any], + sha256: str, + sha256_source: str, +) -> Tuple[Dict[str, Any], List[str]]: + filename = get_filename_from_path(model_path) + stem = os.path.splitext(filename)[0] + category = str(model.get("category") or "") + now = time.time() + payload = dict(existing or {}) + changed_fields: List[str] = [] + + def mark_set(key: str, value: Any, *, force: bool = False) -> None: + if _is_empty_value(value): + return + if force or payload.get(key) != value: + payload[key] = value + changed_fields.append(key) + + def fill(key: str, value: Any) -> None: + if _set_if_missing(payload, key, value): + changed_fields.append(key) + + mark_set("file_name", stem, force=_is_empty_value(payload.get("file_name"))) + fill("filename", filename) + fill("model_name", header_metadata.get("model_name") or stem) + mark_set("file_path", _normalise_metadata_file_path(model_path), force=True) + mark_set("size", file_size, force=True) + mark_set("modified", now, force=True) + mark_set("last_checked_at", now, force=True) + + if category: + fill("category", category) + model_type = str(header_metadata.get("model_type") or _model_type_for_category(category) or "") + if model_type: + fill("model_type", model_type) + fill("sub_type", model_type) + + fill("base_model", header_metadata.get("base_model") or "Unknown") + for key in ( + "base_model_source", + "base_model_inferred", + "base_model_raw", + "description", + "model_description", + "license", + "usage_hint", + "usage_tips", + "preview_url", + "preview_nsfw_level", + "clip_skip", + "metadata_summary", + "header_metadata_keys", + ): + fill(key, header_metadata.get(key)) + + if header_metadata.get("description") and _is_empty_value(payload.get("modelDescription")): + mark_set("modelDescription", str(header_metadata.get("description"))) + + if header_metadata.get("creator"): + fill("creator", header_metadata.get("creator")) + elif header_metadata.get("author"): + fill( + "creator", + { + "username": str(header_metadata.get("author")), + "name": str(header_metadata.get("author")), + }, + ) + fill("author", header_metadata.get("author")) + + tags = _merge_unique_strings(payload.get("tags"), header_metadata.get("tags")) + if tags and tags != payload.get("tags"): + payload["tags"] = tags + changed_fields.append("tags") + + trained_words = _merge_unique_strings( + payload.get("trained_words"), + payload.get("trainedWords"), + header_metadata.get("trained_words"), + ) + if trained_words: + if trained_words != payload.get("trained_words"): + payload["trained_words"] = trained_words + changed_fields.append("trained_words") + if _is_empty_value(payload.get("trainedWords")): + payload["trainedWords"] = trained_words + changed_fields.append("trainedWords") + + if header_metadata.get("images") and _is_empty_value(payload.get("images")): + mark_set("images", header_metadata.get("images")) + + if header_snapshot: + fill("safetensors_header_metadata", header_snapshot) + + if sha256: + hashes = payload.get("hashes") + if not isinstance(hashes, dict): + hashes = {} + if hashes.get("SHA256") != sha256: + hashes["SHA256"] = sha256 + changed_fields.append("hashes") + mark_set("sha256", sha256, force=True) + fill("hash", sha256) + mark_set("hashes", hashes, force=True) + mark_set("hash_status", "completed", force=True) + mark_set("sha256_source", sha256_source or "file", force=True) + else: + fill("hash_status", "pending") + + fill("source", "local") + fill("details_source", "local") + fill("metadata_source", header_metadata.get("metadata_source") or "local_file") + if header_metadata.get("from_safetensors_header"): + mark_set("from_safetensors_header", True, force=True) + mark_set("local_metadata_available", True, force=True) + elif header_snapshot: + mark_set("local_metadata_available", True, force=True) + + fill("preview_nsfw_level", 0) + fill("notes", "") + fill("favorite", False) + fill("exclude", False) + fill("db_checked", False) + fill("skip_metadata_refresh", False) + mark_set("metadata_path", _normalise_metadata_file_path(metadata_path), force=True) + return payload, sorted(set(changed_fields)) + + +def _emit(progress_callback: Optional[ProgressCallback], payload: Dict[str, Any]) -> None: + if not progress_callback: + return + try: + progress_callback(payload) + except Exception: + log.debug("Metadata builder progress callback failed", exc_info=True) + + +def _progress_percent(current: int, total: int, current_file_ratio: float = 0.0) -> float: + if total <= 0: + return 0.0 + current_file_ratio = max(0.0, min(1.0, current_file_ratio)) + return min(99.0, ((max(0, current - 1) + current_file_ratio) / total) * 100.0) + + +def _empty_build_counts() -> Dict[str, Any]: + return { + "scanned_models": 0, + "created_metadata": 0, + "updated_metadata": 0, + "skipped_complete": 0, + "header_hashes": 0, + "calculated_hashes": 0, + "header_metadata_count": 0, + "invalid_metadata": 0, + "errors": [], + "updated": [], + "history": [], + } + + +def _merge_build_counts(target: Dict[str, Any], source: Dict[str, Any]) -> None: + for key in ( + "scanned_models", + "created_metadata", + "updated_metadata", + "skipped_complete", + "header_hashes", + "calculated_hashes", + "header_metadata_count", + "invalid_metadata", + ): + target[key] += int(source.get(key) or 0) + target["errors"].extend(source.get("errors") or []) + target["updated"].extend(source.get("updated") or []) + target["history"].extend(source.get("history") or []) + + +def _build_result_payload( + counts: Dict[str, Any], + *, + success: bool, + cancelled: bool, + total_models: int, + worker_count: int, + cpu_count: int, +) -> Dict[str, Any]: + errors = counts.get("errors") or [] + updated = counts.get("updated") or [] + history = counts.get("history") or [] + return { + "success": success, + "cancelled": cancelled, + "scanned_models": counts.get("scanned_models", 0), + "total_models": total_models, + "created_metadata": counts.get("created_metadata", 0), + "updated_metadata": counts.get("updated_metadata", 0), + "skipped_complete": counts.get("skipped_complete", 0), + "header_hashes": counts.get("header_hashes", 0), + "calculated_hashes": counts.get("calculated_hashes", 0), + "header_metadata_count": counts.get("header_metadata_count", 0), + "invalid_metadata": counts.get("invalid_metadata", 0), + "worker_count": worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "updated_count": len(updated), + "history": history[:500], + "history_count": len(history), + } + + +def _history_items_for_model(model: Dict[str, Any], result: Dict[str, Any]) -> List[Dict[str, Any]]: + updated_items = result.get("updated") or [] + if updated_items: + return [dict(item) for item in updated_items if isinstance(item, dict)] + + filename = get_filename_from_path(str(model.get("path") or "")) or str(model.get("filename") or "Model") + model_path = str(model.get("path") or "").strip() + metadata_path = get_metadata_sidecar_path(model_path) if model_path else "" + base_item = { + "filename": filename, + "relative_path": model.get("relative_path") or filename, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + } + + errors = result.get("errors") or [] + if errors: + return [ + { + **base_item, + **item, + "action": "error", + "message": item.get("message") or "Metadata build failed.", + } + for item in errors + if isinstance(item, dict) + ] + + if result.get("cancelled"): + return [{**base_item, "action": "cancelled", "message": "Cancelled"}] + + if int(result.get("skipped_complete") or 0) > 0: + return [{**base_item, "action": "skipped", "message": "Already had SHA256"}] + + return [{**base_item, "action": "checked", "message": "Checked"}] + + +def _build_missing_local_metadata_parallel( + model_items: List[Dict[str, Any]], + *, + worker_count: int, + cpu_count: int, + progress_callback: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCallback] = None, +) -> Dict[str, Any]: + total = len(model_items) + counts = _empty_build_counts() + state_lock = threading.Lock() + active_models: Dict[str, Dict[str, Any]] = {} + completed = 0 + cancelled = False + + def progress_percent_unlocked() -> float: + if total <= 0: + return 100.0 + active_ratio = sum(float(item.get("ratio") or 0.0) for item in active_models.values()) + return min(99.0, ((completed + active_ratio) / total) * 100.0) + + def emit_parallel_progress(stage: str, message: str, payload: Optional[Dict[str, Any]] = None) -> None: + data = payload or {} + with state_lock: + active_list = [ + { + "filename": item.get("filename") or "", + "path": item.get("path") or "", + "stage": item.get("stage") or "", + "percent": max(0.0, min(99.0, float(item.get("ratio") or 0.0) * 100.0)), + "bytes_read": item.get("bytes_read") or 0, + "total_bytes": item.get("total_bytes") or 0, + } + for item in active_models.values() + ][:worker_count] + snapshot = { + "stage": stage, + "message": message, + "current": completed, + "total": total, + "percent": progress_percent_unlocked(), + "worker_count": worker_count, + "cpu_count": cpu_count, + "active_models": active_list, + "active_worker_count": len(active_models), + "created_metadata": counts["created_metadata"], + "updated_metadata": counts["updated_metadata"], + "skipped_complete": counts["skipped_complete"], + "calculated_hashes": counts["calculated_hashes"], + "header_hashes": counts["header_hashes"], + "error_count": len(counts["errors"]), + } + if active_list: + first_active = active_list[0] + snapshot["current_model"] = first_active.get("filename") or "" + snapshot["current_path"] = first_active.get("path") or "" + snapshot["bytes_read"] = first_active.get("bytes_read") or 0 + snapshot["total_bytes"] = first_active.get("total_bytes") or 0 + snapshot.update(data) + _emit(progress_callback, snapshot) + + def run_one_model(model: Dict[str, Any]) -> Dict[str, Any]: + model_path = str(model.get("path") or "").strip() + filename = get_filename_from_path(model_path) + model_key = _model_identity_key(model) or model_path + + def child_progress(data: Dict[str, Any]) -> None: + if not isinstance(data, dict): + return + stage = str(data.get("stage") or "running") + child_status = str(data.get("status") or "").lower() + child_percent = max(0.0, min(100.0, float(data.get("percent") or 0.0))) + with state_lock: + if stage in {"done", "model_done"} or child_status in {"done", "cancelled"}: + active_models.pop(model_key, None) + else: + active_models[model_key] = { + "filename": filename, + "path": model_path, + "stage": stage, + "ratio": min(0.99, child_percent / 100.0), + "bytes_read": data.get("bytes_read") or 0, + "total_bytes": data.get("total_bytes") or 0, + } + if stage == "done" or child_status == "done": + return + emit_parallel_progress(stage, data.get("message") or f"Processing {filename}") + + return build_missing_local_metadata( + models=[model], + force_rescan=False, + worker_count=1, + progress_callback=child_progress, + is_cancelled=is_cancelled, + ) + + _emit( + progress_callback, + { + "stage": "scanning", + "message": f"Found {total} local model files.", + "current": 0, + "total": total, + "percent": 0, + "worker_count": worker_count, + "cpu_count": cpu_count, + }, + ) + + executor = ThreadPoolExecutor(max_workers=worker_count) + futures = {executor.submit(run_one_model, model): model for model in model_items} + try: + for future in as_completed(futures): + model = futures[future] + model_path = str(model.get("path") or "").strip() + filename = get_filename_from_path(model_path) + model_key = _model_identity_key(model) or model_path + try: + item_result = future.result() + except CancelledError: + item_result = {"success": False, "cancelled": True} + except Exception as exc: + log.warning(f"Could not build local metadata for {model_path}: {exc}", exc_info=True) + item_result = { + "success": False, + "errors": [ + { + "filename": filename, + "relative_path": model.get("relative_path") or filename, + "model_path": model_path, + "metadata_path": get_metadata_sidecar_path(model_path), + "message": str(exc) or "Metadata build failed.", + } + ], + } + history_items = item_result.get("history") or _history_items_for_model(model, item_result) + if "history" not in item_result: + item_result["history"] = history_items + + with state_lock: + completed += 1 + active_models.pop(model_key, None) + if item_result.get("cancelled"): + cancelled = True + _merge_build_counts(counts, item_result) + + emit_parallel_progress( + "model_done", + f"Processed {filename}", + { + "current_model": filename, + "current_path": model_path, + "current": completed, + "history_items": history_items, + }, + ) + + if is_cancelled and is_cancelled(): + cancelled = True + for pending in futures: + if not pending.done(): + pending.cancel() + finally: + executor.shutdown(wait=True, cancel_futures=True) + + result = _build_result_payload( + counts, + success=not cancelled, + cancelled=cancelled, + total_models=total, + worker_count=worker_count, + cpu_count=cpu_count, + ) + _emit( + progress_callback, + { + "stage": "cancelled" if cancelled else "done", + "message": "Metadata build cancelled" if cancelled else "Local metadata build completed.", + "status": "cancelled" if cancelled else "done", + "current": completed, + "total": total, + "percent": 100, + **result, + }, + ) + return result + + +def build_missing_local_metadata( + models: Optional[List[Dict[str, Any]]] = None, + *, + force_rescan: bool = True, + worker_count: Optional[int] = None, + progress_callback: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCallback] = None, +) -> Dict[str, Any]: + """ + Create missing sidecar metadata files and add missing SHA256 values. + + Existing sidecars are preserved and only filled with local technical values + when missing. No remote metadata providers are used. + """ + if models is None: + from .scanner import get_model_files + + models = get_model_files(force_rescan=force_rescan) + + all_models = _dedupe_models(models or []) + model_items = [ + model for model in all_models + if _is_model_file_path(str(model.get("path") or "").strip()) + ] + total = len(model_items) + resolved_worker_count, cpu_count = _resolve_worker_count(total, worker_count) + if resolved_worker_count > 1: + return _build_missing_local_metadata_parallel( + model_items, + worker_count=resolved_worker_count, + cpu_count=cpu_count, + progress_callback=progress_callback, + is_cancelled=is_cancelled, + ) + + scanned_models = 0 + created_metadata = 0 + updated_metadata = 0 + skipped_complete = 0 + header_hashes = 0 + calculated_hashes = 0 + header_metadata_count = 0 + invalid_metadata = 0 + errors: List[Dict[str, Any]] = [] + updated: List[Dict[str, Any]] = [] + history: List[Dict[str, Any]] = [] + + _emit( + progress_callback, + { + "stage": "scanning", + "message": f"Found {total} local model files.", + "current": 0, + "total": total, + "percent": 0, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + }, + ) + + for index, model in enumerate(model_items, start=1): + if is_cancelled and is_cancelled(): + _emit( + progress_callback, + { + "stage": "cancelled", + "message": "Metadata build cancelled", + "status": "cancelled", + "current": index - 1, + "total": total, + "percent": _progress_percent(index, total), + }, + ) + return { + "success": False, + "cancelled": True, + "scanned_models": scanned_models, + "total_models": total, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "header_hashes": header_hashes, + "calculated_hashes": calculated_hashes, + "header_metadata_count": header_metadata_count, + "invalid_metadata": invalid_metadata, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "history": history[:500], + "history_count": len(history), + } + + model_path = str(model.get("path") or "").strip() + filename = get_filename_from_path(model_path) + relative_path = model.get("relative_path") or filename + metadata_path, metadata_exists = _select_metadata_path(model_path) + scanned_models += 1 + + _emit( + progress_callback, + { + "stage": "header", + "message": f"Reading local metadata: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 0.1), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + }, + ) + + try: + file_size = os.path.getsize(model_path) + existing_raw = read_json_safe(metadata_path, {}) if metadata_exists else {} + if not isinstance(existing_raw, dict): + existing_raw = {} + invalid_metadata += 1 + + existing_sha256 = extract_sha256_from_metadata(existing_raw) + if metadata_exists and existing_sha256: + skipped_complete += 1 + history_item = { + "action": "skipped", + "filename": filename, + "relative_path": relative_path, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + "size": file_size, + "size_label": format_size_bytes(file_size), + "sha256": existing_sha256, + "sha256_source": "existing_metadata", + "message": "Already had SHA256", + } + history.append(history_item) + _emit( + progress_callback, + { + "stage": "model_done", + "message": f"Metadata already has SHA256: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 1.0), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "calculated_hashes": calculated_hashes, + "header_hashes": header_hashes, + "error_count": len(errors), + "history_items": [history_item], + }, + ) + continue + + header_metadata = extract_safetensors_header_metadata(model_path) + header_snapshot = extract_local_header_snapshot(model_path) + if header_metadata or header_snapshot: + header_metadata_count += 1 + + sha256 = existing_sha256 or normalize_sha256(header_metadata.get("sha256")) + sha256_source = "existing_metadata" if existing_sha256 else "" + if sha256 and not existing_sha256: + sha256_source = str(header_metadata.get("sha256_source") or "safetensors_header") + header_hashes += 1 + + if not sha256: + _emit( + progress_callback, + { + "stage": "hashing", + "message": f"Calculating SHA256: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 0.35), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "bytes_read": 0, + "total_bytes": file_size, + }, + ) + last_progress_at = [0.0] + + def on_hash_progress(bytes_read: int, total_bytes: int) -> None: + now = time.time() + if now - last_progress_at[0] < 0.2 and bytes_read < total_bytes: + return + ratio = 0.35 + if total_bytes > 0: + ratio = 0.35 + (min(1.0, bytes_read / total_bytes) * 0.55) + _emit( + progress_callback, + { + "stage": "hashing", + "message": f"Calculating SHA256: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, ratio), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "bytes_read": bytes_read, + "total_bytes": total_bytes, + }, + ) + last_progress_at[0] = now + + hash_source = [""] + + def on_hash_source(source: str) -> None: + hash_source[0] = source or "file" + + sha256 = calculate_file_sha256( + model_path, + chunk_size=1024 * 1024 * 4, + on_progress=on_hash_progress, + is_cancelled=is_cancelled, + use_safetensors_header=True, + on_hash_source=on_hash_source, + ) or "" + sha256 = normalize_sha256(sha256) + sha256_source = hash_source[0] or "file" + if sha256_source == "safetensors_header": + header_hashes += 1 + elif sha256: + calculated_hashes += 1 + + _emit( + progress_callback, + { + "stage": "writing", + "message": f"Writing metadata: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 0.95), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + }, + ) + + payload, changed_fields = _build_local_metadata_payload( + existing=existing_raw, + model=model, + model_path=model_path, + metadata_path=metadata_path, + file_size=file_size, + header_metadata=header_metadata, + header_snapshot=header_snapshot, + sha256=sha256, + sha256_source=sha256_source, + ) + + should_write = not metadata_exists or bool(changed_fields) or not existing_sha256 + history_item = None + if should_write: + write_json_atomic(metadata_path, payload, indent=2) + action = "created" if not metadata_exists else "updated" + if metadata_exists: + updated_metadata += 1 + else: + created_metadata += 1 + history_item = { + "action": action, + "filename": filename, + "relative_path": relative_path, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + "size": file_size, + "size_label": format_size_bytes(file_size), + "sha256": sha256, + "sha256_source": sha256_source, + "changed_fields": changed_fields, + } + updated.append(history_item) + history.append(history_item) + else: + skipped_complete += 1 + history_item = { + "action": "skipped", + "filename": filename, + "relative_path": relative_path, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + "size": file_size, + "size_label": format_size_bytes(file_size), + "sha256": sha256, + "sha256_source": sha256_source, + "message": "No changes needed", + } + history.append(history_item) + + _emit( + progress_callback, + { + "stage": "model_done", + "message": f"Processed {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 1.0), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "calculated_hashes": calculated_hashes, + "header_hashes": header_hashes, + "error_count": len(errors), + "history_items": [history_item] if history_item else [], + }, + ) + except HashCalculationCancelled: + _emit( + progress_callback, + { + "stage": "cancelled", + "message": "Metadata build cancelled", + "status": "cancelled", + "current": index - 1, + "total": total, + "percent": _progress_percent(index, total), + "current_model": filename, + "current_path": model_path, + }, + ) + return { + "success": False, + "cancelled": True, + "scanned_models": scanned_models, + "total_models": total, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "header_hashes": header_hashes, + "calculated_hashes": calculated_hashes, + "header_metadata_count": header_metadata_count, + "invalid_metadata": invalid_metadata, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "history": history[:500], + "history_count": len(history), + } + except Exception as exc: + log.warning(f"Could not build local metadata for {model_path}: {exc}", exc_info=True) + error_item = { + "filename": filename, + "relative_path": relative_path, + "model_path": model_path, + "metadata_path": metadata_path, + "message": str(exc) or "Metadata build failed.", + } + errors.append(error_item) + history_item = { + **error_item, + "action": "error", + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + } + history.append(history_item) + _emit( + progress_callback, + { + "stage": "model_done", + "message": f"Error processing {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 1.0), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "error_count": len(errors), + "history_items": [history_item], + }, + ) + + result = { + "success": True, + "cancelled": False, + "scanned_models": scanned_models, + "total_models": total, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "header_hashes": header_hashes, + "calculated_hashes": calculated_hashes, + "header_metadata_count": header_metadata_count, + "invalid_metadata": invalid_metadata, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "updated_count": len(updated), + "history": history[:500], + "history_count": len(history), + } + _emit( + progress_callback, + { + "stage": "done", + "message": "Local metadata build completed.", + "status": "done", + "current": total, + "total": total, + "percent": 100, + **result, + }, + ) + log.info( + "Local metadata build finished: " + f"models={scanned_models}, created={created_metadata}, updated={updated_metadata}, " + f"skipped={skipped_complete}, header_hashes={header_hashes}, " + f"calculated_hashes={calculated_hashes}, errors={len(errors)}" + ) + return result diff --git a/web/css/resolver-main.css b/web/css/resolver-main.css index b65eaa8..1ae8f84 100644 --- a/web/css/resolver-main.css +++ b/web/css/resolver-main.css @@ -6344,6 +6344,10 @@ a.mr-image-preview-resource-name:hover { align-content: stretch; } +.mr-options-section-fill .mr-options-build-stack { + grid-template-rows: auto auto auto auto auto minmax(0, 1fr); +} + .mr-options-panel-title { margin: 0 0 6px 0; font-size: 16px; @@ -6451,6 +6455,244 @@ a.mr-image-preview-resource-name:hover { color: var(--mr-color-error-light); } +.mr-options-inline-hint { + display: block; + margin-top: 3px; +} + +.mr-options-build-tabs { + align-self: flex-start; + align-items: flex-end; + width: fit-content; + height: 36px; + min-height: 36px; + margin-top: 12px; + overflow: visible; + --mr-tab-active-bg: rgba(20, 20, 24, 0.96); +} + +.mr-options-build-tabs .mr-options-build-tab { + gap: 5px; + min-width: 94px; + padding-right: 18px; + padding-left: 18px; +} + +.mr-options-build-tabs .mr-options-build-tab:not(.mr-tab-active):hover, +.mr-options-build-tabs .mr-options-build-tab:not(.mr-tab-active):focus, +.mr-options-build-tabs .mr-options-build-tab:not(.mr-tab-active):active, +.mr-options-build-tabs .mr-options-build-tab.mr-tab-active { + padding-right: 18px !important; + padding-left: 18px !important; +} + +.mr-options-build-tab-count { + position: relative; + z-index: 2; + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 18px; + height: 18px; + padding: 0 6px; + border-radius: 999px; + color: var(--mr-text); + background: rgba(255, 255, 255, 0.1); + font-size: 11px; + font-weight: 700; + line-height: 1; + pointer-events: none; +} + +.mr-options-build-tab.mr-tab-active .mr-options-build-tab-count { + background: rgba(255, 255, 255, 0.14); +} + +.mr-options-build-tab-panel { + min-width: 0; +} + +.mr-options-build-tab-panel.is-hidden { + display: none; +} + +.mr-options-build-progress { + display: grid; + gap: 8px; + margin-top: 10px; + min-width: 0; + padding: 10px; + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 6px; + background: rgba(0, 0, 0, 0.14); +} + +.mr-options-build-progress[hidden] { + display: none; +} + +.mr-options-build-progress-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + color: var(--mr-text-secondary); + font-size: 12px; +} + +.mr-options-build-progress-head strong { + color: var(--mr-text); + font-size: 12px; +} + +.mr-options-build-progress-bar { + height: 7px; + overflow: hidden; + border-radius: 999px; + background: rgba(255, 255, 255, 0.08); +} + +.mr-options-build-progress-bar span { + display: block; + width: 0; + height: 100%; + border-radius: inherit; + background: var(--mr-accent); + transition: width 0.2s ease; +} + +.mr-options-build-current { + display: grid; + gap: 3px; + min-width: 0; + color: var(--mr-text-secondary); + font-size: 12px; + line-height: 1.35; +} + +.mr-options-build-current span, +.mr-options-build-current code { + overflow-wrap: anywhere; +} + +.mr-options-build-current code { + color: var(--mr-text-muted); + font-family: ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace; + font-size: 11px; +} + +.mr-options-build-worker-list { + display: grid; + gap: 8px; + max-height: 260px; + min-height: 0; + overflow: auto; + padding-right: 2px; +} + +.mr-options-build-worker-list[hidden] { + display: none; +} + +.mr-options-build-worker { + display: grid; + gap: 5px; + min-width: 0; + padding: 8px 9px; + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 6px; + background: rgba(255, 255, 255, 0.035); +} + +.mr-options-build-worker-head, +.mr-options-build-worker-meta { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + min-width: 0; + color: var(--mr-text-secondary); + font-size: 11px; +} + +.mr-options-build-worker-head strong { + color: var(--mr-text); + font-size: 11px; +} + +.mr-options-build-worker-name { + min-width: 0; + overflow: hidden; + color: var(--mr-text); + font-size: 12px; + font-weight: 700; + text-overflow: ellipsis; + white-space: nowrap; +} + +.mr-options-build-worker code { + min-width: 0; + overflow: hidden; + color: var(--mr-text-muted); + font-family: ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace; + font-size: 10.5px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.mr-options-build-worker-bar { + height: 5px; + overflow: hidden; + border-radius: 999px; + background: rgba(255, 255, 255, 0.08); +} + +.mr-options-build-worker-bar span { + display: block; + width: 0; + height: 100%; + border-radius: inherit; + background: var(--mr-color-success); + transition: width 0.2s ease; +} + +.mr-options-history-action { + display: inline-flex; + align-items: center; + min-height: 20px; + padding: 2px 7px; + border-radius: 999px; + color: var(--mr-text); + background: rgba(255, 255, 255, 0.08); + font-size: 11px; + font-weight: 700; +} + +.mr-options-history-action.is-created { + color: var(--mr-color-success); + background: rgba(34, 197, 94, 0.1); +} + +.mr-options-history-action.is-updated { + color: var(--mr-accent); + background: rgba(96, 165, 250, 0.1); +} + +.mr-options-history-action.is-skipped, +.mr-options-history-action.is-checked { + color: var(--mr-text-secondary); +} + +.mr-options-history-action.is-error { + color: var(--mr-color-error-light); + background: rgba(248, 113, 113, 0.1); +} + +.mr-options-history-action.is-cancelled { + color: var(--mr-text-secondary); + background: rgba(255, 255, 255, 0.06); +} + .mr-options-audit-results { margin-top: 12px; min-width: 0; @@ -6533,6 +6775,40 @@ a.mr-image-preview-resource-name:hover { width: 12%; } +.mr-options-history-table { + min-width: 900px; +} + +.mr-options-history-table th:nth-child(1), +.mr-options-history-table td:nth-child(1) { + width: 30%; +} + +.mr-options-history-table th:nth-child(2), +.mr-options-history-table td:nth-child(2) { + width: 12%; +} + +.mr-options-history-table th:nth-child(3), +.mr-options-history-table td:nth-child(3) { + width: 11%; +} + +.mr-options-history-table th:nth-child(4), +.mr-options-history-table td:nth-child(4) { + width: 13%; +} + +.mr-options-history-table th:nth-child(5), +.mr-options-history-table td:nth-child(5) { + width: 8%; +} + +.mr-options-history-table th:nth-child(6), +.mr-options-history-table td:nth-child(6) { + width: 26%; +} + .mr-options-audit-model { color: var(--mr-text); font-weight: 700; diff --git a/web/resolver/views/options_methods.js b/web/resolver/views/options_methods.js index e9c659b..b2d1414 100644 --- a/web/resolver/views/options_methods.js +++ b/web/resolver/views/options_methods.js @@ -243,6 +243,13 @@ export const optionsMethods = { 07 +
@@ -253,7 +260,7 @@ export const optionsMethods = { Maintenance - 08 + 09
@@ -773,6 +780,92 @@ export const optionsMethods = { + @@ -861,9 +954,39 @@ export const optionsMethods = { const metadataSizeMismatchesEl = this.contentElement.querySelector('#mr-options-metadata-size-mismatches'); const metadataSizeMissingEl = this.contentElement.querySelector('#mr-options-metadata-size-missing'); const metadataSizeErrorsEl = this.contentElement.querySelector('#mr-options-metadata-size-errors'); + const metadataBuildStartBtn = this.contentElement.querySelector('#mr-options-metadata-build-start'); + const metadataBuildCancelBtn = this.contentElement.querySelector('#mr-options-metadata-build-cancel'); + const metadataBuildStatus = this.contentElement.querySelector('#mr-options-metadata-build-status'); + const metadataBuildProgress = this.contentElement.querySelector('#mr-options-metadata-build-progress'); + const metadataBuildStageEl = this.contentElement.querySelector('#mr-options-metadata-build-stage'); + const metadataBuildPercentEl = this.contentElement.querySelector('#mr-options-metadata-build-percent'); + const metadataBuildBarEl = this.contentElement.querySelector('#mr-options-metadata-build-bar'); + const metadataBuildCurrentModelEl = this.contentElement.querySelector('#mr-options-metadata-build-current-model'); + const metadataBuildCurrentPathEl = this.contentElement.querySelector('#mr-options-metadata-build-current-path'); + const metadataBuildBytesEl = this.contentElement.querySelector('#mr-options-metadata-build-bytes'); + const metadataBuildWorkerListEl = this.contentElement.querySelector('#mr-options-metadata-build-worker-list'); + const metadataBuildResults = this.contentElement.querySelector('#mr-options-metadata-build-results'); + const metadataBuildTabButtons = Array.from(this.contentElement.querySelectorAll('.mr-options-build-tab')); + const metadataBuildProgressPanel = this.contentElement.querySelector('#mr-options-metadata-build-progress-panel'); + const metadataBuildHistoryPanel = this.contentElement.querySelector('#mr-options-metadata-build-history-panel'); + const metadataBuildHistoryCountEl = this.contentElement.querySelector('#mr-options-metadata-build-history-count'); + const metadataBuildCpuEl = this.contentElement.querySelector('#mr-options-metadata-build-cpu'); + const metadataBuildWorkersEl = this.contentElement.querySelector('#mr-options-metadata-build-workers'); + const metadataBuildWorkerInput = this.contentElement.querySelector('#mr-options-metadata-build-worker-count'); + const metadataBuildWorkerHint = this.contentElement.querySelector('#mr-options-metadata-build-worker-hint'); + const metadataBuildScannedEl = this.contentElement.querySelector('#mr-options-metadata-build-scanned'); + const metadataBuildCreatedEl = this.contentElement.querySelector('#mr-options-metadata-build-created'); + const metadataBuildUpdatedEl = this.contentElement.querySelector('#mr-options-metadata-build-updated'); + const metadataBuildSkippedEl = this.contentElement.querySelector('#mr-options-metadata-build-skipped'); + const metadataBuildHashesEl = this.contentElement.querySelector('#mr-options-metadata-build-hashes'); + const metadataBuildErrorsEl = this.contentElement.querySelector('#mr-options-metadata-build-errors'); const optionsMain = this.contentElement.querySelector('.mr-options-main'); const navButtons = Array.from(this.contentElement.querySelectorAll('.mr-options-nav-btn')); const optionSections = Array.from(this.contentElement.querySelectorAll('.mr-options-section')); + if (this.metadataBuildPollTimer) { + window.clearTimeout(this.metadataBuildPollTimer); + this.metadataBuildPollTimer = null; + } const trackedInputs = [ civitaiInput, civitaiSessionInput, @@ -1177,6 +1300,540 @@ export const optionsMethods = { } }; + const setMetadataBuildBusy = (busy) => { + if (metadataBuildStartBtn) metadataBuildStartBtn.disabled = busy; + if (metadataBuildCancelBtn) { + metadataBuildCancelBtn.hidden = !busy; + metadataBuildCancelBtn.disabled = false; + } + }; + + const setMetadataBuildStatus = (text, mode = '') => { + if (!metadataBuildStatus) return; + metadataBuildStatus.textContent = text; + setStatusMode(metadataBuildStatus, mode); + }; + + const setMetadataBuildSummaryValue = (element, value, mode = '') => { + if (!element) return; + element.textContent = formatAuditNumber(value); + setStatusMode(element, mode); + }; + + const setMetadataBuildTab = (tabName = 'progress') => { + const activeTab = tabName === 'history' ? 'history' : 'progress'; + metadataBuildTabButtons.forEach((button) => { + const isActive = button.dataset.buildTab === activeTab; + button.classList.toggle('is-active', isActive); + button.classList.toggle('mr-tab-active', isActive); + button.setAttribute('aria-selected', isActive ? 'true' : 'false'); + }); + metadataBuildProgressPanel?.classList.toggle('is-hidden', activeTab !== 'progress'); + metadataBuildHistoryPanel?.classList.toggle('is-hidden', activeTab !== 'history'); + this.metadataBuildActiveTab = activeTab; + }; + + const getMetadataBuildHistoryKey = (item = {}) => { + const path = item.model_path || item.path || item.metadata_path || item.filename || ''; + const action = item.action || ''; + const sha = item.sha256 || ''; + return `${action}::${path}::${sha}`; + }; + + const appendMetadataBuildHistory = (data = {}) => { + if (!this.metadataBuildHistory) this.metadataBuildHistory = []; + if (!this.metadataBuildHistoryKeys) this.metadataBuildHistoryKeys = new Set(); + const source = data?.result && typeof data.result === 'object' ? data.result : data; + const items = []; + if (Array.isArray(data.history_items)) items.push(...data.history_items); + if (Array.isArray(source.history)) items.push(...source.history); + if (!items.length && Array.isArray(source.updated)) items.push(...source.updated); + items.forEach((rawItem) => { + if (!rawItem || typeof rawItem !== 'object') return; + const item = { ...rawItem }; + const key = getMetadataBuildHistoryKey(item); + if (!key || this.metadataBuildHistoryKeys.has(key)) return; + this.metadataBuildHistoryKeys.add(key); + this.metadataBuildHistory.unshift(item); + }); + if (this.metadataBuildHistory.length > 1000) { + this.metadataBuildHistory = this.metadataBuildHistory.slice(0, 1000); + this.metadataBuildHistoryKeys = new Set(this.metadataBuildHistory.map(getMetadataBuildHistoryKey)); + } + if (metadataBuildHistoryCountEl) { + metadataBuildHistoryCountEl.textContent = this.metadataBuildHistory.length.toLocaleString(); + } + }; + + const getMetadataBuildWorkerBounds = () => { + const capabilities = this.metadataBuildCapabilities || {}; + const min = Math.max(1, Number(capabilities.min_worker_count || 1)); + const max = Math.max(min, Number(capabilities.max_worker_count || 64)); + return { min, max }; + }; + + const normalizeMetadataBuildWorkerCount = () => { + const { min, max } = getMetadataBuildWorkerBounds(); + const fallback = Number(this.metadataBuildCapabilities?.default_worker_count || metadataBuildWorkerInput?.value || 1); + const raw = Number(metadataBuildWorkerInput?.value || fallback || 1); + const workerCount = Math.max(min, Math.min(max, Number.isFinite(raw) ? Math.round(raw) : fallback)); + if (metadataBuildWorkerInput) { + metadataBuildWorkerInput.min = String(min); + metadataBuildWorkerInput.max = String(max); + metadataBuildWorkerInput.value = String(workerCount); + } + return workerCount; + }; + + const renderMetadataBuildCapabilities = (data = {}) => { + this.metadataBuildCapabilities = data && typeof data === 'object' ? data : {}; + const cpuCount = Number(this.metadataBuildCapabilities.cpu_count || navigator.hardwareConcurrency || 1); + const defaultWorkers = Number(this.metadataBuildCapabilities.default_worker_count || Math.min(cpuCount, 4) || 1); + const maxWorkers = Number(this.metadataBuildCapabilities.max_worker_count || 64); + if (metadataBuildCpuEl) metadataBuildCpuEl.textContent = cpuCount.toLocaleString(); + if (metadataBuildWorkerInput && !metadataBuildWorkerInput.dataset.userEdited) { + metadataBuildWorkerInput.value = String(defaultWorkers); + } + normalizeMetadataBuildWorkerCount(); + if (metadataBuildWorkerHint) { + metadataBuildWorkerHint.textContent = `CPU cores: ${cpuCount.toLocaleString()}. Max workers: ${maxWorkers.toLocaleString()}.`; + } + if (metadataBuildWorkersEl) metadataBuildWorkersEl.textContent = normalizeMetadataBuildWorkerCount().toLocaleString(); + }; + + const loadMetadataBuildCapabilities = async () => { + try { + const data = await this.fetchJson( + '/model_resolver/metadata-build/capabilities', + { silent: true }, + 'Load metadata build capabilities' + ); + renderMetadataBuildCapabilities(data); + } catch (error) { + renderMetadataBuildCapabilities({ + cpu_count: navigator.hardwareConcurrency || 1, + default_worker_count: Math.min(navigator.hardwareConcurrency || 1, 4), + min_worker_count: 1, + max_worker_count: 64 + }); + } + }; + + const updateMetadataBuildSummary = (data = {}) => { + const source = data?.result && typeof data.result === 'object' ? data.result : data; + const created = Number(source.created_metadata || 0); + const updated = Number(source.updated_metadata || 0); + const calculatedHashes = Number(source.calculated_hashes || 0); + const errorCount = Number(source.error_count || 0); + if (metadataBuildCpuEl && source.cpu_count) { + metadataBuildCpuEl.textContent = Number(source.cpu_count || 0).toLocaleString(); + } + if (metadataBuildWorkersEl) { + const activeWorkers = Number(source.active_worker_count ?? source.worker_count ?? normalizeMetadataBuildWorkerCount()); + metadataBuildWorkersEl.textContent = Number.isFinite(activeWorkers) ? activeWorkers.toLocaleString() : '0'; + } + setMetadataBuildSummaryValue(metadataBuildScannedEl, source.scanned_models || source.current || 0); + setMetadataBuildSummaryValue(metadataBuildCreatedEl, created, created > 0 ? 'is-valid' : ''); + setMetadataBuildSummaryValue(metadataBuildUpdatedEl, updated, updated > 0 ? 'is-valid' : ''); + setMetadataBuildSummaryValue(metadataBuildSkippedEl, source.skipped_complete || 0); + setMetadataBuildSummaryValue(metadataBuildHashesEl, calculatedHashes, calculatedHashes > 0 ? 'is-pending' : ''); + setMetadataBuildSummaryValue(metadataBuildErrorsEl, errorCount, errorCount > 0 ? 'is-invalid' : ''); + }; + + const metadataBuildStageLabels = { + queued: 'Queued', + scanning: 'Scanning', + header: 'Reading header', + hashing: 'Calculating SHA256', + writing: 'Writing metadata', + model_done: 'Processed', + done: 'Done', + cancelled: 'Cancelled', + error: 'Error' + }; + + const getMetadataBuildWorkerPercent = (item = {}) => { + const direct = Number(item.percent); + if (Number.isFinite(direct)) { + return Math.max(0, Math.min(100, direct)); + } + const bytesRead = Number(item.bytes_read || 0); + const totalBytes = Number(item.total_bytes || 0); + if (totalBytes > 0) { + return Math.max(0, Math.min(100, (bytesRead / totalBytes) * 100)); + } + return 0; + }; + + const renderMetadataBuildWorkers = (data = {}, activeModels = []) => { + if (!metadataBuildWorkerListEl) return; + const workerItems = activeModels.length + ? activeModels + : (data.current_model ? [{ + filename: data.current_model, + path: data.current_path || data.metadata_path || '', + stage: data.stage || '', + percent: data.percent || 0, + bytes_read: data.bytes_read || 0, + total_bytes: data.total_bytes || 0 + }] : []); + + if (!workerItems.length) { + metadataBuildWorkerListEl.innerHTML = ''; + metadataBuildWorkerListEl.hidden = true; + return; + } + + metadataBuildWorkerListEl.hidden = false; + metadataBuildWorkerListEl.innerHTML = workerItems.map((item, index) => { + const workerPercent = getMetadataBuildWorkerPercent(item); + const stageLabel = metadataBuildStageLabels[item.stage] || item.stage || 'Running'; + const bytesRead = Number(item.bytes_read || 0); + const totalBytes = Number(item.total_bytes || 0); + const bytesLabel = totalBytes > 0 + ? `${this.formatBytes(bytesRead)} / ${this.formatBytes(totalBytes)}` + : ''; + const filename = item.filename || 'Model'; + const path = item.path || ''; + return ` +
+
+ Worker ${index + 1} + ${Math.round(workerPercent)}% +
+
${this.escapeHtml(filename)}
+
+ ${this.escapeHtml(stageLabel)} + ${bytesLabel ? `${this.escapeHtml(bytesLabel)}` : ''} +
+ ${path ? `${this.escapeHtml(path)}` : ''} + +
+ `; + }).join(''); + }; + + const updateMetadataBuildProgressView = (data = {}) => { + if (metadataBuildProgress) metadataBuildProgress.hidden = false; + const stage = String(data.stage || data.status || '').trim(); + const percent = Math.max(0, Math.min(100, Number(data.percent || 0))); + if (metadataBuildStageEl) metadataBuildStageEl.textContent = metadataBuildStageLabels[stage] || stage || 'Running'; + if (metadataBuildPercentEl) metadataBuildPercentEl.textContent = `${Math.round(percent)}%`; + if (metadataBuildBarEl) metadataBuildBarEl.style.width = `${percent}%`; + + const current = Number(data.current || 0); + const total = Number(data.total || data.total_models || 0); + const activeModels = Array.isArray(data.active_models) ? data.active_models : []; + const currentModel = data.current_model || data.filename || ''; + const suffix = total > 0 ? ` (${current.toLocaleString()} / ${total.toLocaleString()})` : ''; + if (metadataBuildCurrentModelEl) { + if (activeModels.length) { + metadataBuildCurrentModelEl.textContent = `${activeModels.length.toLocaleString()} active worker${activeModels.length === 1 ? '' : 's'}${suffix}`; + } else { + metadataBuildCurrentModelEl.textContent = currentModel + ? `${currentModel}${suffix}` + : (data.message || `Preparing${suffix}`); + } + } + if (metadataBuildCurrentPathEl) { + const pathText = activeModels.length + ? '' + : (data.current_path || data.metadata_path || ''); + metadataBuildCurrentPathEl.textContent = pathText; + metadataBuildCurrentPathEl.hidden = !pathText; + } + + const bytesRead = Number(data.bytes_read || 0); + const totalBytes = Number(data.total_bytes || 0); + if (metadataBuildBytesEl) { + metadataBuildBytesEl.textContent = activeModels.length + ? '' + : totalBytes > 0 + ? `${this.formatBytes(bytesRead)} / ${this.formatBytes(totalBytes)}` + : ''; + } + renderMetadataBuildWorkers(data, activeModels); + }; + + const renderMetadataBuildResults = (data = null) => { + if (!metadataBuildResults) return; + const source = data?.result && typeof data.result === 'object' ? data.result : data; + const historyItems = Array.isArray(this.metadataBuildHistory) + ? this.metadataBuildHistory + : []; + if (!source && !historyItems.length) { + metadataBuildResults.hidden = true; + metadataBuildResults.innerHTML = ''; + return; + } + + const fallbackItems = source && Array.isArray(source.history) + ? source.history + : (source && Array.isArray(source.updated) ? source.updated : []); + const displayItems = historyItems.length ? historyItems : fallbackItems; + const errors = Array.isArray(source?.errors) ? source.errors : []; + const visibleItems = displayItems.slice(0, 500); + let html = ''; + + if (visibleItems.length) { + const rows = visibleItems.map((item) => { + const modelLabel = item.relative_path || item.filename || item.model_path || 'Model'; + const modelPath = item.model_path || ''; + const metadataPath = item.metadata_path || ''; + const shaSource = item.sha256_source || ''; + const shaLabel = item.sha256 ? `${String(item.sha256).slice(0, 12)}...` : 'Pending'; + const changedFieldList = Array.isArray(item.changed_fields) + ? item.changed_fields.filter(Boolean) + : []; + const changedFields = changedFieldList.join(', '); + const changedFieldsSummary = changedFieldList.length + ? `${changedFieldList.length} field${changedFieldList.length === 1 ? '' : 's'} changed` + : ''; + const changedFieldsTitle = changedFields ? ` title="${this.escapeHtml(changedFields)}"` : ''; + const action = item.action || 'checked'; + const actionClass = String(action).toLowerCase().replace(/[^a-z0-9_-]+/g, '-'); + const message = item.message || changedFieldsSummary; + const buildContext = modelPath ? { + context_scope: 'local_model', + open_folder_label: 'Show File in Folder', + name: item.filename || modelLabel, + filename: item.filename || modelLabel, + relative_path: item.relative_path || modelLabel, + path: modelPath, + resolved_path: modelPath, + open_path: modelPath, + folder_path: item.base_directory || '', + category: item.category || '', + metadata_path: metadataPath, + size: item.size || 0, + file_size: item.size || 0, + sha256: item.sha256 || '', + context_source: 'metadata_builder' + } : null; + const contextAttrs = buildContext + ? this.getContextMenuAttrs(buildContext, 'Right-click for model options') + : ''; + return ` + + +
${this.escapeHtml(modelLabel)}
+
${this.escapeHtml(modelPath)}
+ + + ${this.escapeHtml(action)} + ${message ? `
${this.escapeHtml(message)}
` : ''} + + ${this.escapeHtml(item.category || '')} + +
${this.escapeHtml(shaLabel)}
+
${this.escapeHtml(shaSource)}
+ + ${this.escapeHtml(item.size_label || this.formatBytes(item.size || 0))} + +
${this.escapeHtml(metadataPath)}
+ + + `; + }).join(''); + html += ` +
+ + + + + + + + + + + + ${rows} +
ModelActionFolderSHA256SizeMetadata file
+
+ `; + if (displayItems.length > visibleItems.length) { + html += `
Showing latest ${visibleItems.length.toLocaleString()} of ${displayItems.length.toLocaleString()} history items.
`; + } + } else if (source?.cancelled) { + html += '
Metadata build was cancelled before any history item was captured.
'; + } else { + html += '
No metadata history captured yet.
'; + } + + if (errors.length) { + const errorRows = errors.slice(0, 10).map((item) => { + const targetPath = item.metadata_path || item.model_path || ''; + return `
  • ${this.escapeHtml(item.message || 'Error')}${this.escapeHtml(targetPath)}
    • `; + }).join(''); + html += ` +
    +
    Write errors
    + +
    + `; + } + + metadataBuildResults.innerHTML = html; + metadataBuildResults.hidden = false; + }; + + const stopMetadataBuildPolling = () => { + if (this.metadataBuildPollTimer) { + window.clearTimeout(this.metadataBuildPollTimer); + this.metadataBuildPollTimer = null; + } + }; + + const isMetadataBuildTerminal = (data = {}) => { + const statusValue = String(data.status || '').toLowerCase(); + return ['done', 'error', 'cancelled'].includes(statusValue); + }; + + const getMetadataBuildProgressKey = (data = {}) => ( + String(data.progress_id || data?.result?.progress_id || this.metadataBuildProgressId || '').trim() + ); + + const finishMetadataBuild = (data = {}) => { + const progressKey = getMetadataBuildProgressKey(data); + if (progressKey && this.metadataBuildFinishedProgressId === progressKey) { + return; + } + if (progressKey) { + this.metadataBuildFinishedProgressId = progressKey; + } + + appendMetadataBuildHistory(data); + const source = data?.result && typeof data.result === 'object' ? data.result : data; + setMetadataBuildBusy(false); + renderMetadataBuildResults(source); + + if (data.status === 'cancelled' || source.cancelled) { + setMetadataBuildStatus(`Metadata build cancelled after ${Number(source.scanned_models || data.current || 0).toLocaleString()} model${Number(source.scanned_models || data.current || 0) === 1 ? '' : 's'}.`, 'is-pending'); + this.showNotification('Metadata build cancelled', 'warning'); + return; + } + + if (data.status === 'error') { + setMetadataBuildStatus(data.message || 'Metadata build failed.', 'is-invalid'); + this.showNotification('Metadata build failed', 'error'); + return; + } + + const created = Number(source.created_metadata || 0); + const updated = Number(source.updated_metadata || 0); + const errorCount = Number(source.error_count || 0); + const calculatedHashes = Number(source.calculated_hashes || 0); + const headerHashes = Number(source.header_hashes || 0); + const changed = created + updated; + const hashText = ` ${calculatedHashes.toLocaleString()} full SHA256 hash${calculatedHashes === 1 ? '' : 'es'} calculated, ${headerHashes.toLocaleString()} read from headers.`; + if (errorCount > 0) { + setMetadataBuildStatus(`Metadata build finished with ${errorCount.toLocaleString()} error${errorCount === 1 ? '' : 's'}.${hashText}`, 'is-invalid'); + this.showNotification('Metadata build finished with errors', 'warning'); + } else if (changed > 0) { + setMetadataBuildStatus(`Created ${created.toLocaleString()} and updated ${updated.toLocaleString()} metadata file${changed === 1 ? '' : 's'}.${hashText}`, 'is-valid'); + this.showNotification('Local metadata updated', 'success'); + } else { + setMetadataBuildStatus(`Checked ${Number(source.scanned_models || 0).toLocaleString()} model${Number(source.scanned_models || 0) === 1 ? '' : 's'}. All existing metadata already had SHA256.`, 'is-valid'); + this.showNotification('Local metadata already complete', 'success'); + } + }; + + const pollMetadataBuildProgress = async () => { + const progressId = this.metadataBuildProgressId; + if (!progressId) return; + try { + const data = await this.fetchJson( + `/model_resolver/metadata-build/progress/${encodeURIComponent(progressId)}`, + { silent: true }, + 'Metadata build progress' + ); + appendMetadataBuildHistory(data); + updateMetadataBuildSummary(data); + updateMetadataBuildProgressView(data); + renderMetadataBuildResults(data); + if (isMetadataBuildTerminal(data)) { + stopMetadataBuildPolling(); + finishMetadataBuild(data); + if (this.metadataBuildProgressId === progressId) { + this.metadataBuildProgressId = ''; + } + return; + } + stopMetadataBuildPolling(); + this.metadataBuildPollTimer = window.setTimeout(() => { + pollMetadataBuildProgress(); + }, 700); + } catch (error) { + stopMetadataBuildPolling(); + this.metadataBuildProgressId = ''; + setMetadataBuildBusy(false); + setMetadataBuildStatus(error.message || 'Metadata build progress failed.', 'is-invalid'); + } + }; + + const runMetadataBuild = async () => { + stopMetadataBuildPolling(); + this.metadataBuildProgressId = ''; + this.metadataBuildFinishedProgressId = ''; + this.metadataBuildHistory = []; + this.metadataBuildHistoryKeys = new Set(); + if (metadataBuildHistoryCountEl) metadataBuildHistoryCountEl.textContent = '0'; + setMetadataBuildBusy(true); + updateMetadataBuildSummary({}); + renderMetadataBuildResults(null); + setMetadataBuildTab('progress'); + updateMetadataBuildProgressView({ + stage: 'queued', + message: 'Preparing local metadata build...', + percent: 0, + current: 0, + total: 0 + }); + setMetadataBuildStatus('Preparing local metadata build...', 'is-pending'); + try { + const workerCount = normalizeMetadataBuildWorkerCount(); + const start = await this.fetchJson('/model_resolver/metadata-build/start', { + method: 'POST', + body: JSON.stringify({ + force_rescan: true, + worker_count: workerCount + }) + }, 'Start metadata build'); + if (!start?.progress_id) { + throw new Error('Metadata build did not return a progress id.'); + } + this.metadataBuildProgressId = start.progress_id; + setMetadataBuildStatus('Building local metadata...', 'is-pending'); + pollMetadataBuildProgress(); + } catch (error) { + this.metadataBuildProgressId = ''; + setMetadataBuildBusy(false); + setMetadataBuildStatus(error.message || 'Metadata build failed to start.', 'is-invalid'); + } + }; + + const cancelMetadataBuild = async () => { + const progressId = this.metadataBuildProgressId; + if (!progressId) return; + if (metadataBuildCancelBtn) metadataBuildCancelBtn.disabled = true; + setMetadataBuildStatus('Stopping metadata build...', 'is-pending'); + try { + await this.fetchJson( + `/model_resolver/metadata-build/cancel/${encodeURIComponent(progressId)}`, + { method: 'POST', silent: true }, + 'Cancel metadata build' + ); + stopMetadataBuildPolling(); + pollMetadataBuildProgress(); + } catch (error) { + setMetadataBuildStatus(error.message || 'Could not cancel metadata build.', 'is-invalid'); + if (metadataBuildCancelBtn) metadataBuildCancelBtn.disabled = false; + } + }; + const setTokenCheckStatus = (statusEl, text, mode = '') => { if (!statusEl) return; statusEl.textContent = text; @@ -1736,6 +2393,7 @@ export const optionsMethods = { optionsMain?.classList.toggle( 'mr-options-main-fill-section', targetId === 'mr-options-section-metadata-audit' + || targetId === 'mr-options-section-metadata-build' ); setActiveNav(targetId); }; @@ -2132,6 +2790,7 @@ export const optionsMethods = { }; populateDefaultRootSelects(); + loadMetadataBuildCapabilities(); setStatus('Saved only on this machine.'); setVisibleSection('mr-options-section-sources'); @@ -2219,6 +2878,34 @@ export const optionsMethods = { }); } + if (metadataBuildStartBtn) { + metadataBuildStartBtn.addEventListener('click', () => { + runMetadataBuild(); + }); + } + + if (metadataBuildCancelBtn) { + metadataBuildCancelBtn.addEventListener('click', () => { + cancelMetadataBuild(); + }); + } + + metadataBuildTabButtons.forEach((button) => { + button.addEventListener('click', () => { + setMetadataBuildTab(button.dataset.buildTab || 'progress'); + }); + }); + + if (metadataBuildWorkerInput) { + metadataBuildWorkerInput.addEventListener('change', () => { + metadataBuildWorkerInput.dataset.userEdited = 'true'; + const workerCount = normalizeMetadataBuildWorkerCount(); + if (metadataBuildWorkersEl && !this.metadataBuildProgressId) { + metadataBuildWorkersEl.textContent = workerCount.toLocaleString(); + } + }); + } + if (detectTemplatesBtn) { detectTemplatesBtn.addEventListener('click', () => { applyDetectedPathTemplates();