diff --git a/__init__.py b/__init__.py index 2644f9b..25810f4 100644 --- a/__init__.py +++ b/__init__.py @@ -140,6 +140,7 @@ class ModelResolverExtension: self.loaded_progress = JobProgressTracker("Loading loaded models...") self.search_tracker = JobProgressTracker("Searching...") self.hash_tracker = JobProgressTracker("Preparing hash calculation...") + self.metadata_builder_progress = JobProgressTracker("Building local metadata...") self.search_result_timestamps = {} def initialize(self): @@ -188,6 +189,10 @@ class ModelResolverExtension: ) from .core.path_templates import infer_download_path_templates from .core.metadata_audit import audit_metadata_sizes + from .core.metadata_builder import ( + build_missing_local_metadata, + get_metadata_build_capabilities, + ) from .core.scanner import get_model_files, invalidate_model_files_cache, find_local_file_path from .core.path_utils import ( extract_safetensors_header_sha256, @@ -1142,6 +1147,132 @@ class ModelResolverExtension: ) return web.json_response(result) + @routes.get("/model_resolver/metadata-build/capabilities") + @json_api_endpoint("metadata-build-capabilities") + async def metadata_build_capabilities_route(request): + """Return local CPU and worker limits for the metadata builder.""" + return web.json_response(get_metadata_build_capabilities()) + + @routes.post("/model_resolver/metadata-build/start") + @json_api_endpoint("metadata-build-start") + async def metadata_build_start_route(request): + """Create missing local metadata sidecars and fill missing SHA256 values.""" + import uuid + + payload = {} + try: + if request.can_read_body: + payload = await request.json() + except Exception: + payload = {} + if not isinstance(payload, dict): + payload = {} + + force_rescan = to_bool(payload.get("force_rescan"), True) + worker_count = to_int(payload.get("worker_count"), 0) + self.metadata_builder_progress.cleanup() + progress_id = f"metadata_build_{uuid.uuid4().hex}" + self.metadata_builder_progress.update( + progress_id, + status="queued", + stage="queued", + message="Preparing local metadata build...", + percent=0, + current=0, + total=0, + requested_worker_count=worker_count, + ) + + def update_metadata_build_progress(progress_payload): + if not isinstance(progress_payload, dict): + return + data = dict(progress_payload) + status = data.pop("status", None) + stage = data.get("stage") or "" + if not status: + if stage == "done": + status = "done" + elif stage == "cancelled": + status = "cancelled" + else: + status = "running" + self.metadata_builder_progress.update( + progress_id, + status=status, + **data, + ) + + def is_metadata_build_cancelled(): + return self.metadata_builder_progress.is_cancelled(progress_id) + + def run_metadata_build_task(): + try: + result = build_missing_local_metadata( + force_rescan=force_rescan, + worker_count=worker_count, + progress_callback=update_metadata_build_progress, + is_cancelled=is_metadata_build_cancelled, + ) + if result.get("cancelled") or is_metadata_build_cancelled(): + self.metadata_builder_progress.update( + progress_id, + status="cancelled", + stage="cancelled", + message="Metadata build cancelled", + percent=100, + result=result, + **result, + ) + return + + self.metadata_builder_progress.update( + progress_id, + status="done", + stage="done", + message="Local metadata build completed.", + percent=100, + result=result, + **result, + ) + except Exception as exc: + self.logger.exception(f"Metadata build failed: {exc}") + self.metadata_builder_progress.update( + progress_id, + status="error", + stage="error", + message=str(exc) or "Metadata build failed", + percent=100, + error=str(exc) or "Metadata build failed", + ) + + threading.Thread(target=run_metadata_build_task, daemon=True).start() + return web.json_response( + { + "success": True, + "progress_id": progress_id, + } + ) + + @routes.get("/model_resolver/metadata-build/progress/{progress_id}") + @json_api_endpoint("metadata-build-progress") + async def metadata_build_progress_route(request): + """Return progress for local metadata sidecar creation.""" + return get_progress_response( + self.metadata_builder_progress, + request, + not_found_status=404, + ) + + @routes.post("/model_resolver/metadata-build/cancel/{progress_id}") + @json_api_endpoint("metadata-build-cancel") + async def metadata_build_cancel_route(request): + """Cancel local metadata sidecar creation.""" + return cancel_progress_response( + self.metadata_builder_progress, + request, + cancel_message="Stopping metadata build...", + ) + @routes.post("/model_resolver/loaded") @json_api_endpoint("get_loaded_models") async def get_loaded_models(request): diff --git a/core/metadata_builder.py b/core/metadata_builder.py new file mode 100644 index 0000000..abd745e --- /dev/null +++ b/core/metadata_builder.py @@ -0,0 +1,1134 @@ +""" +Local metadata sidecar creation and repair helpers. + +This module intentionally uses only local model files and safetensors headers. +It does not call remote providers. +""" + +import os +import threading +import time +from collections.abc import Iterable +from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed +from typing import Any, Callable, Dict, List, Optional, Tuple + +from .log_system import create_module_logger +from .path_utils import ( + HashCalculationCancelled, + calculate_file_sha256, + extract_safetensors_header_metadata, + get_filename_from_path, + get_path_identity, + read_json_safe, + read_safetensors_header, + write_json_atomic, +) +from .type_utils import ( + MODEL_EXTENSIONS, + extract_sha256_from_metadata, + format_size_bytes, + normalize_category_to_model_type, + normalize_sha256, +) + +log = create_module_logger(__name__) + +ProgressCallback = Callable[[Dict[str, Any]], None] +CancelCallback = Callable[[], bool] + +LOCAL_HEADER_MAX_KEYS = 300 +LOCAL_HEADER_MAX_LIST_ITEMS = 64 +LOCAL_HEADER_MAX_CHARS = 20000 +MIN_METADATA_BUILD_WORKERS = 1 +MAX_METADATA_BUILD_WORKERS = 64 +DEFAULT_METADATA_BUILD_WORKER_LIMIT = 4 + + +def _metadata_sidecar_paths(model_path: str) -> List[str]: + directory = os.path.dirname(model_path) + filename = get_filename_from_path(model_path) + base_name = os.path.splitext(filename)[0] + candidates = [ + os.path.join(directory, f"{base_name}.metadata.json"), + os.path.join(directory, f"{filename}.metadata.json"), + ] + + result: List[str] = [] + seen = set() + for candidate in candidates: + try: + key = os.path.normcase(os.path.abspath(candidate)) + except (OSError, ValueError): + key = os.path.normcase(candidate) + if key in seen: + continue + seen.add(key) + result.append(candidate) + return result + + +def get_metadata_sidecar_path(model_path: str) -> str: + """Return the canonical local metadata sidecar path for a model file.""" + return _metadata_sidecar_paths(model_path)[0] + + +def _select_metadata_path(model_path: str) -> Tuple[str, bool]: + for candidate in _metadata_sidecar_paths(model_path): + if os.path.isfile(candidate): + return candidate, True + return get_metadata_sidecar_path(model_path), False + + +def _is_model_file_path(path: str) -> bool: + if not path or not os.path.isfile(path): + return False + + filename = get_filename_from_path(path).lower() + if filename.endswith((".metadata.json", ".civitai.info")): + return False + + return os.path.splitext(filename)[1].lower() in MODEL_EXTENSIONS + + +def _model_identity_key(model: Dict[str, Any]) -> str: + model_path = str(model.get("path") or "").strip() + if not model_path: + return "" + try: + return get_path_identity(model_path) + except (OSError, ValueError): + try: + return os.path.normcase(os.path.abspath(model_path)) + except (OSError, ValueError): + return os.path.normcase(model_path) + + +def _dedupe_models(models: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + seen = set() + for model in models or []: + if not isinstance(model, dict): + continue + identity = _model_identity_key(model) + if not identity or identity in seen: + continue + seen.add(identity) + result.append(model) + return result + + +def _coerce_positive_int(value: Any) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return 0 + return number if number > 0 else 0 + + +def _resolve_worker_count(total_models: int, requested_workers: Optional[int] = None) -> Tuple[int, int]: + cpu_count = os.cpu_count() or 1 + requested = _coerce_positive_int(requested_workers) + if requested: + workers = requested + else: + workers = min(cpu_count, DEFAULT_METADATA_BUILD_WORKER_LIMIT) + workers = max(MIN_METADATA_BUILD_WORKERS, min(MAX_METADATA_BUILD_WORKERS, workers)) + if total_models > 0: + workers = min(total_models, workers) + return workers, cpu_count + + +def get_metadata_build_capabilities() -> Dict[str, Any]: + """Return local concurrency limits for the metadata builder UI.""" + cpu_count = os.cpu_count() or 1 + return { + "success": True, + "cpu_count": cpu_count, + "default_worker_count": max( + MIN_METADATA_BUILD_WORKERS, + min(cpu_count, DEFAULT_METADATA_BUILD_WORKER_LIMIT), + ), + "min_worker_count": MIN_METADATA_BUILD_WORKERS, + "max_worker_count": MAX_METADATA_BUILD_WORKERS, + } + + +def _limited_local_value(value: Any, *, depth: int = 0) -> Any: + if depth > 5: + return str(value)[:LOCAL_HEADER_MAX_CHARS] + if isinstance(value, str): + text = value.strip() + return text[:LOCAL_HEADER_MAX_CHARS].rstrip() if len(text) > LOCAL_HEADER_MAX_CHARS else text + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, dict): + limited: Dict[str, Any] = {} + for index, key in enumerate(sorted(value.keys(), key=lambda item: str(item).lower())): + if index >= LOCAL_HEADER_MAX_KEYS: + limited["_truncated"] = True + break + limited[str(key)] = _limited_local_value(value.get(key), depth=depth + 1) + return limited + if isinstance(value, (list, tuple, set)): + items = list(value) + limited_list = [ + _limited_local_value(item, depth=depth + 1) + for item in items[:LOCAL_HEADER_MAX_LIST_ITEMS] + ] + if len(items) > LOCAL_HEADER_MAX_LIST_ITEMS: + limited_list.append({"_truncated": True}) + return limited_list + return str(value)[:LOCAL_HEADER_MAX_CHARS] + + +def extract_local_header_snapshot(model_path: str) -> Dict[str, Any]: + """Return a compact snapshot of safetensors __metadata__ values.""" + header_json = read_safetensors_header(model_path) + if not isinstance(header_json, dict): + return {} + + metadata = header_json.get("__metadata__") + if not isinstance(metadata, dict) or not metadata: + return {} + + keys = sorted(str(key) for key in metadata.keys()) + limited_metadata = {} + for key in keys[:LOCAL_HEADER_MAX_KEYS]: + limited_metadata[key] = _limited_local_value(metadata.get(key)) + + snapshot: Dict[str, Any] = { + "source": "safetensors_header", + "metadata_keys": keys[:LOCAL_HEADER_MAX_KEYS], + "metadata": limited_metadata, + } + if len(keys) > LOCAL_HEADER_MAX_KEYS: + snapshot["truncated"] = True + snapshot["total_metadata_keys"] = len(keys) + return snapshot + + +def _is_empty_value(value: Any) -> bool: + return value in (None, "", [], {}) + + +def _set_if_missing(target: Dict[str, Any], key: str, value: Any) -> bool: + if _is_empty_value(value) or not _is_empty_value(target.get(key)): + return False + target[key] = value + return True + + +def _merge_unique_strings(*values: Any) -> List[str]: + result: List[str] = [] + seen = set() + + def collect(value: Any) -> None: + if value in (None, ""): + return + if isinstance(value, str): + items = [item.strip() for item in value.split(",")] + elif isinstance(value, Iterable) and not isinstance(value, (dict, bytes, bytearray)): + items = list(value) + else: + items = [value] + for item in items: + text = str(item or "").strip() + if not text: + continue + key = text.lower() + if key in seen: + continue + seen.add(key) + result.append(text) + + for value in values: + collect(value) + return result + + +def _normalise_metadata_file_path(path_value: str) -> str: + return str(path_value or "").replace(os.sep, "/") + + +def _model_type_for_category(category: str) -> str: + model_type = normalize_category_to_model_type(category) + return model_type if model_type and model_type != "unknown" else "" + + +def _build_local_metadata_payload( + *, + existing: Dict[str, Any], + model: Dict[str, Any], + model_path: str, + metadata_path: str, + file_size: int, + header_metadata: Dict[str, Any], + header_snapshot: Dict[str, Any], + sha256: str, + sha256_source: str, +) -> Tuple[Dict[str, Any], List[str]]: + filename = get_filename_from_path(model_path) + stem = os.path.splitext(filename)[0] + category = str(model.get("category") or "") + now = time.time() + payload = dict(existing or {}) + changed_fields: List[str] = [] + + def mark_set(key: str, value: Any, *, force: bool = False) -> None: + if _is_empty_value(value): + return + if force or payload.get(key) != value: + payload[key] = value + changed_fields.append(key) + + def fill(key: str, value: Any) -> None: + if _set_if_missing(payload, key, value): + changed_fields.append(key) + + mark_set("file_name", stem, force=_is_empty_value(payload.get("file_name"))) + fill("filename", filename) + fill("model_name", header_metadata.get("model_name") or stem) + mark_set("file_path", _normalise_metadata_file_path(model_path), force=True) + mark_set("size", file_size, force=True) + mark_set("modified", now, force=True) + mark_set("last_checked_at", now, force=True) + + if category: + fill("category", category) + model_type = str(header_metadata.get("model_type") or _model_type_for_category(category) or "") + if model_type: + fill("model_type", model_type) + fill("sub_type", model_type) + + fill("base_model", header_metadata.get("base_model") or "Unknown") + for key in ( + "base_model_source", + "base_model_inferred", + "base_model_raw", + "description", + "model_description", + "license", + "usage_hint", + "usage_tips", + "preview_url", + "preview_nsfw_level", + "clip_skip", + "metadata_summary", + "header_metadata_keys", + ): + fill(key, header_metadata.get(key)) + + if header_metadata.get("description") and _is_empty_value(payload.get("modelDescription")): + mark_set("modelDescription", str(header_metadata.get("description"))) + + if header_metadata.get("creator"): + fill("creator", header_metadata.get("creator")) + elif header_metadata.get("author"): + fill( + "creator", + { + "username": str(header_metadata.get("author")), + "name": str(header_metadata.get("author")), + }, + ) + fill("author", header_metadata.get("author")) + + tags = _merge_unique_strings(payload.get("tags"), header_metadata.get("tags")) + if tags and tags != payload.get("tags"): + payload["tags"] = tags + changed_fields.append("tags") + + trained_words = _merge_unique_strings( + payload.get("trained_words"), + payload.get("trainedWords"), + header_metadata.get("trained_words"), + ) + if trained_words: + if trained_words != payload.get("trained_words"): + payload["trained_words"] = trained_words + changed_fields.append("trained_words") + if _is_empty_value(payload.get("trainedWords")): + payload["trainedWords"] = trained_words + changed_fields.append("trainedWords") + + if header_metadata.get("images") and _is_empty_value(payload.get("images")): + mark_set("images", header_metadata.get("images")) + + if header_snapshot: + fill("safetensors_header_metadata", header_snapshot) + + if sha256: + hashes = payload.get("hashes") + if not isinstance(hashes, dict): + hashes = {} + if hashes.get("SHA256") != sha256: + hashes["SHA256"] = sha256 + changed_fields.append("hashes") + mark_set("sha256", sha256, force=True) + fill("hash", sha256) + mark_set("hashes", hashes, force=True) + mark_set("hash_status", "completed", force=True) + mark_set("sha256_source", sha256_source or "file", force=True) + else: + fill("hash_status", "pending") + + fill("source", "local") + fill("details_source", "local") + fill("metadata_source", header_metadata.get("metadata_source") or "local_file") + if header_metadata.get("from_safetensors_header"): + mark_set("from_safetensors_header", True, force=True) + mark_set("local_metadata_available", True, force=True) + elif header_snapshot: + mark_set("local_metadata_available", True, force=True) + + fill("preview_nsfw_level", 0) + fill("notes", "") + fill("favorite", False) + fill("exclude", False) + fill("db_checked", False) + fill("skip_metadata_refresh", False) + mark_set("metadata_path", _normalise_metadata_file_path(metadata_path), force=True) + return payload, sorted(set(changed_fields)) + + +def _emit(progress_callback: Optional[ProgressCallback], payload: Dict[str, Any]) -> None: + if not progress_callback: + return + try: + progress_callback(payload) + except Exception: + log.debug("Metadata builder progress callback failed", exc_info=True) + + +def _progress_percent(current: int, total: int, current_file_ratio: float = 0.0) -> float: + if total <= 0: + return 0.0 + current_file_ratio = max(0.0, min(1.0, current_file_ratio)) + return min(99.0, ((max(0, current - 1) + current_file_ratio) / total) * 100.0) + + +def _empty_build_counts() -> Dict[str, Any]: + return { + "scanned_models": 0, + "created_metadata": 0, + "updated_metadata": 0, + "skipped_complete": 0, + "header_hashes": 0, + "calculated_hashes": 0, + "header_metadata_count": 0, + "invalid_metadata": 0, + "errors": [], + "updated": [], + "history": [], + } + + +def _merge_build_counts(target: Dict[str, Any], source: Dict[str, Any]) -> None: + for key in ( + "scanned_models", + "created_metadata", + "updated_metadata", + "skipped_complete", + "header_hashes", + "calculated_hashes", + "header_metadata_count", + "invalid_metadata", + ): + target[key] += int(source.get(key) or 0) + target["errors"].extend(source.get("errors") or []) + target["updated"].extend(source.get("updated") or []) + target["history"].extend(source.get("history") or []) + + +def _build_result_payload( + counts: Dict[str, Any], + *, + success: bool, + cancelled: bool, + total_models: int, + worker_count: int, + cpu_count: int, +) -> Dict[str, Any]: + errors = counts.get("errors") or [] + updated = counts.get("updated") or [] + history = counts.get("history") or [] + return { + "success": success, + "cancelled": cancelled, + "scanned_models": counts.get("scanned_models", 0), + "total_models": total_models, + "created_metadata": counts.get("created_metadata", 0), + "updated_metadata": counts.get("updated_metadata", 0), + "skipped_complete": counts.get("skipped_complete", 0), + "header_hashes": counts.get("header_hashes", 0), + "calculated_hashes": counts.get("calculated_hashes", 0), + "header_metadata_count": counts.get("header_metadata_count", 0), + "invalid_metadata": counts.get("invalid_metadata", 0), + "worker_count": worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "updated_count": len(updated), + "history": history[:500], + "history_count": len(history), + } + + +def _history_items_for_model(model: Dict[str, Any], result: Dict[str, Any]) -> List[Dict[str, Any]]: + updated_items = result.get("updated") or [] + if updated_items: + return [dict(item) for item in updated_items if isinstance(item, dict)] + + filename = get_filename_from_path(str(model.get("path") or "")) or str(model.get("filename") or "Model") + model_path = str(model.get("path") or "").strip() + metadata_path = get_metadata_sidecar_path(model_path) if model_path else "" + base_item = { + "filename": filename, + "relative_path": model.get("relative_path") or filename, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + } + + errors = result.get("errors") or [] + if errors: + return [ + { + **base_item, + **item, + "action": "error", + "message": item.get("message") or "Metadata build failed.", + } + for item in errors + if isinstance(item, dict) + ] + + if result.get("cancelled"): + return [{**base_item, "action": "cancelled", "message": "Cancelled"}] + + if int(result.get("skipped_complete") or 0) > 0: + return [{**base_item, "action": "skipped", "message": "Already had SHA256"}] + + return [{**base_item, "action": "checked", "message": "Checked"}] + + +def _build_missing_local_metadata_parallel( + model_items: List[Dict[str, Any]], + *, + worker_count: int, + cpu_count: int, + progress_callback: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCallback] = None, +) -> Dict[str, Any]: + total = len(model_items) + counts = _empty_build_counts() + state_lock = threading.Lock() + active_models: Dict[str, Dict[str, Any]] = {} + completed = 0 + cancelled = False + + def progress_percent_unlocked() -> float: + if total <= 0: + return 100.0 + active_ratio = sum(float(item.get("ratio") or 0.0) for item in active_models.values()) + return min(99.0, ((completed + active_ratio) / total) * 100.0) + + def emit_parallel_progress(stage: str, message: str, payload: Optional[Dict[str, Any]] = None) -> None: + data = payload or {} + with state_lock: + active_list = [ + { + "filename": item.get("filename") or "", + "path": item.get("path") or "", + "stage": item.get("stage") or "", + "percent": max(0.0, min(99.0, float(item.get("ratio") or 0.0) * 100.0)), + "bytes_read": item.get("bytes_read") or 0, + "total_bytes": item.get("total_bytes") or 0, + } + for item in active_models.values() + ][:worker_count] + snapshot = { + "stage": stage, + "message": message, + "current": completed, + "total": total, + "percent": progress_percent_unlocked(), + "worker_count": worker_count, + "cpu_count": cpu_count, + "active_models": active_list, + "active_worker_count": len(active_models), + "created_metadata": counts["created_metadata"], + "updated_metadata": counts["updated_metadata"], + "skipped_complete": counts["skipped_complete"], + "calculated_hashes": counts["calculated_hashes"], + "header_hashes": counts["header_hashes"], + "error_count": len(counts["errors"]), + } + if active_list: + first_active = active_list[0] + snapshot["current_model"] = first_active.get("filename") or "" + snapshot["current_path"] = first_active.get("path") or "" + snapshot["bytes_read"] = first_active.get("bytes_read") or 0 + snapshot["total_bytes"] = first_active.get("total_bytes") or 0 + snapshot.update(data) + _emit(progress_callback, snapshot) + + def run_one_model(model: Dict[str, Any]) -> Dict[str, Any]: + model_path = str(model.get("path") or "").strip() + filename = get_filename_from_path(model_path) + model_key = _model_identity_key(model) or model_path + + def child_progress(data: Dict[str, Any]) -> None: + if not isinstance(data, dict): + return + stage = str(data.get("stage") or "running") + child_status = str(data.get("status") or "").lower() + child_percent = max(0.0, min(100.0, float(data.get("percent") or 0.0))) + with state_lock: + if stage in {"done", "model_done"} or child_status in {"done", "cancelled"}: + active_models.pop(model_key, None) + else: + active_models[model_key] = { + "filename": filename, + "path": model_path, + "stage": stage, + "ratio": min(0.99, child_percent / 100.0), + "bytes_read": data.get("bytes_read") or 0, + "total_bytes": data.get("total_bytes") or 0, + } + if stage == "done" or child_status == "done": + return + emit_parallel_progress(stage, data.get("message") or f"Processing {filename}") + + return build_missing_local_metadata( + models=[model], + force_rescan=False, + worker_count=1, + progress_callback=child_progress, + is_cancelled=is_cancelled, + ) + + _emit( + progress_callback, + { + "stage": "scanning", + "message": f"Found {total} local model files.", + "current": 0, + "total": total, + "percent": 0, + "worker_count": worker_count, + "cpu_count": cpu_count, + }, + ) + + executor = ThreadPoolExecutor(max_workers=worker_count) + futures = {executor.submit(run_one_model, model): model for model in model_items} + try: + for future in as_completed(futures): + model = futures[future] + model_path = str(model.get("path") or "").strip() + filename = get_filename_from_path(model_path) + model_key = _model_identity_key(model) or model_path + try: + item_result = future.result() + except CancelledError: + item_result = {"success": False, "cancelled": True} + except Exception as exc: + log.warning(f"Could not build local metadata for {model_path}: {exc}", exc_info=True) + item_result = { + "success": False, + "errors": [ + { + "filename": filename, + "relative_path": model.get("relative_path") or filename, + "model_path": model_path, + "metadata_path": get_metadata_sidecar_path(model_path), + "message": str(exc) or "Metadata build failed.", + } + ], + } + history_items = item_result.get("history") or _history_items_for_model(model, item_result) + if "history" not in item_result: + item_result["history"] = history_items + + with state_lock: + completed += 1 + active_models.pop(model_key, None) + if item_result.get("cancelled"): + cancelled = True + _merge_build_counts(counts, item_result) + + emit_parallel_progress( + "model_done", + f"Processed {filename}", + { + "current_model": filename, + "current_path": model_path, + "current": completed, + "history_items": history_items, + }, + ) + + if is_cancelled and is_cancelled(): + cancelled = True + for pending in futures: + if not pending.done(): + pending.cancel() + finally: + executor.shutdown(wait=True, cancel_futures=True) + + result = _build_result_payload( + counts, + success=not cancelled, + cancelled=cancelled, + total_models=total, + worker_count=worker_count, + cpu_count=cpu_count, + ) + _emit( + progress_callback, + { + "stage": "cancelled" if cancelled else "done", + "message": "Metadata build cancelled" if cancelled else "Local metadata build completed.", + "status": "cancelled" if cancelled else "done", + "current": completed, + "total": total, + "percent": 100, + **result, + }, + ) + return result + + +def build_missing_local_metadata( + models: Optional[List[Dict[str, Any]]] = None, + *, + force_rescan: bool = True, + worker_count: Optional[int] = None, + progress_callback: Optional[ProgressCallback] = None, + is_cancelled: Optional[CancelCallback] = None, +) -> Dict[str, Any]: + """ + Create missing sidecar metadata files and add missing SHA256 values. + + Existing sidecars are preserved and only filled with local technical values + when missing. No remote metadata providers are used. + """ + if models is None: + from .scanner import get_model_files + + models = get_model_files(force_rescan=force_rescan) + + all_models = _dedupe_models(models or []) + model_items = [ + model for model in all_models + if _is_model_file_path(str(model.get("path") or "").strip()) + ] + total = len(model_items) + resolved_worker_count, cpu_count = _resolve_worker_count(total, worker_count) + if resolved_worker_count > 1: + return _build_missing_local_metadata_parallel( + model_items, + worker_count=resolved_worker_count, + cpu_count=cpu_count, + progress_callback=progress_callback, + is_cancelled=is_cancelled, + ) + + scanned_models = 0 + created_metadata = 0 + updated_metadata = 0 + skipped_complete = 0 + header_hashes = 0 + calculated_hashes = 0 + header_metadata_count = 0 + invalid_metadata = 0 + errors: List[Dict[str, Any]] = [] + updated: List[Dict[str, Any]] = [] + history: List[Dict[str, Any]] = [] + + _emit( + progress_callback, + { + "stage": "scanning", + "message": f"Found {total} local model files.", + "current": 0, + "total": total, + "percent": 0, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + }, + ) + + for index, model in enumerate(model_items, start=1): + if is_cancelled and is_cancelled(): + _emit( + progress_callback, + { + "stage": "cancelled", + "message": "Metadata build cancelled", + "status": "cancelled", + "current": index - 1, + "total": total, + "percent": _progress_percent(index, total), + }, + ) + return { + "success": False, + "cancelled": True, + "scanned_models": scanned_models, + "total_models": total, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "header_hashes": header_hashes, + "calculated_hashes": calculated_hashes, + "header_metadata_count": header_metadata_count, + "invalid_metadata": invalid_metadata, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "history": history[:500], + "history_count": len(history), + } + + model_path = str(model.get("path") or "").strip() + filename = get_filename_from_path(model_path) + relative_path = model.get("relative_path") or filename + metadata_path, metadata_exists = _select_metadata_path(model_path) + scanned_models += 1 + + _emit( + progress_callback, + { + "stage": "header", + "message": f"Reading local metadata: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 0.1), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + }, + ) + + try: + file_size = os.path.getsize(model_path) + existing_raw = read_json_safe(metadata_path, {}) if metadata_exists else {} + if not isinstance(existing_raw, dict): + existing_raw = {} + invalid_metadata += 1 + + existing_sha256 = extract_sha256_from_metadata(existing_raw) + if metadata_exists and existing_sha256: + skipped_complete += 1 + history_item = { + "action": "skipped", + "filename": filename, + "relative_path": relative_path, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + "size": file_size, + "size_label": format_size_bytes(file_size), + "sha256": existing_sha256, + "sha256_source": "existing_metadata", + "message": "Already had SHA256", + } + history.append(history_item) + _emit( + progress_callback, + { + "stage": "model_done", + "message": f"Metadata already has SHA256: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 1.0), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "calculated_hashes": calculated_hashes, + "header_hashes": header_hashes, + "error_count": len(errors), + "history_items": [history_item], + }, + ) + continue + + header_metadata = extract_safetensors_header_metadata(model_path) + header_snapshot = extract_local_header_snapshot(model_path) + if header_metadata or header_snapshot: + header_metadata_count += 1 + + sha256 = existing_sha256 or normalize_sha256(header_metadata.get("sha256")) + sha256_source = "existing_metadata" if existing_sha256 else "" + if sha256 and not existing_sha256: + sha256_source = str(header_metadata.get("sha256_source") or "safetensors_header") + header_hashes += 1 + + if not sha256: + _emit( + progress_callback, + { + "stage": "hashing", + "message": f"Calculating SHA256: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 0.35), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "bytes_read": 0, + "total_bytes": file_size, + }, + ) + last_progress_at = [0.0] + + def on_hash_progress(bytes_read: int, total_bytes: int) -> None: + now = time.time() + if now - last_progress_at[0] < 0.2 and bytes_read < total_bytes: + return + ratio = 0.35 + if total_bytes > 0: + ratio = 0.35 + (min(1.0, bytes_read / total_bytes) * 0.55) + _emit( + progress_callback, + { + "stage": "hashing", + "message": f"Calculating SHA256: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, ratio), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "bytes_read": bytes_read, + "total_bytes": total_bytes, + }, + ) + last_progress_at[0] = now + + hash_source = [""] + + def on_hash_source(source: str) -> None: + hash_source[0] = source or "file" + + sha256 = calculate_file_sha256( + model_path, + chunk_size=1024 * 1024 * 4, + on_progress=on_hash_progress, + is_cancelled=is_cancelled, + use_safetensors_header=True, + on_hash_source=on_hash_source, + ) or "" + sha256 = normalize_sha256(sha256) + sha256_source = hash_source[0] or "file" + if sha256_source == "safetensors_header": + header_hashes += 1 + elif sha256: + calculated_hashes += 1 + + _emit( + progress_callback, + { + "stage": "writing", + "message": f"Writing metadata: {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 0.95), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + }, + ) + + payload, changed_fields = _build_local_metadata_payload( + existing=existing_raw, + model=model, + model_path=model_path, + metadata_path=metadata_path, + file_size=file_size, + header_metadata=header_metadata, + header_snapshot=header_snapshot, + sha256=sha256, + sha256_source=sha256_source, + ) + + should_write = not metadata_exists or bool(changed_fields) or not existing_sha256 + history_item = None + if should_write: + write_json_atomic(metadata_path, payload, indent=2) + action = "created" if not metadata_exists else "updated" + if metadata_exists: + updated_metadata += 1 + else: + created_metadata += 1 + history_item = { + "action": action, + "filename": filename, + "relative_path": relative_path, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + "size": file_size, + "size_label": format_size_bytes(file_size), + "sha256": sha256, + "sha256_source": sha256_source, + "changed_fields": changed_fields, + } + updated.append(history_item) + history.append(history_item) + else: + skipped_complete += 1 + history_item = { + "action": "skipped", + "filename": filename, + "relative_path": relative_path, + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + "model_path": model_path, + "metadata_path": metadata_path, + "size": file_size, + "size_label": format_size_bytes(file_size), + "sha256": sha256, + "sha256_source": sha256_source, + "message": "No changes needed", + } + history.append(history_item) + + _emit( + progress_callback, + { + "stage": "model_done", + "message": f"Processed {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 1.0), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "calculated_hashes": calculated_hashes, + "header_hashes": header_hashes, + "error_count": len(errors), + "history_items": [history_item] if history_item else [], + }, + ) + except HashCalculationCancelled: + _emit( + progress_callback, + { + "stage": "cancelled", + "message": "Metadata build cancelled", + "status": "cancelled", + "current": index - 1, + "total": total, + "percent": _progress_percent(index, total), + "current_model": filename, + "current_path": model_path, + }, + ) + return { + "success": False, + "cancelled": True, + "scanned_models": scanned_models, + "total_models": total, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "header_hashes": header_hashes, + "calculated_hashes": calculated_hashes, + "header_metadata_count": header_metadata_count, + "invalid_metadata": invalid_metadata, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "history": history[:500], + "history_count": len(history), + } + except Exception as exc: + log.warning(f"Could not build local metadata for {model_path}: {exc}", exc_info=True) + error_item = { + "filename": filename, + "relative_path": relative_path, + "model_path": model_path, + "metadata_path": metadata_path, + "message": str(exc) or "Metadata build failed.", + } + errors.append(error_item) + history_item = { + **error_item, + "action": "error", + "category": model.get("category") or "", + "base_directory": model.get("base_directory") or "", + } + history.append(history_item) + _emit( + progress_callback, + { + "stage": "model_done", + "message": f"Error processing {filename}", + "current": index, + "total": total, + "percent": _progress_percent(index, total, 1.0), + "current_model": filename, + "current_path": model_path, + "metadata_path": metadata_path, + "error_count": len(errors), + "history_items": [history_item], + }, + ) + + result = { + "success": True, + "cancelled": False, + "scanned_models": scanned_models, + "total_models": total, + "created_metadata": created_metadata, + "updated_metadata": updated_metadata, + "skipped_complete": skipped_complete, + "header_hashes": header_hashes, + "calculated_hashes": calculated_hashes, + "header_metadata_count": header_metadata_count, + "invalid_metadata": invalid_metadata, + "worker_count": resolved_worker_count, + "cpu_count": cpu_count, + "error_count": len(errors), + "errors": errors[:50], + "updated": updated[:200], + "updated_count": len(updated), + "history": history[:500], + "history_count": len(history), + } + _emit( + progress_callback, + { + "stage": "done", + "message": "Local metadata build completed.", + "status": "done", + "current": total, + "total": total, + "percent": 100, + **result, + }, + ) + log.info( + "Local metadata build finished: " + f"models={scanned_models}, created={created_metadata}, updated={updated_metadata}, " + f"skipped={skipped_complete}, header_hashes={header_hashes}, " + f"calculated_hashes={calculated_hashes}, errors={len(errors)}" + ) + return result diff --git a/web/css/resolver-main.css b/web/css/resolver-main.css index b65eaa8..1ae8f84 100644 --- a/web/css/resolver-main.css +++ b/web/css/resolver-main.css @@ -6344,6 +6344,10 @@ a.mr-image-preview-resource-name:hover { align-content: stretch; } +.mr-options-section-fill .mr-options-build-stack { + grid-template-rows: auto auto auto auto auto minmax(0, 1fr); +} + .mr-options-panel-title { margin: 0 0 6px 0; font-size: 16px; @@ -6451,6 +6455,244 @@ a.mr-image-preview-resource-name:hover { color: var(--mr-color-error-light); } +.mr-options-inline-hint { + display: block; + margin-top: 3px; +} + +.mr-options-build-tabs { + align-self: flex-start; + align-items: flex-end; + width: fit-content; + height: 36px; + min-height: 36px; + margin-top: 12px; + overflow: visible; + --mr-tab-active-bg: rgba(20, 20, 24, 0.96); +} + +.mr-options-build-tabs .mr-options-build-tab { + gap: 5px; + min-width: 94px; + padding-right: 18px; + padding-left: 18px; +} + +.mr-options-build-tabs .mr-options-build-tab:not(.mr-tab-active):hover, +.mr-options-build-tabs .mr-options-build-tab:not(.mr-tab-active):focus, +.mr-options-build-tabs .mr-options-build-tab:not(.mr-tab-active):active, +.mr-options-build-tabs .mr-options-build-tab.mr-tab-active { + padding-right: 18px !important; + padding-left: 18px !important; +} + +.mr-options-build-tab-count { + position: relative; + z-index: 2; + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 18px; + height: 18px; + padding: 0 6px; + border-radius: 999px; + color: var(--mr-text); + background: rgba(255, 255, 255, 0.1); + font-size: 11px; + font-weight: 700; + line-height: 1; + pointer-events: none; +} + +.mr-options-build-tab.mr-tab-active .mr-options-build-tab-count { + background: rgba(255, 255, 255, 0.14); +} + +.mr-options-build-tab-panel { + min-width: 0; +} + +.mr-options-build-tab-panel.is-hidden { + display: none; +} + +.mr-options-build-progress { + display: grid; + gap: 8px; + margin-top: 10px; + min-width: 0; + padding: 10px; + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 6px; + background: rgba(0, 0, 0, 0.14); +} + +.mr-options-build-progress[hidden] { + display: none; +} + +.mr-options-build-progress-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + color: var(--mr-text-secondary); + font-size: 12px; +} + +.mr-options-build-progress-head strong { + color: var(--mr-text); + font-size: 12px; +} + +.mr-options-build-progress-bar { + height: 7px; + overflow: hidden; + border-radius: 999px; + background: rgba(255, 255, 255, 0.08); +} + +.mr-options-build-progress-bar span { + display: block; + width: 0; + height: 100%; + border-radius: inherit; + background: var(--mr-accent); + transition: width 0.2s ease; +} + +.mr-options-build-current { + display: grid; + gap: 3px; + min-width: 0; + color: var(--mr-text-secondary); + font-size: 12px; + line-height: 1.35; +} + +.mr-options-build-current span, +.mr-options-build-current code { + overflow-wrap: anywhere; +} + +.mr-options-build-current code { + color: var(--mr-text-muted); + font-family: ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace; + font-size: 11px; +} + +.mr-options-build-worker-list { + display: grid; + gap: 8px; + max-height: 260px; + min-height: 0; + overflow: auto; + padding-right: 2px; +} + +.mr-options-build-worker-list[hidden] { + display: none; +} + +.mr-options-build-worker { + display: grid; + gap: 5px; + min-width: 0; + padding: 8px 9px; + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 6px; + background: rgba(255, 255, 255, 0.035); +} + +.mr-options-build-worker-head, +.mr-options-build-worker-meta { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + min-width: 0; + color: var(--mr-text-secondary); + font-size: 11px; +} + +.mr-options-build-worker-head strong { + color: var(--mr-text); + font-size: 11px; +} + +.mr-options-build-worker-name { + min-width: 0; + overflow: hidden; + color: var(--mr-text); + font-size: 12px; + font-weight: 700; + text-overflow: ellipsis; + white-space: nowrap; +} + +.mr-options-build-worker code { + min-width: 0; + overflow: hidden; + color: var(--mr-text-muted); + font-family: ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace; + font-size: 10.5px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.mr-options-build-worker-bar { + height: 5px; + overflow: hidden; + border-radius: 999px; + background: rgba(255, 255, 255, 0.08); +} + +.mr-options-build-worker-bar span { + display: block; + width: 0; + height: 100%; + border-radius: inherit; + background: var(--mr-color-success); + transition: width 0.2s ease; +} + +.mr-options-history-action { + display: inline-flex; + align-items: center; + min-height: 20px; + padding: 2px 7px; + border-radius: 999px; + color: var(--mr-text); + background: rgba(255, 255, 255, 0.08); + font-size: 11px; + font-weight: 700; +} + +.mr-options-history-action.is-created { + color: var(--mr-color-success); + background: rgba(34, 197, 94, 0.1); +} + +.mr-options-history-action.is-updated { + color: var(--mr-accent); + background: rgba(96, 165, 250, 0.1); +} + +.mr-options-history-action.is-skipped, +.mr-options-history-action.is-checked { + color: var(--mr-text-secondary); +} + +.mr-options-history-action.is-error { + color: var(--mr-color-error-light); + background: rgba(248, 113, 113, 0.1); +} + +.mr-options-history-action.is-cancelled { + color: var(--mr-text-secondary); + background: rgba(255, 255, 255, 0.06); +} + .mr-options-audit-results { margin-top: 12px; min-width: 0; @@ -6533,6 +6775,40 @@ a.mr-image-preview-resource-name:hover { width: 12%; } +.mr-options-history-table { + min-width: 900px; +} + +.mr-options-history-table th:nth-child(1), +.mr-options-history-table td:nth-child(1) { + width: 30%; +} + +.mr-options-history-table th:nth-child(2), +.mr-options-history-table td:nth-child(2) { + width: 12%; +} + +.mr-options-history-table th:nth-child(3), +.mr-options-history-table td:nth-child(3) { + width: 11%; +} + +.mr-options-history-table th:nth-child(4), +.mr-options-history-table td:nth-child(4) { + width: 13%; +} + +.mr-options-history-table th:nth-child(5), +.mr-options-history-table td:nth-child(5) { + width: 8%; +} + +.mr-options-history-table th:nth-child(6), +.mr-options-history-table td:nth-child(6) { + width: 26%; +} + .mr-options-audit-model { color: var(--mr-text); font-weight: 700; diff --git a/web/resolver/views/options_methods.js b/web/resolver/views/options_methods.js index e9c659b..b2d1414 100644 --- a/web/resolver/views/options_methods.js +++ b/web/resolver/views/options_methods.js @@ -243,6 +243,13 @@ export const optionsMethods = { 07 +
@@ -253,7 +260,7 @@ export const optionsMethods = { Maintenance - 08 + 09
@@ -773,6 +780,92 @@ export const optionsMethods = { + @@ -861,9 +954,39 @@ export const optionsMethods = { const metadataSizeMismatchesEl = this.contentElement.querySelector('#mr-options-metadata-size-mismatches'); const metadataSizeMissingEl = this.contentElement.querySelector('#mr-options-metadata-size-missing'); const metadataSizeErrorsEl = this.contentElement.querySelector('#mr-options-metadata-size-errors'); + const metadataBuildStartBtn = this.contentElement.querySelector('#mr-options-metadata-build-start'); + const metadataBuildCancelBtn = this.contentElement.querySelector('#mr-options-metadata-build-cancel'); + const metadataBuildStatus = this.contentElement.querySelector('#mr-options-metadata-build-status'); + const metadataBuildProgress = this.contentElement.querySelector('#mr-options-metadata-build-progress'); + const metadataBuildStageEl = this.contentElement.querySelector('#mr-options-metadata-build-stage'); + const metadataBuildPercentEl = this.contentElement.querySelector('#mr-options-metadata-build-percent'); + const metadataBuildBarEl = this.contentElement.querySelector('#mr-options-metadata-build-bar'); + const metadataBuildCurrentModelEl = this.contentElement.querySelector('#mr-options-metadata-build-current-model'); + const metadataBuildCurrentPathEl = this.contentElement.querySelector('#mr-options-metadata-build-current-path'); + const metadataBuildBytesEl = this.contentElement.querySelector('#mr-options-metadata-build-bytes'); + const metadataBuildWorkerListEl = this.contentElement.querySelector('#mr-options-metadata-build-worker-list'); + const metadataBuildResults = this.contentElement.querySelector('#mr-options-metadata-build-results'); + const metadataBuildTabButtons = Array.from(this.contentElement.querySelectorAll('.mr-options-build-tab')); + const metadataBuildProgressPanel = this.contentElement.querySelector('#mr-options-metadata-build-progress-panel'); + const metadataBuildHistoryPanel = this.contentElement.querySelector('#mr-options-metadata-build-history-panel'); + const metadataBuildHistoryCountEl = this.contentElement.querySelector('#mr-options-metadata-build-history-count'); + const metadataBuildCpuEl = this.contentElement.querySelector('#mr-options-metadata-build-cpu'); + const metadataBuildWorkersEl = this.contentElement.querySelector('#mr-options-metadata-build-workers'); + const metadataBuildWorkerInput = this.contentElement.querySelector('#mr-options-metadata-build-worker-count'); + const metadataBuildWorkerHint = this.contentElement.querySelector('#mr-options-metadata-build-worker-hint'); + const metadataBuildScannedEl = this.contentElement.querySelector('#mr-options-metadata-build-scanned'); + const metadataBuildCreatedEl = this.contentElement.querySelector('#mr-options-metadata-build-created'); + const metadataBuildUpdatedEl = this.contentElement.querySelector('#mr-options-metadata-build-updated'); + const metadataBuildSkippedEl = this.contentElement.querySelector('#mr-options-metadata-build-skipped'); + const metadataBuildHashesEl = this.contentElement.querySelector('#mr-options-metadata-build-hashes'); + const metadataBuildErrorsEl = this.contentElement.querySelector('#mr-options-metadata-build-errors'); const optionsMain = this.contentElement.querySelector('.mr-options-main'); const navButtons = Array.from(this.contentElement.querySelectorAll('.mr-options-nav-btn')); const optionSections = Array.from(this.contentElement.querySelectorAll('.mr-options-section')); + if (this.metadataBuildPollTimer) { + window.clearTimeout(this.metadataBuildPollTimer); + this.metadataBuildPollTimer = null; + } const trackedInputs = [ civitaiInput, civitaiSessionInput, @@ -1177,6 +1300,540 @@ export const optionsMethods = { } }; + const setMetadataBuildBusy = (busy) => { + if (metadataBuildStartBtn) metadataBuildStartBtn.disabled = busy; + if (metadataBuildCancelBtn) { + metadataBuildCancelBtn.hidden = !busy; + metadataBuildCancelBtn.disabled = false; + } + }; + + const setMetadataBuildStatus = (text, mode = '') => { + if (!metadataBuildStatus) return; + metadataBuildStatus.textContent = text; + setStatusMode(metadataBuildStatus, mode); + }; + + const setMetadataBuildSummaryValue = (element, value, mode = '') => { + if (!element) return; + element.textContent = formatAuditNumber(value); + setStatusMode(element, mode); + }; + + const setMetadataBuildTab = (tabName = 'progress') => { + const activeTab = tabName === 'history' ? 'history' : 'progress'; + metadataBuildTabButtons.forEach((button) => { + const isActive = button.dataset.buildTab === activeTab; + button.classList.toggle('is-active', isActive); + button.classList.toggle('mr-tab-active', isActive); + button.setAttribute('aria-selected', isActive ? 'true' : 'false'); + }); + metadataBuildProgressPanel?.classList.toggle('is-hidden', activeTab !== 'progress'); + metadataBuildHistoryPanel?.classList.toggle('is-hidden', activeTab !== 'history'); + this.metadataBuildActiveTab = activeTab; + }; + + const getMetadataBuildHistoryKey = (item = {}) => { + const path = item.model_path || item.path || item.metadata_path || item.filename || ''; + const action = item.action || ''; + const sha = item.sha256 || ''; + return `${action}::${path}::${sha}`; + }; + + const appendMetadataBuildHistory = (data = {}) => { + if (!this.metadataBuildHistory) this.metadataBuildHistory = []; + if (!this.metadataBuildHistoryKeys) this.metadataBuildHistoryKeys = new Set(); + const source = data?.result && typeof data.result === 'object' ? data.result : data; + const items = []; + if (Array.isArray(data.history_items)) items.push(...data.history_items); + if (Array.isArray(source.history)) items.push(...source.history); + if (!items.length && Array.isArray(source.updated)) items.push(...source.updated); + items.forEach((rawItem) => { + if (!rawItem || typeof rawItem !== 'object') return; + const item = { ...rawItem }; + const key = getMetadataBuildHistoryKey(item); + if (!key || this.metadataBuildHistoryKeys.has(key)) return; + this.metadataBuildHistoryKeys.add(key); + this.metadataBuildHistory.unshift(item); + }); + if (this.metadataBuildHistory.length > 1000) { + this.metadataBuildHistory = this.metadataBuildHistory.slice(0, 1000); + this.metadataBuildHistoryKeys = new Set(this.metadataBuildHistory.map(getMetadataBuildHistoryKey)); + } + if (metadataBuildHistoryCountEl) { + metadataBuildHistoryCountEl.textContent = this.metadataBuildHistory.length.toLocaleString(); + } + }; + + const getMetadataBuildWorkerBounds = () => { + const capabilities = this.metadataBuildCapabilities || {}; + const min = Math.max(1, Number(capabilities.min_worker_count || 1)); + const max = Math.max(min, Number(capabilities.max_worker_count || 64)); + return { min, max }; + }; + + const normalizeMetadataBuildWorkerCount = () => { + const { min, max } = getMetadataBuildWorkerBounds(); + const fallback = Number(this.metadataBuildCapabilities?.default_worker_count || metadataBuildWorkerInput?.value || 1); + const raw = Number(metadataBuildWorkerInput?.value || fallback || 1); + const workerCount = Math.max(min, Math.min(max, Number.isFinite(raw) ? Math.round(raw) : fallback)); + if (metadataBuildWorkerInput) { + metadataBuildWorkerInput.min = String(min); + metadataBuildWorkerInput.max = String(max); + metadataBuildWorkerInput.value = String(workerCount); + } + return workerCount; + }; + + const renderMetadataBuildCapabilities = (data = {}) => { + this.metadataBuildCapabilities = data && typeof data === 'object' ? data : {}; + const cpuCount = Number(this.metadataBuildCapabilities.cpu_count || navigator.hardwareConcurrency || 1); + const defaultWorkers = Number(this.metadataBuildCapabilities.default_worker_count || Math.min(cpuCount, 4) || 1); + const maxWorkers = Number(this.metadataBuildCapabilities.max_worker_count || 64); + if (metadataBuildCpuEl) metadataBuildCpuEl.textContent = cpuCount.toLocaleString(); + if (metadataBuildWorkerInput && !metadataBuildWorkerInput.dataset.userEdited) { + metadataBuildWorkerInput.value = String(defaultWorkers); + } + normalizeMetadataBuildWorkerCount(); + if (metadataBuildWorkerHint) { + metadataBuildWorkerHint.textContent = `CPU cores: ${cpuCount.toLocaleString()}. Max workers: ${maxWorkers.toLocaleString()}.`; + } + if (metadataBuildWorkersEl) metadataBuildWorkersEl.textContent = normalizeMetadataBuildWorkerCount().toLocaleString(); + }; + + const loadMetadataBuildCapabilities = async () => { + try { + const data = await this.fetchJson( + '/model_resolver/metadata-build/capabilities', + { silent: true }, + 'Load metadata build capabilities' + ); + renderMetadataBuildCapabilities(data); + } catch (error) { + renderMetadataBuildCapabilities({ + cpu_count: navigator.hardwareConcurrency || 1, + default_worker_count: Math.min(navigator.hardwareConcurrency || 1, 4), + min_worker_count: 1, + max_worker_count: 64 + }); + } + }; + + const updateMetadataBuildSummary = (data = {}) => { + const source = data?.result && typeof data.result === 'object' ? data.result : data; + const created = Number(source.created_metadata || 0); + const updated = Number(source.updated_metadata || 0); + const calculatedHashes = Number(source.calculated_hashes || 0); + const errorCount = Number(source.error_count || 0); + if (metadataBuildCpuEl && source.cpu_count) { + metadataBuildCpuEl.textContent = Number(source.cpu_count || 0).toLocaleString(); + } + if (metadataBuildWorkersEl) { + const activeWorkers = Number(source.active_worker_count ?? source.worker_count ?? normalizeMetadataBuildWorkerCount()); + metadataBuildWorkersEl.textContent = Number.isFinite(activeWorkers) ? activeWorkers.toLocaleString() : '0'; + } + setMetadataBuildSummaryValue(metadataBuildScannedEl, source.scanned_models || source.current || 0); + setMetadataBuildSummaryValue(metadataBuildCreatedEl, created, created > 0 ? 'is-valid' : ''); + setMetadataBuildSummaryValue(metadataBuildUpdatedEl, updated, updated > 0 ? 'is-valid' : ''); + setMetadataBuildSummaryValue(metadataBuildSkippedEl, source.skipped_complete || 0); + setMetadataBuildSummaryValue(metadataBuildHashesEl, calculatedHashes, calculatedHashes > 0 ? 'is-pending' : ''); + setMetadataBuildSummaryValue(metadataBuildErrorsEl, errorCount, errorCount > 0 ? 'is-invalid' : ''); + }; + + const metadataBuildStageLabels = { + queued: 'Queued', + scanning: 'Scanning', + header: 'Reading header', + hashing: 'Calculating SHA256', + writing: 'Writing metadata', + model_done: 'Processed', + done: 'Done', + cancelled: 'Cancelled', + error: 'Error' + }; + + const getMetadataBuildWorkerPercent = (item = {}) => { + const direct = Number(item.percent); + if (Number.isFinite(direct)) { + return Math.max(0, Math.min(100, direct)); + } + const bytesRead = Number(item.bytes_read || 0); + const totalBytes = Number(item.total_bytes || 0); + if (totalBytes > 0) { + return Math.max(0, Math.min(100, (bytesRead / totalBytes) * 100)); + } + return 0; + }; + + const renderMetadataBuildWorkers = (data = {}, activeModels = []) => { + if (!metadataBuildWorkerListEl) return; + const workerItems = activeModels.length + ? activeModels + : (data.current_model ? [{ + filename: data.current_model, + path: data.current_path || data.metadata_path || '', + stage: data.stage || '', + percent: data.percent || 0, + bytes_read: data.bytes_read || 0, + total_bytes: data.total_bytes || 0 + }] : []); + + if (!workerItems.length) { + metadataBuildWorkerListEl.innerHTML = ''; + metadataBuildWorkerListEl.hidden = true; + return; + } + + metadataBuildWorkerListEl.hidden = false; + metadataBuildWorkerListEl.innerHTML = workerItems.map((item, index) => { + const workerPercent = getMetadataBuildWorkerPercent(item); + const stageLabel = metadataBuildStageLabels[item.stage] || item.stage || 'Running'; + const bytesRead = Number(item.bytes_read || 0); + const totalBytes = Number(item.total_bytes || 0); + const bytesLabel = totalBytes > 0 + ? `${this.formatBytes(bytesRead)} / ${this.formatBytes(totalBytes)}` + : ''; + const filename = item.filename || 'Model'; + const path = item.path || ''; + return ` +
+
+ Worker ${index + 1} + ${Math.round(workerPercent)}% +
+
${this.escapeHtml(filename)}
+
+ ${this.escapeHtml(stageLabel)} + ${bytesLabel ? `${this.escapeHtml(bytesLabel)}` : ''} +
+ ${path ? `${this.escapeHtml(path)}` : ''} + +
+ `; + }).join(''); + }; + + const updateMetadataBuildProgressView = (data = {}) => { + if (metadataBuildProgress) metadataBuildProgress.hidden = false; + const stage = String(data.stage || data.status || '').trim(); + const percent = Math.max(0, Math.min(100, Number(data.percent || 0))); + if (metadataBuildStageEl) metadataBuildStageEl.textContent = metadataBuildStageLabels[stage] || stage || 'Running'; + if (metadataBuildPercentEl) metadataBuildPercentEl.textContent = `${Math.round(percent)}%`; + if (metadataBuildBarEl) metadataBuildBarEl.style.width = `${percent}%`; + + const current = Number(data.current || 0); + const total = Number(data.total || data.total_models || 0); + const activeModels = Array.isArray(data.active_models) ? data.active_models : []; + const currentModel = data.current_model || data.filename || ''; + const suffix = total > 0 ? ` (${current.toLocaleString()} / ${total.toLocaleString()})` : ''; + if (metadataBuildCurrentModelEl) { + if (activeModels.length) { + metadataBuildCurrentModelEl.textContent = `${activeModels.length.toLocaleString()} active worker${activeModels.length === 1 ? '' : 's'}${suffix}`; + } else { + metadataBuildCurrentModelEl.textContent = currentModel + ? `${currentModel}${suffix}` + : (data.message || `Preparing${suffix}`); + } + } + if (metadataBuildCurrentPathEl) { + const pathText = activeModels.length + ? '' + : (data.current_path || data.metadata_path || ''); + metadataBuildCurrentPathEl.textContent = pathText; + metadataBuildCurrentPathEl.hidden = !pathText; + } + + const bytesRead = Number(data.bytes_read || 0); + const totalBytes = Number(data.total_bytes || 0); + if (metadataBuildBytesEl) { + metadataBuildBytesEl.textContent = activeModels.length + ? '' + : totalBytes > 0 + ? `${this.formatBytes(bytesRead)} / ${this.formatBytes(totalBytes)}` + : ''; + } + renderMetadataBuildWorkers(data, activeModels); + }; + + const renderMetadataBuildResults = (data = null) => { + if (!metadataBuildResults) return; + const source = data?.result && typeof data.result === 'object' ? data.result : data; + const historyItems = Array.isArray(this.metadataBuildHistory) + ? this.metadataBuildHistory + : []; + if (!source && !historyItems.length) { + metadataBuildResults.hidden = true; + metadataBuildResults.innerHTML = ''; + return; + } + + const fallbackItems = source && Array.isArray(source.history) + ? source.history + : (source && Array.isArray(source.updated) ? source.updated : []); + const displayItems = historyItems.length ? historyItems : fallbackItems; + const errors = Array.isArray(source?.errors) ? source.errors : []; + const visibleItems = displayItems.slice(0, 500); + let html = ''; + + if (visibleItems.length) { + const rows = visibleItems.map((item) => { + const modelLabel = item.relative_path || item.filename || item.model_path || 'Model'; + const modelPath = item.model_path || ''; + const metadataPath = item.metadata_path || ''; + const shaSource = item.sha256_source || ''; + const shaLabel = item.sha256 ? `${String(item.sha256).slice(0, 12)}...` : 'Pending'; + const changedFieldList = Array.isArray(item.changed_fields) + ? item.changed_fields.filter(Boolean) + : []; + const changedFields = changedFieldList.join(', '); + const changedFieldsSummary = changedFieldList.length + ? `${changedFieldList.length} field${changedFieldList.length === 1 ? '' : 's'} changed` + : ''; + const changedFieldsTitle = changedFields ? ` title="${this.escapeHtml(changedFields)}"` : ''; + const action = item.action || 'checked'; + const actionClass = String(action).toLowerCase().replace(/[^a-z0-9_-]+/g, '-'); + const message = item.message || changedFieldsSummary; + const buildContext = modelPath ? { + context_scope: 'local_model', + open_folder_label: 'Show File in Folder', + name: item.filename || modelLabel, + filename: item.filename || modelLabel, + relative_path: item.relative_path || modelLabel, + path: modelPath, + resolved_path: modelPath, + open_path: modelPath, + folder_path: item.base_directory || '', + category: item.category || '', + metadata_path: metadataPath, + size: item.size || 0, + file_size: item.size || 0, + sha256: item.sha256 || '', + context_source: 'metadata_builder' + } : null; + const contextAttrs = buildContext + ? this.getContextMenuAttrs(buildContext, 'Right-click for model options') + : ''; + return ` + + +
${this.escapeHtml(modelLabel)}
+
${this.escapeHtml(modelPath)}
+ + + ${this.escapeHtml(action)} + ${message ? `
${this.escapeHtml(message)}
` : ''} + + ${this.escapeHtml(item.category || '')} + +
${this.escapeHtml(shaLabel)}
+
${this.escapeHtml(shaSource)}
+ + ${this.escapeHtml(item.size_label || this.formatBytes(item.size || 0))} + +
${this.escapeHtml(metadataPath)}
+ + + `; + }).join(''); + html += ` +
+ + + + + + + + + + + + ${rows} +
ModelActionFolderSHA256SizeMetadata file
+
+ `; + if (displayItems.length > visibleItems.length) { + html += `
Showing latest ${visibleItems.length.toLocaleString()} of ${displayItems.length.toLocaleString()} history items.
`; + } + } else if (source?.cancelled) { + html += '
Metadata build was cancelled before any history item was captured.
'; + } else { + html += '
No metadata history captured yet.
'; + } + + if (errors.length) { + const errorRows = errors.slice(0, 10).map((item) => { + const targetPath = item.metadata_path || item.model_path || ''; + return `
  • ${this.escapeHtml(item.message || 'Error')}${this.escapeHtml(targetPath)}
    • `; + }).join(''); + html += ` +
    +
    Write errors
    + +
    + `; + } + + metadataBuildResults.innerHTML = html; + metadataBuildResults.hidden = false; + }; + + const stopMetadataBuildPolling = () => { + if (this.metadataBuildPollTimer) { + window.clearTimeout(this.metadataBuildPollTimer); + this.metadataBuildPollTimer = null; + } + }; + + const isMetadataBuildTerminal = (data = {}) => { + const statusValue = String(data.status || '').toLowerCase(); + return ['done', 'error', 'cancelled'].includes(statusValue); + }; + + const getMetadataBuildProgressKey = (data = {}) => ( + String(data.progress_id || data?.result?.progress_id || this.metadataBuildProgressId || '').trim() + ); + + const finishMetadataBuild = (data = {}) => { + const progressKey = getMetadataBuildProgressKey(data); + if (progressKey && this.metadataBuildFinishedProgressId === progressKey) { + return; + } + if (progressKey) { + this.metadataBuildFinishedProgressId = progressKey; + } + + appendMetadataBuildHistory(data); + const source = data?.result && typeof data.result === 'object' ? data.result : data; + setMetadataBuildBusy(false); + renderMetadataBuildResults(source); + + if (data.status === 'cancelled' || source.cancelled) { + setMetadataBuildStatus(`Metadata build cancelled after ${Number(source.scanned_models || data.current || 0).toLocaleString()} model${Number(source.scanned_models || data.current || 0) === 1 ? '' : 's'}.`, 'is-pending'); + this.showNotification('Metadata build cancelled', 'warning'); + return; + } + + if (data.status === 'error') { + setMetadataBuildStatus(data.message || 'Metadata build failed.', 'is-invalid'); + this.showNotification('Metadata build failed', 'error'); + return; + } + + const created = Number(source.created_metadata || 0); + const updated = Number(source.updated_metadata || 0); + const errorCount = Number(source.error_count || 0); + const calculatedHashes = Number(source.calculated_hashes || 0); + const headerHashes = Number(source.header_hashes || 0); + const changed = created + updated; + const hashText = ` ${calculatedHashes.toLocaleString()} full SHA256 hash${calculatedHashes === 1 ? '' : 'es'} calculated, ${headerHashes.toLocaleString()} read from headers.`; + if (errorCount > 0) { + setMetadataBuildStatus(`Metadata build finished with ${errorCount.toLocaleString()} error${errorCount === 1 ? '' : 's'}.${hashText}`, 'is-invalid'); + this.showNotification('Metadata build finished with errors', 'warning'); + } else if (changed > 0) { + setMetadataBuildStatus(`Created ${created.toLocaleString()} and updated ${updated.toLocaleString()} metadata file${changed === 1 ? '' : 's'}.${hashText}`, 'is-valid'); + this.showNotification('Local metadata updated', 'success'); + } else { + setMetadataBuildStatus(`Checked ${Number(source.scanned_models || 0).toLocaleString()} model${Number(source.scanned_models || 0) === 1 ? '' : 's'}. All existing metadata already had SHA256.`, 'is-valid'); + this.showNotification('Local metadata already complete', 'success'); + } + }; + + const pollMetadataBuildProgress = async () => { + const progressId = this.metadataBuildProgressId; + if (!progressId) return; + try { + const data = await this.fetchJson( + `/model_resolver/metadata-build/progress/${encodeURIComponent(progressId)}`, + { silent: true }, + 'Metadata build progress' + ); + appendMetadataBuildHistory(data); + updateMetadataBuildSummary(data); + updateMetadataBuildProgressView(data); + renderMetadataBuildResults(data); + if (isMetadataBuildTerminal(data)) { + stopMetadataBuildPolling(); + finishMetadataBuild(data); + if (this.metadataBuildProgressId === progressId) { + this.metadataBuildProgressId = ''; + } + return; + } + stopMetadataBuildPolling(); + this.metadataBuildPollTimer = window.setTimeout(() => { + pollMetadataBuildProgress(); + }, 700); + } catch (error) { + stopMetadataBuildPolling(); + this.metadataBuildProgressId = ''; + setMetadataBuildBusy(false); + setMetadataBuildStatus(error.message || 'Metadata build progress failed.', 'is-invalid'); + } + }; + + const runMetadataBuild = async () => { + stopMetadataBuildPolling(); + this.metadataBuildProgressId = ''; + this.metadataBuildFinishedProgressId = ''; + this.metadataBuildHistory = []; + this.metadataBuildHistoryKeys = new Set(); + if (metadataBuildHistoryCountEl) metadataBuildHistoryCountEl.textContent = '0'; + setMetadataBuildBusy(true); + updateMetadataBuildSummary({}); + renderMetadataBuildResults(null); + setMetadataBuildTab('progress'); + updateMetadataBuildProgressView({ + stage: 'queued', + message: 'Preparing local metadata build...', + percent: 0, + current: 0, + total: 0 + }); + setMetadataBuildStatus('Preparing local metadata build...', 'is-pending'); + try { + const workerCount = normalizeMetadataBuildWorkerCount(); + const start = await this.fetchJson('/model_resolver/metadata-build/start', { + method: 'POST', + body: JSON.stringify({ + force_rescan: true, + worker_count: workerCount + }) + }, 'Start metadata build'); + if (!start?.progress_id) { + throw new Error('Metadata build did not return a progress id.'); + } + this.metadataBuildProgressId = start.progress_id; + setMetadataBuildStatus('Building local metadata...', 'is-pending'); + pollMetadataBuildProgress(); + } catch (error) { + this.metadataBuildProgressId = ''; + setMetadataBuildBusy(false); + setMetadataBuildStatus(error.message || 'Metadata build failed to start.', 'is-invalid'); + } + }; + + const cancelMetadataBuild = async () => { + const progressId = this.metadataBuildProgressId; + if (!progressId) return; + if (metadataBuildCancelBtn) metadataBuildCancelBtn.disabled = true; + setMetadataBuildStatus('Stopping metadata build...', 'is-pending'); + try { + await this.fetchJson( + `/model_resolver/metadata-build/cancel/${encodeURIComponent(progressId)}`, + { method: 'POST', silent: true }, + 'Cancel metadata build' + ); + stopMetadataBuildPolling(); + pollMetadataBuildProgress(); + } catch (error) { + setMetadataBuildStatus(error.message || 'Could not cancel metadata build.', 'is-invalid'); + if (metadataBuildCancelBtn) metadataBuildCancelBtn.disabled = false; + } + }; + const setTokenCheckStatus = (statusEl, text, mode = '') => { if (!statusEl) return; statusEl.textContent = text; @@ -1736,6 +2393,7 @@ export const optionsMethods = { optionsMain?.classList.toggle( 'mr-options-main-fill-section', targetId === 'mr-options-section-metadata-audit' + || targetId === 'mr-options-section-metadata-build' ); setActiveNav(targetId); }; @@ -2132,6 +2790,7 @@ export const optionsMethods = { }; populateDefaultRootSelects(); + loadMetadataBuildCapabilities(); setStatus('Saved only on this machine.'); setVisibleSection('mr-options-section-sources'); @@ -2219,6 +2878,34 @@ export const optionsMethods = { }); } + if (metadataBuildStartBtn) { + metadataBuildStartBtn.addEventListener('click', () => { + runMetadataBuild(); + }); + } + + if (metadataBuildCancelBtn) { + metadataBuildCancelBtn.addEventListener('click', () => { + cancelMetadataBuild(); + }); + } + + metadataBuildTabButtons.forEach((button) => { + button.addEventListener('click', () => { + setMetadataBuildTab(button.dataset.buildTab || 'progress'); + }); + }); + + if (metadataBuildWorkerInput) { + metadataBuildWorkerInput.addEventListener('change', () => { + metadataBuildWorkerInput.dataset.userEdited = 'true'; + const workerCount = normalizeMetadataBuildWorkerCount(); + if (metadataBuildWorkersEl && !this.metadataBuildProgressId) { + metadataBuildWorkersEl.textContent = workerCount.toLocaleString(); + } + }); + } + if (detectTemplatesBtn) { detectTemplatesBtn.addEventListener('click', () => { applyDetectedPathTemplates();