Add metadata size audit for model sidecar metadata
This commit is contained in:
Dariusz L
+23
@@ -187,6 +187,7 @@ class ModelResolverExtension:
|
||||
search_local_matches,
|
||||
)
|
||||
from .core.path_templates import infer_download_path_templates
|
||||
from .core.metadata_audit import audit_metadata_sizes
|
||||
from .core.scanner import get_model_files, invalidate_model_files_cache, find_local_file_path
|
||||
from .core.path_utils import (
|
||||
extract_safetensors_header_sha256,
|
||||
@@ -1119,6 +1120,28 @@ class ModelResolverExtension:
|
||||
models = get_model_files(force_rescan=force_rescan)
|
||||
return web.json_response(models)
|
||||
|
||||
@routes.post("/model_resolver/metadata-size-audit")
|
||||
@json_api_endpoint("metadata-size-audit")
|
||||
async def metadata_size_audit_route(request):
|
||||
"""Check local model .metadata.json sidecars for stale size values."""
|
||||
payload = {}
|
||||
try:
|
||||
if request.can_read_body:
|
||||
payload = await request.json()
|
||||
except Exception:
|
||||
payload = {}
|
||||
if not isinstance(payload, dict):
|
||||
payload = {}
|
||||
|
||||
force_rescan = to_bool(payload.get("force_rescan"), True)
|
||||
result = await asyncio.to_thread(
|
||||
audit_metadata_sizes,
|
||||
force_rescan=force_rescan,
|
||||
worker_count=payload.get("worker_count"),
|
||||
batch_size=payload.get("batch_size"),
|
||||
)
|
||||
return web.json_response(result)
|
||||
|
||||
@routes.post("/model_resolver/loaded")
|
||||
@json_api_endpoint("get_loaded_models")
|
||||
async def get_loaded_models(request):
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
"""
|
||||
Metadata maintenance helpers.
|
||||
|
||||
Provides audits for local model sidecar metadata files.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from .log_system import create_module_logger
|
||||
from .path_utils import get_filename_from_path, get_path_identity, read_json_safe
|
||||
from .type_utils import MODEL_EXTENSIONS, format_size_bytes
|
||||
|
||||
log = create_module_logger(__name__)
|
||||
|
||||
|
||||
BYTE_SIZE_KEYS = ("size", "sizeBytes", "size_bytes", "fileSize", "file_size", "bytes")
|
||||
KIB_SIZE_KEYS = ("sizeKB", "size_kb")
|
||||
MAX_AUDIT_WORKERS = 64
|
||||
MIN_AUDIT_WORKERS = 1
|
||||
MIN_AUDIT_BATCH_SIZE = 16
|
||||
MAX_AUDIT_BATCH_SIZE = 256
|
||||
|
||||
|
||||
def _empty_audit_counts() -> Dict[str, Any]:
|
||||
return {
|
||||
"scanned_models": 0,
|
||||
"metadata_files": 0,
|
||||
"checked_metadata": 0,
|
||||
"missing_metadata": 0,
|
||||
"missing_size": 0,
|
||||
"invalid_metadata": 0,
|
||||
"skipped_directories": 0,
|
||||
"skipped_non_model_files": 0,
|
||||
"errors": [],
|
||||
"mismatches": [],
|
||||
}
|
||||
|
||||
|
||||
def _coerce_size_bytes(value: Any, multiplier: int = 1) -> Optional[int]:
|
||||
if value is None or value == "" or isinstance(value, bool):
|
||||
return None
|
||||
|
||||
try:
|
||||
if isinstance(value, str):
|
||||
text = value.strip().replace(",", "").replace("_", "")
|
||||
if not text or text.lower() in {"none", "null", "undefined"}:
|
||||
return None
|
||||
number = float(text)
|
||||
else:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
if not math.isfinite(number) or number < 0:
|
||||
return None
|
||||
|
||||
return int(number * multiplier)
|
||||
|
||||
|
||||
def _iter_size_fields(data: Dict[str, Any], prefix: str = "") -> Iterable[Tuple[str, Any, int]]:
|
||||
for key in BYTE_SIZE_KEYS:
|
||||
if key in data:
|
||||
yield (f"{prefix}{key}", data.get(key), 1)
|
||||
for key in KIB_SIZE_KEYS:
|
||||
if key in data:
|
||||
yield (f"{prefix}{key}", data.get(key), 1024)
|
||||
|
||||
|
||||
def _extract_size_from_object(data: Any, prefix: str = "") -> Optional[Tuple[int, str]]:
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
|
||||
for field_path, value, multiplier in _iter_size_fields(data, prefix):
|
||||
size = _coerce_size_bytes(value, multiplier=multiplier)
|
||||
if size is not None:
|
||||
return size, field_path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _file_entry_matches_model(entry: Dict[str, Any], model_filename: str) -> bool:
|
||||
entry_name = get_filename_from_path(
|
||||
entry.get("name")
|
||||
or entry.get("filename")
|
||||
or entry.get("file_name")
|
||||
or entry.get("fileName")
|
||||
or ""
|
||||
).lower()
|
||||
return bool(entry_name and entry_name == model_filename.lower())
|
||||
|
||||
|
||||
def _select_file_entry(entries: Any, model_filename: str) -> Optional[Dict[str, Any]]:
|
||||
if not isinstance(entries, list):
|
||||
return None
|
||||
|
||||
candidates = [entry for entry in entries if isinstance(entry, dict)]
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
for entry in candidates:
|
||||
if _file_entry_matches_model(entry, model_filename):
|
||||
return entry
|
||||
|
||||
for entry in candidates:
|
||||
if entry.get("primary") and str(entry.get("type") or "").lower() == "model":
|
||||
return entry
|
||||
|
||||
for entry in candidates:
|
||||
if entry.get("primary"):
|
||||
return entry
|
||||
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_metadata_size(metadata: Dict[str, Any], model_filename: str = "") -> Optional[Tuple[int, str]]:
|
||||
"""
|
||||
Return the stored model size and the field path it came from.
|
||||
|
||||
Top-level size is preferred because Model Resolver writes LoRA
|
||||
Manager-compatible sidecars with the authoritative value there.
|
||||
"""
|
||||
top_level = _extract_size_from_object(metadata)
|
||||
if top_level:
|
||||
return top_level
|
||||
|
||||
for key in ("file", "file_info", "selected_file", "selectedFile"):
|
||||
nested = metadata.get(key)
|
||||
found = _extract_size_from_object(nested, f"{key}.")
|
||||
if found:
|
||||
return found
|
||||
|
||||
nested_sources = [
|
||||
("files", metadata.get("files")),
|
||||
("selected_version.files", (metadata.get("selected_version") or {}).get("files") if isinstance(metadata.get("selected_version"), dict) else None),
|
||||
("selectedVersion.files", (metadata.get("selectedVersion") or {}).get("files") if isinstance(metadata.get("selectedVersion"), dict) else None),
|
||||
("civitai.files", (metadata.get("civitai") or {}).get("files") if isinstance(metadata.get("civitai"), dict) else None),
|
||||
]
|
||||
for prefix, entries in nested_sources:
|
||||
entry = _select_file_entry(entries, model_filename)
|
||||
found = _extract_size_from_object(entry, f"{prefix}[].")
|
||||
if found:
|
||||
return found
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _is_model_file_path(path: str) -> bool:
|
||||
if not path or not os.path.isfile(path):
|
||||
return False
|
||||
|
||||
filename = get_filename_from_path(path).lower()
|
||||
if filename.endswith((".metadata.json", ".civitai.info")):
|
||||
return False
|
||||
|
||||
return os.path.splitext(filename)[1].lower() in MODEL_EXTENSIONS
|
||||
|
||||
|
||||
def _metadata_sidecar_paths(model_path: str) -> List[str]:
|
||||
directory = os.path.dirname(model_path)
|
||||
filename = get_filename_from_path(model_path)
|
||||
base_name = os.path.splitext(filename)[0]
|
||||
candidates = [
|
||||
os.path.join(directory, f"{base_name}.metadata.json"),
|
||||
os.path.join(directory, f"{filename}.metadata.json"),
|
||||
]
|
||||
|
||||
result: List[str] = []
|
||||
seen = set()
|
||||
for candidate in candidates:
|
||||
try:
|
||||
key = os.path.normcase(os.path.abspath(candidate))
|
||||
except (OSError, ValueError):
|
||||
key = os.path.normcase(candidate)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(candidate)
|
||||
return result
|
||||
|
||||
|
||||
def _format_signed_size(bytes_value: int) -> str:
|
||||
if bytes_value == 0:
|
||||
return "0 B"
|
||||
sign = "+" if bytes_value > 0 else "-"
|
||||
return f"{sign}{format_size_bytes(abs(bytes_value))}"
|
||||
|
||||
|
||||
def _model_key(model_path: str, metadata_path: str) -> Tuple[str, str]:
|
||||
try:
|
||||
model_identity = get_path_identity(model_path)
|
||||
except (OSError, ValueError):
|
||||
model_identity = os.path.normcase(os.path.abspath(model_path))
|
||||
try:
|
||||
metadata_identity = get_path_identity(metadata_path)
|
||||
except (OSError, ValueError):
|
||||
metadata_identity = os.path.normcase(os.path.abspath(metadata_path))
|
||||
return model_identity, metadata_identity
|
||||
|
||||
|
||||
def _model_identity_key(model: Dict[str, Any]) -> str:
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
if not model_path:
|
||||
return ""
|
||||
try:
|
||||
return get_path_identity(model_path)
|
||||
except (OSError, ValueError):
|
||||
try:
|
||||
return os.path.normcase(os.path.abspath(model_path))
|
||||
except (OSError, ValueError):
|
||||
return os.path.normcase(model_path)
|
||||
|
||||
|
||||
def _dedupe_models(models: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
result: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
for model in models or []:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
identity = _model_identity_key(model)
|
||||
if not identity:
|
||||
continue
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
result.append(model)
|
||||
return result
|
||||
|
||||
|
||||
def _get_worker_count(total_models: int, requested_workers: Optional[int] = None) -> Tuple[int, int]:
|
||||
cpu_count = os.cpu_count() or 4
|
||||
if requested_workers is not None:
|
||||
try:
|
||||
workers = int(requested_workers)
|
||||
except (TypeError, ValueError):
|
||||
workers = 0
|
||||
if workers > 0:
|
||||
return min(total_models or 1, max(MIN_AUDIT_WORKERS, min(workers, MAX_AUDIT_WORKERS))), cpu_count
|
||||
|
||||
# The audit is mostly disk I/O plus small JSON parsing, so a few workers per
|
||||
# CPU core keeps the drive busy without creating an unbounded thread swarm.
|
||||
workers = max(MIN_AUDIT_WORKERS, min(MAX_AUDIT_WORKERS, cpu_count * 4))
|
||||
return min(total_models or 1, workers), cpu_count
|
||||
|
||||
|
||||
def _get_batch_size(
|
||||
total_models: int,
|
||||
worker_count: int,
|
||||
requested_batch_size: Optional[int] = None,
|
||||
) -> int:
|
||||
if requested_batch_size is not None:
|
||||
try:
|
||||
batch_size = int(requested_batch_size)
|
||||
except (TypeError, ValueError):
|
||||
batch_size = 0
|
||||
if batch_size > 0:
|
||||
return max(1, batch_size)
|
||||
|
||||
if total_models <= 0:
|
||||
return MIN_AUDIT_BATCH_SIZE
|
||||
|
||||
target_batches = max(1, worker_count * 4)
|
||||
batch_size = math.ceil(total_models / target_batches)
|
||||
return max(MIN_AUDIT_BATCH_SIZE, min(MAX_AUDIT_BATCH_SIZE, batch_size))
|
||||
|
||||
|
||||
def _make_batches(models: List[Dict[str, Any]], batch_size: int) -> List[List[Dict[str, Any]]]:
|
||||
if not models:
|
||||
return []
|
||||
safe_batch_size = max(1, int(batch_size or 1))
|
||||
return [
|
||||
models[index:index + safe_batch_size]
|
||||
for index in range(0, len(models), safe_batch_size)
|
||||
]
|
||||
|
||||
|
||||
def _audit_one_model(model: Dict[str, Any]) -> Dict[str, Any]:
|
||||
result = _empty_audit_counts()
|
||||
model_path = str(model.get("path") or "").strip()
|
||||
if not model_path:
|
||||
return result
|
||||
|
||||
if os.path.isdir(model_path):
|
||||
result["skipped_directories"] += 1
|
||||
return result
|
||||
|
||||
if not _is_model_file_path(model_path):
|
||||
result["skipped_non_model_files"] += 1
|
||||
return result
|
||||
|
||||
result["scanned_models"] += 1
|
||||
|
||||
try:
|
||||
actual_size = os.path.getsize(model_path)
|
||||
except OSError as exc:
|
||||
result["errors"].append(
|
||||
{
|
||||
"model_path": model_path,
|
||||
"metadata_path": "",
|
||||
"message": str(exc),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
existing_metadata_paths = [
|
||||
metadata_path
|
||||
for metadata_path in _metadata_sidecar_paths(model_path)
|
||||
if os.path.isfile(metadata_path)
|
||||
]
|
||||
if not existing_metadata_paths:
|
||||
result["missing_metadata"] += 1
|
||||
return result
|
||||
|
||||
model_filename = get_filename_from_path(model_path)
|
||||
seen_entries = set()
|
||||
for metadata_path in existing_metadata_paths:
|
||||
entry_key = _model_key(model_path, metadata_path)
|
||||
if entry_key in seen_entries:
|
||||
continue
|
||||
seen_entries.add(entry_key)
|
||||
result["metadata_files"] += 1
|
||||
|
||||
metadata = read_json_safe(metadata_path, None)
|
||||
if not isinstance(metadata, dict):
|
||||
result["invalid_metadata"] += 1
|
||||
result["errors"].append(
|
||||
{
|
||||
"model_path": model_path,
|
||||
"metadata_path": metadata_path,
|
||||
"message": "Metadata JSON is missing, invalid, or not an object.",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
size_info = extract_metadata_size(metadata, model_filename)
|
||||
if not size_info:
|
||||
result["missing_size"] += 1
|
||||
continue
|
||||
|
||||
result["checked_metadata"] += 1
|
||||
metadata_size, size_field = size_info
|
||||
if metadata_size == actual_size:
|
||||
continue
|
||||
|
||||
difference = actual_size - metadata_size
|
||||
result["mismatches"].append(
|
||||
{
|
||||
"filename": model_filename,
|
||||
"relative_path": model.get("relative_path") or model_filename,
|
||||
"category": model.get("category") or "",
|
||||
"base_directory": model.get("base_directory") or "",
|
||||
"model_path": model_path,
|
||||
"metadata_path": metadata_path,
|
||||
"metadata_size": metadata_size,
|
||||
"actual_size": actual_size,
|
||||
"difference": difference,
|
||||
"metadata_size_label": format_size_bytes(metadata_size),
|
||||
"actual_size_label": format_size_bytes(actual_size),
|
||||
"difference_label": _format_signed_size(difference),
|
||||
"size_field": size_field,
|
||||
}
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _merge_audit_counts(target: Dict[str, Any], source: Dict[str, Any]) -> None:
|
||||
for key in (
|
||||
"scanned_models",
|
||||
"metadata_files",
|
||||
"checked_metadata",
|
||||
"missing_metadata",
|
||||
"missing_size",
|
||||
"invalid_metadata",
|
||||
"skipped_directories",
|
||||
"skipped_non_model_files",
|
||||
):
|
||||
target[key] += int(source.get(key) or 0)
|
||||
target["errors"].extend(source.get("errors") or [])
|
||||
target["mismatches"].extend(source.get("mismatches") or [])
|
||||
|
||||
|
||||
def _audit_model_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
result = _empty_audit_counts()
|
||||
for model in batch:
|
||||
_merge_audit_counts(result, _audit_one_model(model))
|
||||
return result
|
||||
|
||||
|
||||
def audit_metadata_sizes(
|
||||
models: Optional[List[Dict[str, Any]]] = None,
|
||||
*,
|
||||
force_rescan: bool = True,
|
||||
worker_count: Optional[int] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Compare sidecar metadata size values against actual local model file sizes.
|
||||
"""
|
||||
if models is None:
|
||||
from .scanner import get_model_files
|
||||
|
||||
models = get_model_files(force_rescan=force_rescan)
|
||||
|
||||
audit_models = _dedupe_models(models or [])
|
||||
total_models = len(audit_models)
|
||||
resolved_worker_count, cpu_count = _get_worker_count(total_models, worker_count)
|
||||
resolved_batch_size = _get_batch_size(total_models, resolved_worker_count, batch_size)
|
||||
batches = _make_batches(audit_models, resolved_batch_size)
|
||||
active_worker_count = min(resolved_worker_count, len(batches) or 1)
|
||||
aggregate = _empty_audit_counts()
|
||||
|
||||
if batches:
|
||||
if active_worker_count <= 1:
|
||||
for batch in batches:
|
||||
_merge_audit_counts(aggregate, _audit_model_batch(batch))
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=active_worker_count) as executor:
|
||||
futures = [executor.submit(_audit_model_batch, batch) for batch in batches]
|
||||
for future in as_completed(futures):
|
||||
_merge_audit_counts(aggregate, future.result())
|
||||
|
||||
mismatches = aggregate["mismatches"]
|
||||
errors = aggregate["errors"]
|
||||
|
||||
mismatches.sort(
|
||||
key=lambda item: (
|
||||
str(item.get("category") or "").lower(),
|
||||
str(item.get("relative_path") or item.get("filename") or "").lower(),
|
||||
str(item.get("metadata_path") or "").lower(),
|
||||
)
|
||||
)
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"cpu_count": cpu_count,
|
||||
"worker_count": active_worker_count,
|
||||
"batch_size": resolved_batch_size,
|
||||
"batch_count": len(batches),
|
||||
"scanned_models": aggregate["scanned_models"],
|
||||
"metadata_files": aggregate["metadata_files"],
|
||||
"checked_metadata": aggregate["checked_metadata"],
|
||||
"missing_metadata": aggregate["missing_metadata"],
|
||||
"missing_size": aggregate["missing_size"],
|
||||
"invalid_metadata": aggregate["invalid_metadata"],
|
||||
"skipped_directories": aggregate["skipped_directories"],
|
||||
"skipped_non_model_files": aggregate["skipped_non_model_files"],
|
||||
"error_count": len(errors),
|
||||
"errors": errors[:50],
|
||||
"mismatch_count": len(mismatches),
|
||||
"mismatches": mismatches,
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Metadata size audit finished: "
|
||||
f"models={aggregate['scanned_models']}, metadata={aggregate['metadata_files']}, "
|
||||
f"size_values={aggregate['checked_metadata']}, mismatches={len(mismatches)}, "
|
||||
f"missing_size={aggregate['missing_size']}, errors={len(errors)}, "
|
||||
f"workers={active_worker_count}, batches={len(batches)}, batch_size={resolved_batch_size}"
|
||||
)
|
||||
return result
|
||||
@@ -6246,6 +6246,13 @@ a.mr-image-preview-resource-name:hover {
|
||||
background: rgba(255, 255, 255, 0.48);
|
||||
}
|
||||
|
||||
.mr-options-main.mr-options-main-fill-section {
|
||||
grid-auto-rows: minmax(0, 1fr);
|
||||
align-content: stretch;
|
||||
align-items: stretch;
|
||||
overflow-y: hidden;
|
||||
}
|
||||
|
||||
.mr-options-card {
|
||||
background: transparent;
|
||||
border: none;
|
||||
@@ -6313,6 +6320,30 @@ a.mr-image-preview-resource-name:hover {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.mr-options-section-fill {
|
||||
grid-template-rows: auto minmax(0, 1fr);
|
||||
align-self: stretch;
|
||||
align-content: stretch;
|
||||
align-items: stretch;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
.mr-options-section-fill .mr-options-grid,
|
||||
.mr-options-section-fill .mr-options-panel {
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
align-content: stretch;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.mr-options-section-fill .mr-options-stack {
|
||||
grid-template-rows: auto auto auto minmax(0, 1fr);
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
align-content: stretch;
|
||||
}
|
||||
|
||||
.mr-options-panel-title {
|
||||
margin: 0 0 6px 0;
|
||||
font-size: 16px;
|
||||
@@ -6408,6 +6439,172 @@ a.mr-image-preview-resource-name:hover {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.mr-options-db-message.is-pending {
|
||||
color: var(--mr-text-secondary);
|
||||
}
|
||||
|
||||
.mr-options-db-message.is-valid {
|
||||
color: var(--mr-color-success);
|
||||
}
|
||||
|
||||
.mr-options-db-message.is-invalid {
|
||||
color: var(--mr-color-error-light);
|
||||
}
|
||||
|
||||
.mr-options-audit-results {
|
||||
margin-top: 12px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.mr-options-section-fill .mr-options-audit-results:not([hidden]) {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.mr-options-audit-table-wrap {
|
||||
overflow: auto;
|
||||
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||
border-radius: 6px;
|
||||
background: rgba(0, 0, 0, 0.14);
|
||||
}
|
||||
|
||||
.mr-options-section-fill .mr-options-audit-table-wrap {
|
||||
flex: 1 1 auto;
|
||||
min-height: 160px;
|
||||
max-height: none;
|
||||
}
|
||||
|
||||
.mr-options-audit-table {
|
||||
width: 100%;
|
||||
min-width: 780px;
|
||||
border-collapse: collapse;
|
||||
table-layout: fixed;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.mr-options-audit-table th,
|
||||
.mr-options-audit-table td {
|
||||
padding: 8px 10px;
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.06);
|
||||
color: var(--mr-text-secondary);
|
||||
vertical-align: top;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.mr-options-audit-table th {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 1;
|
||||
color: var(--mr-text);
|
||||
font-weight: 700;
|
||||
background: rgba(20, 20, 24, 0.96);
|
||||
}
|
||||
|
||||
.mr-options-audit-table tr:last-child td {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.mr-options-audit-row.is-context-menu {
|
||||
cursor: context-menu;
|
||||
}
|
||||
|
||||
.mr-options-audit-row.is-context-menu:hover td {
|
||||
background: rgba(255, 255, 255, 0.045);
|
||||
}
|
||||
|
||||
.mr-options-audit-table th:nth-child(1),
|
||||
.mr-options-audit-table td:nth-child(1) {
|
||||
width: 28%;
|
||||
}
|
||||
|
||||
.mr-options-audit-table th:nth-child(2),
|
||||
.mr-options-audit-table td:nth-child(2) {
|
||||
width: 11%;
|
||||
}
|
||||
|
||||
.mr-options-audit-table th:nth-child(3),
|
||||
.mr-options-audit-table td:nth-child(3),
|
||||
.mr-options-audit-table th:nth-child(4),
|
||||
.mr-options-audit-table td:nth-child(4),
|
||||
.mr-options-audit-table th:nth-child(5),
|
||||
.mr-options-audit-table td:nth-child(5) {
|
||||
width: 12%;
|
||||
}
|
||||
|
||||
.mr-options-audit-model {
|
||||
color: var(--mr-text);
|
||||
font-weight: 700;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.mr-options-audit-path {
|
||||
margin-top: 3px;
|
||||
color: var(--mr-text-muted);
|
||||
font-size: 11px;
|
||||
line-height: 1.3;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.mr-options-audit-delta {
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.mr-options-audit-delta.is-positive {
|
||||
color: var(--mr-color-success);
|
||||
}
|
||||
|
||||
.mr-options-audit-delta.is-negative {
|
||||
color: var(--mr-color-error-light);
|
||||
}
|
||||
|
||||
.mr-options-audit-empty {
|
||||
padding: 12px;
|
||||
border: 1px solid rgba(34, 197, 94, 0.24);
|
||||
border-radius: 6px;
|
||||
color: var(--mr-color-success);
|
||||
background: rgba(34, 197, 94, 0.08);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.mr-options-audit-errors {
|
||||
margin-top: 12px;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid rgba(248, 113, 113, 0.22);
|
||||
border-radius: 6px;
|
||||
background: rgba(248, 113, 113, 0.08);
|
||||
}
|
||||
|
||||
.mr-options-audit-errors-title {
|
||||
margin-bottom: 6px;
|
||||
color: var(--mr-color-error-light);
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.mr-options-audit-errors ul {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.mr-options-audit-errors li {
|
||||
display: grid;
|
||||
gap: 2px;
|
||||
color: var(--mr-text-secondary);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.mr-options-audit-errors code {
|
||||
color: var(--mr-text-muted);
|
||||
font-family: ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace;
|
||||
font-size: 11px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
#model-resolver-modal .mr-options-shell .mr-btn.mr-btn-primary,
|
||||
#model-resolver-modal .mr-options-shell .mr-btn.mr-btn-primary:hover:not(:disabled) {
|
||||
color: #101010;
|
||||
|
||||
@@ -243,6 +243,13 @@ export const optionsMethods = {
|
||||
</span>
|
||||
<span class="mr-options-nav-meta">07</span>
|
||||
</button>
|
||||
<button type="button" class="mr-options-nav-btn" data-target="mr-options-section-metadata-audit">
|
||||
<span class="mr-options-nav-main">
|
||||
<span class="mr-options-nav-icon" aria-hidden="true">${getSvgIcon('hardDrive')}</span>
|
||||
<span>Metadata Sizes</span>
|
||||
</span>
|
||||
<span class="mr-options-nav-meta">08</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mr-options-actions">
|
||||
@@ -723,6 +730,44 @@ export const optionsMethods = {
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="mr-options-section-metadata-audit" class="mr-options-card mr-options-section mr-options-section-fill is-hidden">
|
||||
<div class="mr-options-section-head">
|
||||
<h4 class="mr-options-section-title">Metadata Size Audit</h4>
|
||||
</div>
|
||||
<div class="mr-options-grid">
|
||||
<div class="mr-options-panel">
|
||||
<div class="mr-options-stack">
|
||||
<div class="mr-options-db-summary">
|
||||
<div class="mr-options-db-row">
|
||||
<span>Models scanned</span>
|
||||
<strong id="mr-options-metadata-size-scanned">0</strong>
|
||||
</div>
|
||||
<div class="mr-options-db-row">
|
||||
<span>Metadata checked</span>
|
||||
<strong id="mr-options-metadata-size-checked">0</strong>
|
||||
</div>
|
||||
<div class="mr-options-db-row">
|
||||
<span>Mismatches</span>
|
||||
<strong id="mr-options-metadata-size-mismatches">0</strong>
|
||||
</div>
|
||||
<div class="mr-options-db-row">
|
||||
<span>Missing size</span>
|
||||
<strong id="mr-options-metadata-size-missing">0</strong>
|
||||
</div>
|
||||
<div class="mr-options-db-row">
|
||||
<span>Errors</span>
|
||||
<strong id="mr-options-metadata-size-errors">0</strong>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mr-options-db-actions">
|
||||
<button id="mr-options-metadata-size-scan" type="button" class="mr-btn mr-btn-secondary">${getSvgIcon('refreshCw')} Check Metadata Sizes</button>
|
||||
</div>
|
||||
<div id="mr-options-metadata-size-status" class="mr-options-db-message">Not checked yet.</div>
|
||||
<div id="mr-options-metadata-size-results" class="mr-options-audit-results" hidden></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -803,6 +848,15 @@ export const optionsMethods = {
|
||||
const hfIndexRefreshBtn = this.contentElement.querySelector('#mr-options-hf-index-refresh');
|
||||
const clearAllCacheBtn = this.contentElement.querySelector('#mr-options-clear-all-cache');
|
||||
const clearAllCacheStatus = this.contentElement.querySelector('#mr-options-clear-cache-status');
|
||||
const metadataSizeAuditBtn = this.contentElement.querySelector('#mr-options-metadata-size-scan');
|
||||
const metadataSizeAuditStatus = this.contentElement.querySelector('#mr-options-metadata-size-status');
|
||||
const metadataSizeAuditResults = this.contentElement.querySelector('#mr-options-metadata-size-results');
|
||||
const metadataSizeScannedEl = this.contentElement.querySelector('#mr-options-metadata-size-scanned');
|
||||
const metadataSizeCheckedEl = this.contentElement.querySelector('#mr-options-metadata-size-checked');
|
||||
const metadataSizeMismatchesEl = this.contentElement.querySelector('#mr-options-metadata-size-mismatches');
|
||||
const metadataSizeMissingEl = this.contentElement.querySelector('#mr-options-metadata-size-missing');
|
||||
const metadataSizeErrorsEl = this.contentElement.querySelector('#mr-options-metadata-size-errors');
|
||||
const optionsMain = this.contentElement.querySelector('.mr-options-main');
|
||||
const navButtons = Array.from(this.contentElement.querySelectorAll('.mr-options-nav-btn'));
|
||||
const optionSections = Array.from(this.contentElement.querySelectorAll('.mr-options-section'));
|
||||
const trackedInputs = [
|
||||
@@ -937,6 +991,187 @@ export const optionsMethods = {
|
||||
if (hfIndexRefreshBtn) hfIndexRefreshBtn.disabled = busy;
|
||||
};
|
||||
|
||||
const setMetadataSizeAuditBusy = (busy) => {
|
||||
if (metadataSizeAuditBtn) metadataSizeAuditBtn.disabled = busy;
|
||||
};
|
||||
|
||||
const setStatusMode = (element, mode = '') => {
|
||||
if (!element) return;
|
||||
element.classList.remove('is-valid', 'is-invalid', 'is-pending');
|
||||
if (mode) element.classList.add(mode);
|
||||
};
|
||||
|
||||
const formatAuditNumber = (value) => {
|
||||
const number = Number(value || 0);
|
||||
return Number.isFinite(number) ? number.toLocaleString() : '0';
|
||||
};
|
||||
|
||||
const setMetadataSizeAuditStatus = (text, mode = '') => {
|
||||
if (!metadataSizeAuditStatus) return;
|
||||
metadataSizeAuditStatus.textContent = text;
|
||||
setStatusMode(metadataSizeAuditStatus, mode);
|
||||
};
|
||||
|
||||
const setMetadataSizeSummaryValue = (element, value, mode = '') => {
|
||||
if (!element) return;
|
||||
element.textContent = formatAuditNumber(value);
|
||||
setStatusMode(element, mode);
|
||||
};
|
||||
|
||||
const updateMetadataSizeAuditSummary = (data = {}) => {
|
||||
const mismatchCount = Number(data.mismatch_count || 0);
|
||||
const missingSizeCount = Number(data.missing_size || 0);
|
||||
const errorCount = Number(data.error_count || 0);
|
||||
setMetadataSizeSummaryValue(metadataSizeScannedEl, data.scanned_models || 0);
|
||||
setMetadataSizeSummaryValue(metadataSizeCheckedEl, data.metadata_files ?? data.checked_metadata ?? 0);
|
||||
setMetadataSizeSummaryValue(metadataSizeMismatchesEl, mismatchCount, mismatchCount > 0 ? 'is-invalid' : '');
|
||||
setMetadataSizeSummaryValue(metadataSizeMissingEl, missingSizeCount, missingSizeCount > 0 ? 'is-pending' : '');
|
||||
setMetadataSizeSummaryValue(metadataSizeErrorsEl, errorCount, errorCount > 0 ? 'is-invalid' : '');
|
||||
};
|
||||
|
||||
const renderMetadataSizeAuditResults = (data = null) => {
|
||||
if (!metadataSizeAuditResults) return;
|
||||
if (!data) {
|
||||
metadataSizeAuditResults.hidden = true;
|
||||
metadataSizeAuditResults.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
|
||||
const mismatches = Array.isArray(data.mismatches) ? data.mismatches : [];
|
||||
const errors = Array.isArray(data.errors) ? data.errors : [];
|
||||
const visibleMismatches = mismatches.slice(0, 500);
|
||||
let html = '';
|
||||
|
||||
if (mismatches.length) {
|
||||
const rows = visibleMismatches.map((item) => {
|
||||
const modelLabel = item.relative_path || item.filename || item.model_path || 'Model';
|
||||
const modelPath = item.model_path || '';
|
||||
const metadataPath = item.metadata_path || '';
|
||||
const metadataSize = item.metadata_size_label || this.formatBytes(item.metadata_size);
|
||||
const actualSize = item.actual_size_label || this.formatBytes(item.actual_size);
|
||||
const difference = Number(item.difference || 0);
|
||||
const differenceClass = difference > 0 ? 'is-positive' : (difference < 0 ? 'is-negative' : '');
|
||||
const differenceLabel = item.difference_label || `${difference >= 0 ? '+' : '-'}${this.formatBytes(Math.abs(difference))}`;
|
||||
const sizeField = item.size_field || 'size';
|
||||
const auditContext = modelPath ? {
|
||||
context_scope: 'local_model',
|
||||
open_folder_label: 'Show File in Folder',
|
||||
name: item.filename || modelLabel,
|
||||
filename: item.filename || modelLabel,
|
||||
relative_path: item.relative_path || modelLabel,
|
||||
path: modelPath,
|
||||
resolved_path: modelPath,
|
||||
open_path: modelPath,
|
||||
folder_path: item.base_directory || '',
|
||||
category: item.category || '',
|
||||
metadata_path: metadataPath,
|
||||
size: item.actual_size || 0,
|
||||
file_size: item.actual_size || 0,
|
||||
metadata_size: item.metadata_size || 0,
|
||||
context_source: 'metadata_size_audit'
|
||||
} : null;
|
||||
const contextAttrs = auditContext
|
||||
? this.getContextMenuAttrs(auditContext, 'Right-click for model options')
|
||||
: '';
|
||||
return `
|
||||
<tr class="mr-options-audit-row ${contextAttrs ? 'is-context-menu' : ''}"${contextAttrs}>
|
||||
<td>
|
||||
<div class="mr-options-audit-model">${this.escapeHtml(modelLabel)}</div>
|
||||
<div class="mr-options-audit-path" title="${this.escapeHtml(modelPath)}">${this.escapeHtml(modelPath)}</div>
|
||||
</td>
|
||||
<td>${this.escapeHtml(item.category || '')}</td>
|
||||
<td>
|
||||
<div>${this.escapeHtml(metadataSize)}</div>
|
||||
<div class="mr-options-audit-path">${this.escapeHtml(sizeField)}</div>
|
||||
</td>
|
||||
<td>${this.escapeHtml(actualSize)}</td>
|
||||
<td class="mr-options-audit-delta ${differenceClass}">${this.escapeHtml(differenceLabel)}</td>
|
||||
<td>
|
||||
<div class="mr-options-audit-path" title="${this.escapeHtml(metadataPath)}">${this.escapeHtml(metadataPath)}</div>
|
||||
</td>
|
||||
</tr>
|
||||
`;
|
||||
}).join('');
|
||||
html += `
|
||||
<div class="mr-options-audit-table-wrap">
|
||||
<table class="mr-options-audit-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Model</th>
|
||||
<th>Folder</th>
|
||||
<th>Metadata</th>
|
||||
<th>Actual</th>
|
||||
<th>Delta</th>
|
||||
<th>Metadata file</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
</div>
|
||||
`;
|
||||
if (mismatches.length > visibleMismatches.length) {
|
||||
html += `<div class="mr-options-db-message">Showing first ${visibleMismatches.length.toLocaleString()} of ${mismatches.length.toLocaleString()} mismatches.</div>`;
|
||||
}
|
||||
} else {
|
||||
html += '<div class="mr-options-audit-empty">No metadata size mismatches found.</div>';
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
const errorRows = errors.slice(0, 10).map((item) => {
|
||||
const targetPath = item.metadata_path || item.model_path || '';
|
||||
return `<li><span>${this.escapeHtml(item.message || 'Error')}</span><code>${this.escapeHtml(targetPath)}</code></li>`;
|
||||
}).join('');
|
||||
html += `
|
||||
<div class="mr-options-audit-errors">
|
||||
<div class="mr-options-audit-errors-title">Read errors</div>
|
||||
<ul>${errorRows}</ul>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
metadataSizeAuditResults.innerHTML = html;
|
||||
metadataSizeAuditResults.hidden = false;
|
||||
};
|
||||
|
||||
const runMetadataSizeAudit = async () => {
|
||||
setMetadataSizeAuditBusy(true);
|
||||
updateMetadataSizeAuditSummary({});
|
||||
renderMetadataSizeAuditResults(null);
|
||||
setMetadataSizeAuditStatus('Checking metadata sizes...', 'is-pending');
|
||||
try {
|
||||
const data = await this.fetchJson('/model_resolver/metadata-size-audit', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ force_rescan: true })
|
||||
}, 'Check metadata sizes');
|
||||
updateMetadataSizeAuditSummary(data);
|
||||
renderMetadataSizeAuditResults(data);
|
||||
|
||||
const mismatchCount = Number(data?.mismatch_count || 0);
|
||||
const checkedCount = Number(data?.metadata_files ?? data?.checked_metadata ?? 0);
|
||||
const errorCount = Number(data?.error_count || 0);
|
||||
const workerCount = Number(data?.worker_count || 0);
|
||||
const batchCount = Number(data?.batch_count || 0);
|
||||
const workerText = workerCount > 1
|
||||
? ` using ${workerCount.toLocaleString()} workers across ${batchCount.toLocaleString()} batch${batchCount === 1 ? '' : 'es'}`
|
||||
: '';
|
||||
if (mismatchCount > 0) {
|
||||
setMetadataSizeAuditStatus(`${mismatchCount.toLocaleString()} metadata size mismatch${mismatchCount === 1 ? '' : 'es'} found${workerText}.`, 'is-invalid');
|
||||
this.showNotification('Metadata size mismatches found', 'warning');
|
||||
} else if (errorCount > 0) {
|
||||
setMetadataSizeAuditStatus(`Checked ${checkedCount.toLocaleString()} metadata file${checkedCount === 1 ? '' : 's'}${workerText} with ${errorCount.toLocaleString()} read error${errorCount === 1 ? '' : 's'}.`, 'is-invalid');
|
||||
this.showNotification('Metadata size audit finished with errors', 'warning');
|
||||
} else {
|
||||
setMetadataSizeAuditStatus(`Checked ${checkedCount.toLocaleString()} metadata file${checkedCount === 1 ? '' : 's'}${workerText}. No mismatches found.`, 'is-valid');
|
||||
this.showNotification('Metadata sizes checked', 'success');
|
||||
}
|
||||
} catch (error) {
|
||||
setMetadataSizeAuditStatus(error.message || 'Metadata size audit failed.', 'is-invalid');
|
||||
renderMetadataSizeAuditResults(null);
|
||||
} finally {
|
||||
setMetadataSizeAuditBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const setTokenCheckStatus = (statusEl, text, mode = '') => {
|
||||
if (!statusEl) return;
|
||||
statusEl.textContent = text;
|
||||
@@ -1493,6 +1728,10 @@ export const optionsMethods = {
|
||||
optionSections.forEach((section) => {
|
||||
section.classList.toggle('is-hidden', section.id !== targetId);
|
||||
});
|
||||
optionsMain?.classList.toggle(
|
||||
'mr-options-main-fill-section',
|
||||
targetId === 'mr-options-section-metadata-audit'
|
||||
);
|
||||
setActiveNav(targetId);
|
||||
};
|
||||
|
||||
@@ -1969,6 +2208,12 @@ export const optionsMethods = {
|
||||
});
|
||||
}
|
||||
|
||||
if (metadataSizeAuditBtn) {
|
||||
metadataSizeAuditBtn.addEventListener('click', () => {
|
||||
runMetadataSizeAudit();
|
||||
});
|
||||
}
|
||||
|
||||
if (detectTemplatesBtn) {
|
||||
detectTemplatesBtn.addEventListener('click', () => {
|
||||
applyDetectedPathTemplates();
|
||||
|
||||
Reference in New Issue
Block a user