From 45353ffea1672b54865c9ede3d007224cc366205 Mon Sep 17 00:00:00 2001 From: DemonGatanjieu <207458273+DemonGatanjieu@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:35:23 +0800 Subject: [PATCH] Scans name damaged model files instead of calling them unknown A model Civitai has no record of now has its safetensors header checked (model_identity. safetensors_damage: too short, header longer than the file, not JSON, or a tensor ending past the file's end; only the header is read). Such files get the unmatched reason "damaged": a red badge on the card, a line in the details, a warning on the scan page (the summary counts them) and first place in the unmatched list, with advice to download the model again. Tests: header cases, a scan telling a damaged file from a whole one; the scan report fixtures are whole safetensors files now. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +++ api/scan_summary.py | 1 + docs/architecture/backend.md | 10 ++++++--- model_identity.py | 40 +++++++++++++++++++++++++++++++++-- scraper.py | 8 ++++++- tests/test_file_identity.py | 25 +++++++++++++++++++++- tests/test_scan_report.py | 3 ++- tests/test_scan_retry.py | 15 +++++++++++++ web/modules/locales.js | 8 +++++++ web/modules/model_source.js | 9 +++++++- web/modules/scan_results.js | 2 +- web/modules/ui_scan_lists.js | 11 +++++----- web/modules/ui_scan_page.js | 1 + web/styles/17-model-types.css | 13 ++++++++++++ web/styles/18-scan-page.css | 19 ++++++++++++++++- 15 files changed, 152 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c43c478..96822b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,9 @@ ### ⭐ A Star for the project (点个 Star) - Home's "Found a problem, or have an idea?" band ends with **⭐ Star on GitHub**, and Settings → Help and feedback has **Star it on GitHub**: both open the project page, where the ☆ Star button is. Nothing pops up and nothing is fetched; it is there for whoever wants it. +### ⚠️ Damaged model files are named as such (模型文件损坏会直接说出来) +- A model whose file header cannot be read (a download cut short, or another tool that rewrote the header and failed) used to look like a model Civitai simply does not know, inferred from the file. A scan now checks the header of every model it cannot match and names these **文件损坏**: a red badge on the card, a line in the details, a warning on the scan page, and first place in its unmatched list with advice to download the model again (then **Look up again**). Only the header is read. Models scanned before keep their old reason until they are looked up again. + ### Fixes - **Ctrl+Z after a scan (扫描后的 Ctrl+Z)**: after any scan, the first Ctrl+Z used to seem to do nothing: the workflow's saved fingerprints had changed, which ComfyUI counted as a step. A change of fingerprints alone no longer takes a step. diff --git a/api/scan_summary.py b/api/scan_summary.py index bff5054..98fddea 100644 --- a/api/scan_summary.py +++ b/api/scan_summary.py @@ -61,6 +61,7 @@ def _summarize(): return { "total": total, "matched": matched, "unmatched": len(unmatched), "new": len(new), "pending": sum(item["reason"] in ("offline", "network") for item in unmatched), + "damaged": sum(item["reason"] == "damaged" for item in unmatched), "unmatched_models": unmatched[:MAX_LISTED], "new_models": new[:MAX_LISTED], "skipped": len(skipped), "skipped_models": skipped[:MAX_LISTED], } diff --git a/docs/architecture/backend.md b/docs/architecture/backend.md index 34a496e..716586f 100644 --- a/docs/architecture/backend.md +++ b/docs/architecture/backend.md @@ -138,9 +138,13 @@ read its result. The scan page's picked or listed models go to `/anomalous/scan_ `targets` (`--targets-file` per folder), so one scan has one result. An unmatched model's `.info` (`id` -1) keeps why: `anomalous_unmatched_reason` is -`not_found` (Civitai answered 404), `network` (no answer after the retries) or -`offline`. Online scans look up `network` and `offline` models again; `not_found` ones -wait for "look up again". After `CIVITAI_DOWN_AFTER` models in a row without an answer +`not_found` (Civitai answered 404), `network` (no answer after the retries), +`offline`, or `damaged`: Civitai had no record and the safetensors header does not read +whole (`model_identity.safetensors_damage`: too short, a header longer than the file, not +JSON, or a tensor ending past the file's end; only the header is read). A damaged header +changes the file's hash, so the file is to be downloaded again; the scan summary counts these +(`damaged`). Online scans look up `network` and `offline` models again; `not_found` and +`damaged` ones wait for "look up again". After `CIVITAI_DOWN_AFTER` models in a row without an answer a scan stops asking (`--civitai-down` carries this to the next folders) and infers the rest from the files, instead of waiting for every timeout. diff --git a/model_identity.py b/model_identity.py index 9e541e9..26d5404 100644 --- a/model_identity.py +++ b/model_identity.py @@ -52,8 +52,44 @@ def is_unmatched(info): # Why an unmatched model has no Civitai record: Civitai does not know the file; Civitai could -# not be reached; the scan was offline. Scans before this was recorded leave it empty. -UNMATCHED_REASONS = ("not_found", "network", "offline") +# not be reached; the scan was offline; the file's header is damaged (safetensors_damage), so +# neither Civitai nor the header can tell what it is. Scans before this was recorded leave it empty. +UNMATCHED_REASONS = ("not_found", "network", "offline", "damaged") +MAX_HEADER_BYTES = 100 * 1024 * 1024 + + +def safetensors_damage(file_path): + """What is wrong with a .safetensors file's header, or "" when it reads whole: "short" (no + room for the header's length), "header_size" (longer than the file), "header" (not the JSON + of a tensor list) or "truncated" (a tensor ends past the file's end: a download cut short). + Reads only the header. A file that cannot be opened now says nothing ("").""" + try: + size = os.path.getsize(file_path) + with open(file_path, "rb") as handle: + head = handle.read(8) + if len(head) < 8: + return "short" + header_size = struct.unpack(" MAX_HEADER_BYTES or 8 + header_size > size: + return "header_size" + header = json.loads(handle.read(header_size).decode("utf-8")) + except OSError: + return "" + except ValueError: # bad UTF-8 or JSON + return "header" + if not isinstance(header, dict): + return "header" + data_size = size - 8 - header_size + for name, tensor in header.items(): + if name == "__metadata__": + continue + offsets = tensor.get("data_offsets") if isinstance(tensor, dict) else None + if not (isinstance(offsets, list) and len(offsets) == 2 and all(isinstance(value, int) for value in offsets) + and 0 <= offsets[0] <= offsets[1]): + return "header" + if offsets[1] > data_size: + return "truncated" + return "" def unmatched_reason(info): diff --git a/scraper.py b/scraper.py index 2db0bd9..4d9957d 100644 --- a/scraper.py +++ b/scraper.py @@ -19,7 +19,7 @@ import argparse import shutil from model_policies import is_physical_rename_protected from recycle_bin import move_to_trash -from model_identity import USER_INFO_SUFFIX, computed_file_identity, file_sha256, infer_base_model_from_header, is_unmatched, scan_info_path, sidecar_file_hash, sidecar_info as read_local_info, unmatched_reason +from model_identity import USER_INFO_SUFFIX, computed_file_identity, file_sha256, infer_base_model_from_header, is_unmatched, safetensors_damage, scan_info_path, sidecar_file_hash, sidecar_info as read_local_info, unmatched_reason from civitai_client import CIVITAI_API_KEY, CivitaiUnreachable, download_media, fetch_civitai_info, fetch_model_description @@ -181,6 +181,12 @@ def scan_one(args, state, rename_log, root, filename, previous): file_hash = sidecar_file_hash(previous, file_path, (previous.get("files") or [{}])[0])[0] file_hash = file_hash or calculate_sha256(file_path) civitai_data, reason = ask_civitai(args, state, file_hash) + # A damaged header changes the file's hash and hides its tensors: say so instead of + # "Civitai does not know it", so the user downloads it again. + damage = "" if civitai_data else safetensors_damage(file_path) + if damage: + print(f"\033[91m[-] 文件头损坏 ({damage}),文件可能不完整或被改坏了,建议重新下载: {filename}\033[0m") + reason = "damaged" # What was read from disk is not fetched again: only fresh Civitai data needs the model # page's description and the cover. diff --git a/tests/test_file_identity.py b/tests/test_file_identity.py index c3fb03e..e9e0db4 100644 --- a/tests/test_file_identity.py +++ b/tests/test_file_identity.py @@ -11,10 +11,33 @@ from unittest import mock PLUGIN = Path(__file__).resolve().parents[1] sys.path[:0] = [str(PLUGIN), str(PLUGIN.parents[1])] from api import metadata, models, recipes -from model_identity import computed_file_identity +from model_identity import computed_file_identity, safetensors_damage + + +def safetensors_bytes(tensors, payload=b'\0' * 16, header_size=None): + header = json.dumps({'__metadata__': {'x': 'y'}, **tensors}).encode() + return struct.pack('${escapeHtml(t('modelSourceLabel'))}: ${escapeHtml(source)}`); + if (meta.info_source === 'local' && meta.unmatched_reason === 'damaged') { + parts.push(`⚠ ${escapeHtml(t('modelSourceDamaged'))}`); + } const edited = (meta.user_fields || []).map(field => t(FIELD_KEYS[field] || field)); if (edited.length) parts.push(escapeHtml(t('modelSourceEdited', { fields: edited.join(t('modelSourceListSep')) }))); span.innerHTML = parts.join(' · '); diff --git a/web/modules/scan_results.js b/web/modules/scan_results.js index cbb0973..c664877 100644 --- a/web/modules/scan_results.js +++ b/web/modules/scan_results.js @@ -6,7 +6,7 @@ import { translate as t } from './locales.js'; -const REASON_KEYS = { not_found: 'scanReasonNotFound', network: 'scanReasonNetwork', offline: 'scanReasonOffline' }; +const REASON_KEYS = { not_found: 'scanReasonNotFound', network: 'scanReasonNetwork', offline: 'scanReasonOffline', damaged: 'scanReasonDamaged' }; const ERROR_KEYS = { name_conflict: 'scanErrorNameConflict', duplicate_kept: 'scanErrorDuplicateKept' }; const COUNT_KEYS = [ ['matched', 'scanCountMatched'], ['inferred', 'scanCountInferred'], ['failed', 'scanCountFailed'], diff --git a/web/modules/ui_scan_lists.js b/web/modules/ui_scan_lists.js index d30b1db..63f6fc6 100644 --- a/web/modules/ui_scan_lists.js +++ b/web/modules/ui_scan_lists.js @@ -160,9 +160,9 @@ function listOf(view, summary, last, actions) { STATUS_MARKS[item.status] || ''), }; } - // Unmatched: those the next online scan looks up by itself first. - const items = [...(summary?.unmatched_models || [])] - .sort((a, b) => Number(isPending(b.reason)) - Number(isPending(a.reason))); + // Unmatched: damaged files first (to download again), then those the next online scan looks up by itself. + const rank = item => (item.reason === 'damaged' ? 2 : isPending(item.reason) ? 1 : 0); + const items = [...(summary?.unmatched_models || [])].sort((a, b) => rank(b) - rank(a)); // Civitai does not know it and the file is no known image model: say what it may be. const guess = item => (item.base ? `≈ ${item.base}` : item.reason === 'not_found' ? t('scanReasonNotImage') : ''); return { @@ -170,8 +170,9 @@ function listOf(view, summary, last, actions) { title: t('scanListUnmatched', { count: items.length }), hint: t(actions.offline ? 'scanListUnmatchedHintOffline' : 'scanListUnmatchedHint'), all: items.length && !actions.offline ? [t('scanPageRetry', { count: items.length }), actions.retryAll] : null, - render: item => row(item, [reasonText(item.reason), guess(item)].filter(Boolean).join(' · '), - unmatchedButtons(item, actions), '≈'), + render: item => row(item.reason === 'damaged' ? { ...item, status: 'damaged' } : item, + [reasonText(item.reason), guess(item)].filter(Boolean).join(' · '), + unmatchedButtons(item, actions), item.reason === 'damaged' ? '⚠' : '≈'), }; } diff --git a/web/modules/ui_scan_page.js b/web/modules/ui_scan_page.js index e0b5dcd..306a734 100644 --- a/web/modules/ui_scan_page.js +++ b/web/modules/ui_scan_page.js @@ -254,6 +254,7 @@ function mainPage(owner, panel, { summary, config, last }, busy, progressHost) { stat(summary?.new, t('scanPageNew'), summary?.new ? 'new' : '', summary?.new ? open('new') : null), ); page.append(stats); + if (summary?.damaged) page.append(button('anomalous-scan-damaged-note', t('scanPageDamagedNote', { count: summary.damaged }), open('unmatched'))); if (summary?.skipped) page.append(button('anomalous-scan-skipped-note', t('scanPageSkippedNote', { count: summary.skipped }), open('skipped'))); if (!summary) page.append(el('p', 'anomalous-scan-muted', t('scanPageSummaryFailed'))); diff --git a/web/styles/17-model-types.css b/web/styles/17-model-types.css index 6f40bd0..65980dd 100644 --- a/web/styles/17-model-types.css +++ b/web/styles/17-model-types.css @@ -91,6 +91,19 @@ text-transform: none; } +/* A model whose file header is damaged: to download again. */ +.anomalous-card-badge.is-damaged { + border-color: rgba(239, 68, 68, 0.7); + background: rgba(127, 29, 29, 0.85); + color: #fecaca; + text-transform: none; +} + +.anomalous-source-damaged { + color: var(--amb-danger-text); + font-weight: 600; +} + .anomalous-source-sep { opacity: 0.5; } diff --git a/web/styles/18-scan-page.css b/web/styles/18-scan-page.css index 6f5f864..9b65d2d 100644 --- a/web/styles/18-scan-page.css +++ b/web/styles/18-scan-page.css @@ -50,6 +50,21 @@ color: var(--amb-text-main); } +/* "N model files are damaged": a warning under the counts, to their list. */ +.anomalous-scan-damaged-note { + align-self: flex-start; + padding: 6px 12px; + border: 1px solid color-mix(in srgb, var(--amb-danger-text) 45%, transparent); + border-radius: var(--amb-radius-control); + background: color-mix(in srgb, var(--amb-danger-text) 10%, transparent); + color: var(--amb-danger-text); + font-size: 0.9em; + text-align: left; + cursor: pointer; +} + +.anomalous-scan-damaged-note:hover { background: color-mix(in srgb, var(--amb-danger-text) 16%, transparent); } + .anomalous-scan-stats { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); @@ -568,7 +583,9 @@ label.anomalous-scan-setting { } .anomalous-scan-row.is-matched .anomalous-scan-row-mark { color: var(--amb-ok-text); } -.anomalous-scan-row.is-failed .anomalous-scan-row-mark { color: var(--amb-danger-text); } +.anomalous-scan-row.is-failed .anomalous-scan-row-mark, +.anomalous-scan-row.is-damaged .anomalous-scan-row-mark, +.anomalous-scan-row.is-damaged .anomalous-scan-row-detail { color: var(--amb-danger-text); } .anomalous-scan-row.is-unchanged .anomalous-scan-row-mark { color: var(--amb-text-muted); } .anomalous-scan-row-copy {