Merge feature/next: damaged model files are named as such

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DemonGatanjieu
2026-10-11 03:35:23 +08:00
co-authored by Claude Opus 5.5
15 changed files with 152 additions and 16 deletions
+3
View File
@@ -101,6 +101,9 @@
### ⭐ A Star for the project (点个 Star)
- Home's "Found a problem, or have an idea?" band ends with **⭐ Star on GitHub**, and Settings → Help and feedback has **Star it on GitHub**: both open the project page, where the ☆ Star button is. Nothing pops up and nothing is fetched; it is there for whoever wants it.
### ⚠️ Damaged model files are named as such (模型文件损坏会直接说出来)
- A model whose file header cannot be read (a download cut short, or another tool that rewrote the header and failed) used to look like a model Civitai simply does not know, inferred from the file. A scan now checks the header of every model it cannot match and names these **文件损坏**: a red badge on the card, a line in the details, a warning on the scan page, and first place in its unmatched list with advice to download the model again (then **Look up again**). Only the header is read. Models scanned before keep their old reason until they are looked up again.
### Fixes
- **Ctrl+Z after a scan (扫描后的 Ctrl+Z)**: after any scan, the first Ctrl+Z used to seem to do nothing: the workflow's saved fingerprints had changed, which ComfyUI counted as a step. A change of fingerprints alone no longer takes a step.
+1
View File
@@ -61,6 +61,7 @@ def _summarize():
return {
"total": total, "matched": matched, "unmatched": len(unmatched), "new": len(new),
"pending": sum(item["reason"] in ("offline", "network") for item in unmatched),
"damaged": sum(item["reason"] == "damaged" for item in unmatched),
"unmatched_models": unmatched[:MAX_LISTED], "new_models": new[:MAX_LISTED],
"skipped": len(skipped), "skipped_models": skipped[:MAX_LISTED],
}
+7 -3
View File
@@ -138,9 +138,13 @@ read its result. The scan page's picked or listed models go to `/anomalous/scan_
`targets` (`--targets-file` per folder), so one scan has one result.
An unmatched model's `.info` (`id` -1) keeps why: `anomalous_unmatched_reason` is
`not_found` (Civitai answered 404), `network` (no answer after the retries) or
`offline`. Online scans look up `network` and `offline` models again; `not_found` ones
wait for "look up again". After `CIVITAI_DOWN_AFTER` models in a row without an answer
`not_found` (Civitai answered 404), `network` (no answer after the retries),
`offline`, or `damaged`: Civitai had no record and the safetensors header does not read
whole (`model_identity.safetensors_damage`: too short, a header longer than the file, not
JSON, or a tensor ending past the file's end; only the header is read). A damaged header
changes the file's hash, so the file is to be downloaded again; the scan summary counts these
(`damaged`). Online scans look up `network` and `offline` models again; `not_found` and
`damaged` ones wait for "look up again". After `CIVITAI_DOWN_AFTER` models in a row without an answer
a scan stops asking (`--civitai-down` carries this to the next folders) and infers the
rest from the files, instead of waiting for every timeout.
+38 -2
View File
@@ -56,8 +56,44 @@ def is_unmatched(info):
# Why an unmatched model has no Civitai record: Civitai does not know the file; Civitai could
# not be reached; the scan was offline. Scans before this was recorded leave it empty.
UNMATCHED_REASONS = ("not_found", "network", "offline")
# not be reached; the scan was offline; the file's header is damaged (safetensors_damage), so
# neither Civitai nor the header can tell what it is. Scans before this was recorded leave it empty.
UNMATCHED_REASONS = ("not_found", "network", "offline", "damaged")
MAX_HEADER_BYTES = 100 * 1024 * 1024
def safetensors_damage(file_path):
"""What is wrong with a .safetensors file's header, or "" when it reads whole: "short" (no
room for the header's length), "header_size" (longer than the file), "header" (not the JSON
of a tensor list) or "truncated" (a tensor ends past the file's end: a download cut short).
Reads only the header. A file that cannot be opened now says nothing ("")."""
try:
size = os.path.getsize(file_path)
with open(file_path, "rb") as handle:
head = handle.read(8)
if len(head) < 8:
return "short"
header_size = struct.unpack("<Q", head)[0]
if not header_size or header_size > MAX_HEADER_BYTES or 8 + header_size > size:
return "header_size"
header = json.loads(handle.read(header_size).decode("utf-8"))
except OSError:
return ""
except ValueError: # bad UTF-8 or JSON
return "header"
if not isinstance(header, dict):
return "header"
data_size = size - 8 - header_size
for name, tensor in header.items():
if name == "__metadata__":
continue
offsets = tensor.get("data_offsets") if isinstance(tensor, dict) else None
if not (isinstance(offsets, list) and len(offsets) == 2 and all(isinstance(value, int) for value in offsets)
and 0 <= offsets[0] <= offsets[1]):
return "header"
if offsets[1] > data_size:
return "truncated"
return ""
def unmatched_reason(info):
+7 -1
View File
@@ -19,7 +19,7 @@ import argparse
import shutil
from model_policies import is_physical_rename_protected
from recycle_bin import move_to_trash
from model_identity import USER_INFO_SUFFIX, computed_file_identity, file_sha256, infer_base_model_from_header, is_unmatched, scan_info_path, sidecar_file_hash, sidecar_info as read_local_info, unmatched_reason
from model_identity import USER_INFO_SUFFIX, computed_file_identity, file_sha256, infer_base_model_from_header, is_unmatched, safetensors_damage, scan_info_path, sidecar_file_hash, sidecar_info as read_local_info, unmatched_reason
from civitai_client import CIVITAI_API_KEY, CivitaiUnreachable, download_media, fetch_civitai_info, fetch_model_description
@@ -181,6 +181,12 @@ def scan_one(args, state, rename_log, root, filename, previous):
file_hash = sidecar_file_hash(previous, file_path, (previous.get("files") or [{}])[0])[0]
file_hash = file_hash or calculate_sha256(file_path)
civitai_data, reason = ask_civitai(args, state, file_hash)
# A damaged header changes the file's hash and hides its tensors: say so instead of
# "Civitai does not know it", so the user downloads it again.
damage = "" if civitai_data else safetensors_damage(file_path)
if damage:
print(f"\033[91m[-] 文件头损坏 ({damage}),文件可能不完整或被改坏了,建议重新下载: {filename}\033[0m")
reason = "damaged"
# What was read from disk is not fetched again: only fresh Civitai data needs the model
# page's description and the cover.
+24 -1
View File
@@ -11,10 +11,33 @@ from unittest import mock
PLUGIN = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PLUGIN), str(PLUGIN.parents[1])]
from api import metadata, models, recipes
from model_identity import computed_file_identity
from model_identity import computed_file_identity, safetensors_damage
def safetensors_bytes(tensors, payload=b'\0' * 16, header_size=None):
header = json.dumps({'__metadata__': {'x': 'y'}, **tensors}).encode()
return struct.pack('<Q', len(header) if header_size is None else header_size) + header + payload
class FileIdentityTests(unittest.TestCase):
def test_damaged_headers_are_told_apart_from_whole_files(self):
tensor = lambda end: {'w': {'dtype': 'F32', 'shape': [4], 'data_offsets': [0, end]}}
cases = {
'whole': (safetensors_bytes(tensor(16)), ''),
'short': (b'abc', 'short'),
'header longer than the file': (safetensors_bytes(tensor(16), header_size=10 ** 6), 'header_size'),
'not json': (struct.pack('<Q', 4) + b'{{{{' + b'\0' * 8, 'header'),
'offsets not numbers': (safetensors_bytes({'w': {'data_offsets': ['a', 'b']}}), 'header'),
'cut short': (safetensors_bytes(tensor(64)), 'truncated'),
}
with tempfile.TemporaryDirectory() as directory:
for name, (content, expected) in cases.items():
with self.subTest(name):
path = Path(directory, 'm.safetensors')
path.write_bytes(content)
self.assertEqual(safetensors_damage(str(path)), expected)
self.assertEqual(safetensors_damage(str(Path(directory, 'missing.safetensors'))), '')
def test_header_digests_are_not_file_identity(self):
for key in ('modelspec.hash.blake3', 'modelspec.hash.sha256', 'modelspec.hash_sha256'):
with self.subTest(key=key), tempfile.TemporaryDirectory() as directory:
+2 -1
View File
@@ -35,7 +35,8 @@ class ScanReportTests(unittest.TestCase):
def model(self, name, info=None):
path = self.dir / f"{name}.safetensors"
path.write_bytes(name.encode())
header = json.dumps({"__metadata__": {"name": name}, "w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode()
path.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4) # a whole file (not damaged)
if info is not None:
Path(str(path)[:-12] + ".info").write_text(json.dumps(info), encoding="utf-8")
return path
+15
View File
@@ -73,6 +73,21 @@ class ScanRetryTests(unittest.TestCase):
self.assertNotIn("anomalous_custom_name", info)
self.assertEqual(metadata._read_metadata(str(self.new))["name"], "new")
def test_damaged_files_are_named_as_such(self):
# "model bytes" is no safetensors header; a whole file next to it is only unknown to Civitai.
header = json.dumps({"w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode()
whole = Path(self.temp.name, "whole.safetensors")
whole.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4)
with mock.patch.object(sys, "argv", ["scraper.py", self.temp.name]), \
mock.patch.object(scraper, "fetch_civitai_info", return_value=None), \
mock.patch.object(scraper, "calculate_sha256", return_value=DIGEST):
scraper.main()
reason = lambda path: json.loads(Path(str(path)[:-12] + ".info").read_text(encoding="utf-8"))["anomalous_unmatched_reason"]
self.assertEqual(reason(self.new), "damaged")
self.assertEqual(reason(whole), "not_found")
with mock.patch.object(scan_summary, "get_active_scan_paths", return_value=[self.temp.name]):
self.assertEqual(scan_summary._summarize()["damaged"], 1)
def test_changed_file_is_hashed_again(self):
self.unmatched.write_bytes(b"other bytes, other size")
with mock.patch.object(scraper, "write_report") as report:
+8
View File
@@ -282,6 +282,7 @@ export const i18n = {
scanReasonNotFound: "Civitai 上没有这个文件",
scanReasonNetwork: "当时连不上 Civitai",
scanReasonOffline: "离线扫描,还没查 Civitai",
scanReasonDamaged: "文件头损坏:文件可能没下载完整,或被别的工具改坏了,所以 Civitai 认不出它,也读不出底模。建议重新下载这个模型,换好后点「重新查找」",
scanReasonUnknown: "Civitai 上没找到",
scanErrorNameConflict: "目标文件名已被另一个不同的模型占用,没有改名",
scanErrorDuplicateKept: "发现完全相同的副本,但没能移到回收站,两份都保留了",
@@ -332,6 +333,7 @@ export const i18n = {
scanListSkipped: "不在扫描范围的模型文件 · {count}",
scanListSkippedHint: "这些模型文件不是 .safetensors 格式(比如 .pth、.ckpt、.bin),扫描只认 .safetensors,所以不会去查它们,卡片上也不会标“未扫描”。它们在 ComfyUI 里照样能用;想记下来源,点“查看”后在编辑里填上下载地址。",
scanPageSkippedNote: "另有 {count} 个其他格式的模型文件不在扫描范围,看看是哪些",
scanPageDamagedNote: "⚠ {count} 个模型文件已损坏(文件头读不出来),建议重新下载,点这里查看",
scanReasonNotImage: "也看不出是哪类绘图模型,可能是语音、放大之类的模型",
scanPageShowList: "查看是哪些模型",
scanPageScanNewOffline: "离线扫描 {count} 个新模型",
@@ -341,6 +343,8 @@ export const i18n = {
modelSourceCivitai: "Civitai",
modelSourceLocal: "从文件推断",
modelSourceLocalShort: "本地",
modelSourceDamagedShort: "文件损坏",
modelSourceDamaged: "文件头损坏,建议重新下载",
modelSourceLocalHint: "{reason},底模是从模型文件推断的。可以在扫描页重新联网查找。",
modelSourceNotScanned: "未扫描",
modelSourceNotScannedHint: "还没扫描:到扫描页补全封面、触发词和底模。",
@@ -2539,6 +2543,7 @@ export const i18n = {
scanReasonNotFound: "Civitai does not know this file",
scanReasonNetwork: "Civitai could not be reached then",
scanReasonOffline: "Offline scan, Civitai not asked yet",
scanReasonDamaged: "Damaged header: the file may not have downloaded whole, or another tool broke it, so Civitai cannot tell it and its base model cannot be read. Download this model again, then press Look up again",
scanReasonUnknown: "Not found on Civitai",
scanErrorNameConflict: "a different model already has the target file name; not renamed",
scanErrorDuplicateKept: "an identical copy could not be moved to the Recycle Bin; both kept",
@@ -2589,6 +2594,7 @@ export const i18n = {
scanListSkipped: "Model files a scan does not read · {count}",
scanListSkippedHint: "These model files are not .safetensors (.pth, .ckpt, .bin and so on). Scans read only .safetensors, so these are never looked up and their cards never say “Not scanned”. They work in ComfyUI as usual; to keep where one came from, open it and add its download link in the editor.",
scanPageSkippedNote: "{count} more model files in other formats are not scanned; see which",
scanPageDamagedNote: "⚠ {count} model files are damaged (their header cannot be read); download them again — see which",
scanReasonNotImage: "and the file is no known image model: maybe a voice, upscale or similar model",
scanPageShowList: "Show which models",
scanPageScanNewOffline: "Scan {count} new models offline",
@@ -2598,6 +2604,8 @@ export const i18n = {
modelSourceCivitai: "Civitai",
modelSourceLocal: "Inferred from the file",
modelSourceLocalShort: "Local",
modelSourceDamagedShort: "Damaged",
modelSourceDamaged: "Damaged header, download it again",
modelSourceLocalHint: "{reason}; the base model was inferred from the file. You can look it up again on the scan page.",
modelSourceNotScanned: "Not scanned",
modelSourceNotScannedHint: "Not scanned yet: the scan page fills in cover, trigger words and base model.",
+8 -1
View File
@@ -30,7 +30,11 @@ export function renderSourceBadge(model) {
const baseModel = String(meta.baseModel || '');
const badge = document.createElement('div');
badge.className = 'anomalous-card-badge';
if (meta.info_source === 'local') {
if (meta.info_source === 'local' && meta.unmatched_reason === 'damaged') {
badge.classList.add('is-inferred', 'is-damaged');
badge.textContent = `⚠ ${t('modelSourceDamagedShort')}`;
badge.title = reasonText('damaged');
} else if (meta.info_source === 'local') {
badge.classList.add('is-inferred', ...(baseModel ? [colourClass(baseModel)] : []));
badge.textContent = `≈ ${baseModel || t('modelSourceLocalShort')}`;
badge.title = t('modelSourceLocalHint', { reason: reasonText(meta.unmatched_reason) });
@@ -55,6 +59,9 @@ export function renderSourceLine(span, model, meta) {
: meta.info_source === 'local' ? t('modelSourceLocal')
: scannable(model) ? t('modelSourceNotScanned') : '';
if (source) parts.push(`<strong>${escapeHtml(t('modelSourceLabel'))}:</strong> ${escapeHtml(source)}`);
if (meta.info_source === 'local' && meta.unmatched_reason === 'damaged') {
parts.push(`<span class="anomalous-source-damaged">⚠ ${escapeHtml(t('modelSourceDamaged'))}</span>`);
}
const edited = (meta.user_fields || []).map(field => t(FIELD_KEYS[field] || field));
if (edited.length) parts.push(escapeHtml(t('modelSourceEdited', { fields: edited.join(t('modelSourceListSep')) })));
span.innerHTML = parts.join('<span class="anomalous-source-sep"> · </span>');
+1 -1
View File
@@ -6,7 +6,7 @@
import { translate as t } from './locales.js';
const REASON_KEYS = { not_found: 'scanReasonNotFound', network: 'scanReasonNetwork', offline: 'scanReasonOffline' };
const REASON_KEYS = { not_found: 'scanReasonNotFound', network: 'scanReasonNetwork', offline: 'scanReasonOffline', damaged: 'scanReasonDamaged' };
const ERROR_KEYS = { name_conflict: 'scanErrorNameConflict', duplicate_kept: 'scanErrorDuplicateKept' };
const COUNT_KEYS = [
['matched', 'scanCountMatched'], ['inferred', 'scanCountInferred'], ['failed', 'scanCountFailed'],
+6 -5
View File
@@ -160,9 +160,9 @@ function listOf(view, summary, last, actions) {
STATUS_MARKS[item.status] || ''),
};
}
// Unmatched: those the next online scan looks up by itself first.
const items = [...(summary?.unmatched_models || [])]
.sort((a, b) => Number(isPending(b.reason)) - Number(isPending(a.reason)));
// Unmatched: damaged files first (to download again), then those the next online scan looks up by itself.
const rank = item => (item.reason === 'damaged' ? 2 : isPending(item.reason) ? 1 : 0);
const items = [...(summary?.unmatched_models || [])].sort((a, b) => rank(b) - rank(a));
// Civitai does not know it and the file is no known image model: say what it may be.
const guess = item => (item.base ? `≈ ${item.base}` : item.reason === 'not_found' ? t('scanReasonNotImage') : '');
return {
@@ -170,8 +170,9 @@ function listOf(view, summary, last, actions) {
title: t('scanListUnmatched', { count: items.length }),
hint: t(actions.offline ? 'scanListUnmatchedHintOffline' : 'scanListUnmatchedHint'),
all: items.length && !actions.offline ? [t('scanPageRetry', { count: items.length }), actions.retryAll] : null,
render: item => row(item, [reasonText(item.reason), guess(item)].filter(Boolean).join(' · '),
unmatchedButtons(item, actions), '≈'),
render: item => row(item.reason === 'damaged' ? { ...item, status: 'damaged' } : item,
[reasonText(item.reason), guess(item)].filter(Boolean).join(' · '),
unmatchedButtons(item, actions), item.reason === 'damaged' ? '⚠' : '≈'),
};
}
+1
View File
@@ -254,6 +254,7 @@ function mainPage(owner, panel, { summary, config, last }, busy, progressHost) {
stat(summary?.new, t('scanPageNew'), summary?.new ? 'new' : '', summary?.new ? open('new') : null),
);
page.append(stats);
if (summary?.damaged) page.append(button('anomalous-scan-damaged-note', t('scanPageDamagedNote', { count: summary.damaged }), open('unmatched')));
if (summary?.skipped) page.append(button('anomalous-scan-skipped-note', t('scanPageSkippedNote', { count: summary.skipped }), open('skipped')));
if (!summary) page.append(el('p', 'anomalous-scan-muted', t('scanPageSummaryFailed')));
+13
View File
@@ -91,6 +91,19 @@
text-transform: none;
}
/* A model whose file header is damaged: to download again. */
.anomalous-card-badge.is-damaged {
border-color: rgba(239, 68, 68, 0.7);
background: rgba(127, 29, 29, 0.85);
color: #fecaca;
text-transform: none;
}
.anomalous-source-damaged {
color: var(--amb-danger-text);
font-weight: 600;
}
.anomalous-source-sep {
opacity: 0.5;
}
+18 -1
View File
@@ -50,6 +50,21 @@
color: var(--amb-text-main);
}
/* "N model files are damaged": a warning under the counts, to their list. */
.anomalous-scan-damaged-note {
align-self: flex-start;
padding: 6px 12px;
border: 1px solid color-mix(in srgb, var(--amb-danger-text) 45%, transparent);
border-radius: var(--amb-radius-control);
background: color-mix(in srgb, var(--amb-danger-text) 10%, transparent);
color: var(--amb-danger-text);
font-size: 0.9em;
text-align: left;
cursor: pointer;
}
.anomalous-scan-damaged-note:hover { background: color-mix(in srgb, var(--amb-danger-text) 16%, transparent); }
.anomalous-scan-stats {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
@@ -568,7 +583,9 @@ label.anomalous-scan-setting {
}
.anomalous-scan-row.is-matched .anomalous-scan-row-mark { color: var(--amb-ok-text); }
.anomalous-scan-row.is-failed .anomalous-scan-row-mark { color: var(--amb-danger-text); }
.anomalous-scan-row.is-failed .anomalous-scan-row-mark,
.anomalous-scan-row.is-damaged .anomalous-scan-row-mark,
.anomalous-scan-row.is-damaged .anomalous-scan-row-detail { color: var(--amb-danger-text); }
.anomalous-scan-row.is-unchanged .anomalous-scan-row-mark { color: var(--amb-text-muted); }
.anomalous-scan-row-copy {