diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 84a60b2..a7c4585 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -98,7 +98,10 @@ covered by the `styles.css` manifest. `api/scan_summary.py` counts models matched on Civitai, unmatched and not yet scanned for the scan page and lists the last two, by the same sidecar rules the scraper uses (`sidecar_info`, `is_unmatched`, `unmatched_reason` in `model_identity.py`); it also - lists the model files in other formats, which scans never read, so the page can say so. + lists the model files in other formats, which scans never read, so the page can say so, + the models set to "don't scan", and the new ones whose header names no drawing model kind. + `api/scan_choice.py` saves that choice (`POST /anomalous/scan_choice`, this computer only) + into each model's `.anomalous.json`. - `api/mcp_server.py` serves `/anomalous/mcp`, the Model Context Protocol endpoint AI apps connect to (Streamable HTTP; the 2026-07-28 stateless protocol and the earlier `initialize`-based ones; this computer only, Host and Origin checked against DNS diff --git a/CHANGELOG.md b/CHANGELOG.md index 119f0a1..3cfc2a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,10 @@ - **Docked, the details fit (侧边栏模式不再挤)**: in a narrow panel the left column becomes one row of pills above the values (默认参数, your sets, new, 全部出图, 版本). - **Saving a workflow, redone (保存工作流的窗口重做)**: the cover is large on the left, chosen from the canvas, the current cover or your recent pictures (as thumbnails); name, tags and notes on the right; Cancel and Save always in view. The name starts as the canvas's workflow name when ComfyUI has saved it under one. Esc cancels, Enter in the name saves. +### 🔇 Models you never want scanned (不需要扫描的模型) +- A model whose header looks like no drawing model (a voice model such as F5-TTS kept in checkpoints, say) is no longer just "not scanned" forever. The scan page asks about them: **都不扫**, **都照常扫**, or **逐个看**. Set to not be scanned, a model shows **不扫描** instead of 未扫描, a scan of everything skips it (picked by hand it is still scanned), and the scan page lists these models to change it back. A model's details have the same button. Nothing is decided for you: scanning still keeps a fingerprint that repairs workflows. +- **Not on Civitai is no fault (Civitai 上没有不再像出错)**: the count of models Civitai did not find warns only when some file is damaged or a lookup is still to do; each such model says its fingerprint is kept for repairing workflows. + ### ⚙️ Settings in tabs (设置分组) - The settings page is split into tabs: **Look, Models, Workflows, Prompts, Backup, Help**. "Change where downloads go" opens Models directly. - **Where downloads (and now imports) go is a choice**: by base model (the new default), in Downloads then by base model, all in a Downloads folder, straight in the type's folder, or your own folder typed by hand; an example path shows the result. Without a saved setting, downloads now go by base model instead of into a Downloads folder. diff --git a/api/__init__.py b/api/__init__.py index 23f32ca..b345bca 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -1,6 +1,6 @@ from . import ( activity_log, audio_catalog, folder_types, gallery_routes, materials, media_routes, model_catalog, - model_media, model_metadata, model_resolution, model_type_listing, kept_images, node_material, recipe_packages, recipes, scan_report, scan_summary, + model_media, model_metadata, model_resolution, model_type_listing, kept_images, node_material, recipe_packages, recipes, scan_choice, scan_report, scan_summary, lora_info, mcp_server, model_download, model_import, model_placement, translation_routes, version_manager, backup, ) from .scanner import * @@ -115,6 +115,7 @@ def setup_routes(app): app.router.add_get('/anomalous/scan_summary', scan_summary.api_scan_summary) app.router.add_get('/anomalous/last_scan', scan_report.api_last_scan) app.router.add_get('/anomalous/scan_model', scan_report.api_scan_model) + app.router.add_post('/anomalous/scan_choice', scan_choice.api_set_scan_choice) app.router.add_get('/anomalous/lora_info', lora_info.api_lora_info) # Audio & Voice Studio Routes diff --git a/api/metadata.py b/api/metadata.py index bdeb2dd..0985c98 100644 --- a/api/metadata.py +++ b/api/metadata.py @@ -111,6 +111,8 @@ def _read_metadata(file_path): "info_source": "", "unmatched_reason": "", # for "local": not_found | network | offline | "" (scanned before this was kept) "user_fields": [], + "scan_choice": "", # the user's "skip" / "scan" (model_identity.scan_choice) + "not_drawing": False, # not scanned yet, and its header looks like no drawing model } info_files = [f"{base_path}.info", f"{base_path}.civitai.info"] @@ -175,10 +177,16 @@ def _read_metadata(file_path): if scan is not None: metadata["info_source"] = "local" if is_unmatched(scan) else "civitai" metadata["unmatched_reason"] = unmatched_reason(scan) - if not metadata["baseModel"] and scan is None: - metadata["base_guess"] = inspect_file(file_path).get("base") or "" + if scan is None: + found = inspect_file(file_path) + if not metadata["baseModel"]: + metadata["base_guess"] = found.get("base") or "" + # A header that reads but names no drawing model kind (a voice model, say). + metadata["not_drawing"] = found["format"] == "safetensors" and not found["kind"] # The user's own layer comes last and wins, also when a field was cleared on purpose (""). user = read_json(base_path + USER_INFO_SUFFIX) or {} + if user.get("scan") in ("skip", "scan"): + metadata["scan_choice"] = user["scan"] for key, field in USER_FIELDS.items(): if isinstance(user.get(key), str): metadata[field] = user[key] diff --git a/api/scan_choice.py b/api/scan_choice.py new file mode 100644 index 0000000..23641b1 --- /dev/null +++ b/api/scan_choice.py @@ -0,0 +1,72 @@ +"""The user's choice to leave a model out of scans, or keep it in: POST /anomalous/scan_choice, +kept in each model's .anomalous.json ("scan"; model_identity.scan_choice reads it). The +scan page asks it about new models that look like no drawing model; a model's details change it.""" + +import asyncio +import os + +from aiohttp import web + +from .mcp_server import _is_local +from .path_utils import atomic_write_json, resolve_folder_subdir +from .utils import resolve_within + +try: + from ..model_identity import USER_INFO_SUFFIX, read_json +except ImportError: # loaded outside the package (tests) + from model_identity import USER_INFO_SUFFIX, read_json + +MAX_CHOICE_FILES = 2000 + + +def _set_scan_choice(targets, choice): + """Writes the user's scan choice into each picked model's .anomalous.json ("scan": + "skip" or "scan"; "" removes it). `targets`: [{type, path_idx, subfolder, files}] as a scan + takes them. Returns (models changed, file names left as they were: not found, not a + .safetensors, or an unreadable .anomalous.json).""" + if choice not in ("skip", "scan", "") or not isinstance(targets, list): + raise ValueError("Invalid scan choice") + picked = [] + for target in targets: + names = target.get("files") if isinstance(target, dict) else None + if not isinstance(names, list) or not all(isinstance(name, str) for name in names): + raise ValueError("each target needs a list of file names") + _, target_dir = resolve_folder_subdir(target.get("type"), int(target.get("path_idx", 0)), target.get("subfolder") or "/") + picked += [(target_dir, name) for name in names] + if len(picked) > MAX_CHOICE_FILES: + raise ValueError("Too many models") + changed, failed = 0, [] + for target_dir, name in picked: + try: + path = resolve_within(target_dir, name) + except ValueError: + failed.append(name) + continue + user_file = os.path.splitext(path)[0] + USER_INFO_SUFFIX + user = read_json(user_file) + if not name.endswith(".safetensors") or not os.path.isfile(path) or (user is None and os.path.exists(user_file)): + failed.append(name) + continue + user = user or {"format": 1} + if choice: + user["scan"] = choice + else: + user.pop("scan", None) + atomic_write_json(user_file, user) + changed += 1 + return changed, failed + + +async def api_set_scan_choice(request): + """POST /anomalous/scan_choice {targets, choice} - leave models out of scans ("skip"), keep + them in ("scan"), or forget the choice (""). Only from this computer.""" + if not _is_local(request): + return web.json_response({"status": "error", "message": "Only from this computer"}, status=403) + try: + data = await request.json() + changed, failed = await asyncio.to_thread(_set_scan_choice, data.get("targets"), data.get("choice", "")) + except (AttributeError, KeyError, TypeError, ValueError): + return web.json_response({"status": "error", "message": "Invalid scan choice"}, status=400) + except OSError: + return web.json_response({"status": "error", "message": "Could not save the choice"}, status=500) + return web.json_response({"status": "success", "changed": changed, "failed": failed}) diff --git a/api/scan_summary.py b/api/scan_summary.py index 98fddea..8fa1c39 100644 --- a/api/scan_summary.py +++ b/api/scan_summary.py @@ -1,7 +1,8 @@ """What the scan page shows before a scan: how many models are matched on Civitai, unmatched or new, and which ones the last two are (located like the scan report's models, so the page can open them); also the model files a scan never reads (other formats than .safetensors), so the -page can say why they are not there.""" +page can say why they are not there, the models the user set to "don't scan", and the new ones +whose header looks like no drawing model (a voice model, say), for the page to ask about.""" import asyncio import os @@ -12,9 +13,11 @@ from .folder_types import get_active_scan_paths from .scan_report import locate_root try: - from ..model_identity import is_unmatched, sidecar_info, unmatched_reason + from ..model_identity import is_unmatched, scan_choice, sidecar_info, unmatched_reason + from ..model_kind import inspect_file except ImportError: # loaded outside the package (tests) - from model_identity import is_unmatched, sidecar_info, unmatched_reason + from model_identity import is_unmatched, scan_choice, sidecar_info, unmatched_reason + from model_kind import inspect_file MAX_LISTED = 2000 # models listed per kind; the counts stay exact # Model files in other formats: listed as "not scanned" (scans read only .safetensors). @@ -25,10 +28,12 @@ def _summarize(): """Counts the files the scanner handles (.safetensors in the active scan folders), each once. `pending`: unmatched models Civitai was never asked about (offline, or unreachable then), which the next online scan looks up. `skipped`: model files in other formats, outside hidden - folders (tools keep their working files there).""" + folders (tools keep their working files there). `excluded`: models set to "don't scan", not in + the other counts. A new model not chosen about whose header reads but names no drawing model + kind carries `not_drawing` (counted too); only new models' headers are read.""" seen = set() total = matched = 0 - unmatched, new, skipped = [], [], [] + unmatched, new, skipped, excluded = [], [], [], [] for base_dir in get_active_scan_paths(): located = locate_root(base_dir) or ("", 0) for root, _dirs, files in os.walk(base_dir): @@ -46,14 +51,22 @@ def _summarize(): skipped.append({"type": located[0], "path_idx": located[1], "filename": filename, "rel": os.path.relpath(path, base_dir).replace(os.sep, "/")}) continue + base = os.path.splitext(path)[0] + item = {"type": located[0], "path_idx": located[1], "filename": filename, + "rel": os.path.relpath(path, base_dir).replace(os.sep, "/")} + choice = scan_choice(base) + if choice == "skip": + excluded.append(item) + continue total += 1 - info = sidecar_info(os.path.splitext(path)[0]) + info = sidecar_info(base) if info is not None and not is_unmatched(info): matched += 1 continue - item = {"type": located[0], "path_idx": located[1], "filename": filename, - "rel": os.path.relpath(path, base_dir).replace(os.sep, "/")} if info is None: + found = inspect_file(path) + if not choice and found["format"] == "safetensors" and not found["kind"]: + item["not_drawing"] = True new.append(item) else: item.update(reason=unmatched_reason(info), base=info.get("baseModel") or "") @@ -62,6 +75,8 @@ def _summarize(): "total": total, "matched": matched, "unmatched": len(unmatched), "new": len(new), "pending": sum(item["reason"] in ("offline", "network") for item in unmatched), "damaged": sum(item["reason"] == "damaged" for item in unmatched), + "not_drawing": sum(bool(item.get("not_drawing")) for item in new), + "excluded": len(excluded), "excluded_models": excluded[:MAX_LISTED], "unmatched_models": unmatched[:MAX_LISTED], "new_models": new[:MAX_LISTED], "skipped": len(skipped), "skipped_models": skipped[:MAX_LISTED], } diff --git a/docs/architecture/backend.md b/docs/architecture/backend.md index 55b80f1..1c9d979 100644 --- a/docs/architecture/backend.md +++ b/docs/architecture/backend.md @@ -148,6 +148,16 @@ changes the file's hash, so the file is to be downloaded again; the scan summary a scan stops asking (`--civitai-down` carries this to the next folders) and infers the rest from the files, instead of waiting for every timeout. +A model can be left out of scans: `"scan": "skip"` in its `.anomalous.json` (the +user's own layer, which scans never write; `model_identity.scan_choice`). A scan of a whole +folder skips it and the scan summary counts it apart (`excluded`), not in the other counts; +picked by hand (`targets`) it is scanned all the same. `"scan": "scan"` records that the user +was asked and kept it. Nothing decides this alone: the summary only marks new models whose +safetensors header reads but names no drawing model kind (`model_kind.inspect_file`, kind "", +a voice model say) as `not_drawing`, and the scan page asks. `POST /anomalous/scan_choice` +(`api/scan_choice.py`, this computer only) takes the picked models as scan `targets`, at most +2000, and writes only `.safetensors` files' layers that are readable. + ## MCP endpoint `POST /anomalous/mcp` (`api/mcp_server.py`) answers one JSON-RPC message per POST with diff --git a/model_identity.py b/model_identity.py index 4eb134a..5b4e235 100644 --- a/model_identity.py +++ b/model_identity.py @@ -33,6 +33,17 @@ def read_json(path): return data if isinstance(data, dict) else None +SCAN_CHOICES = ("skip", "scan") + + +def scan_choice(base): + """What the user chose about scanning a model, kept in .anomalous.json ("scan"): + "skip" (a scan of its folder leaves it out; picking it by hand still scans it), "scan" + (asked once, scanned like any other) or "" (not chosen). ``base``: the path without extension.""" + choice = (read_json(base + USER_INFO_SUFFIX) or {}).get("scan") + return choice if choice in SCAN_CHOICES else "" + + def scan_info_path(base): """The sidecar holding a scan result: .info, else another tool's .civitai.info. A .civitai.info with only edits made in older versions (no Civitai ids) does not count, diff --git a/model_kind.py b/model_kind.py index 4a45049..18c4cae 100644 --- a/model_kind.py +++ b/model_kind.py @@ -270,5 +270,5 @@ def inspect_file(path): """classify() of a file on disk, plus its "format" (None when its header cannot be read).""" fmt, header = read_head(path) found = classify(fmt, header) - found["format"] = fmt + found["format"] = fmt if header else None return found diff --git a/scraper.py b/scraper.py index 4d9957d..65815f2 100644 --- a/scraper.py +++ b/scraper.py @@ -19,7 +19,7 @@ import argparse import shutil from model_policies import is_physical_rename_protected from recycle_bin import move_to_trash -from model_identity import USER_INFO_SUFFIX, computed_file_identity, file_sha256, infer_base_model_from_header, is_unmatched, safetensors_damage, scan_info_path, sidecar_file_hash, sidecar_info as read_local_info, unmatched_reason +from model_identity import USER_INFO_SUFFIX, computed_file_identity, file_sha256, infer_base_model_from_header, is_unmatched, safetensors_damage, scan_choice, scan_info_path, sidecar_file_hash, sidecar_info as read_local_info, unmatched_reason from civitai_client import CIVITAI_API_KEY, CivitaiUnreachable, download_media, fetch_civitai_info, fetch_model_description @@ -75,6 +75,11 @@ def write_scan_progress(progress_file, phase, total=0, current=0, filename=""): pass +def left_out(root, filename, selected): + """A model the user set to "don't scan" (model_identity.scan_choice), in a scan of its whole folder.""" + return not selected and scan_choice(os.path.join(root, os.path.splitext(filename)[0])) == "skip" + + def count_scan_files(target_folder, target_files_basenames, unmatched_only=False): total = 0 selected = set(target_files_basenames) @@ -82,6 +87,7 @@ def count_scan_files(target_folder, target_files_basenames, unmatched_only=False total += sum( 1 for filename in files if filename.endswith(".safetensors") and (not selected or filename in selected) + and not left_out(root, filename, selected) and (not unmatched_only or is_unmatched(read_local_info(os.path.join(root, os.path.splitext(filename)[0])))) ) return total @@ -423,6 +429,9 @@ def main(): continue if target_files_basenames and filename not in target_files_basenames: continue + if left_out(root, filename, target_files_basenames): + print(f"[*] 已设为不扫描，跳过: {filename}") + continue previous = read_local_info(os.path.join(root, os.path.splitext(filename)[0])) if args.retry_unmatched and not is_unmatched(previous): continue diff --git a/tests/test_scan_report.py b/tests/test_scan_report.py index aa3b191..4e9329a 100644 --- a/tests/test_scan_report.py +++ b/tests/test_scan_report.py @@ -35,8 +35,8 @@ class ScanReportTests(unittest.TestCase): def model(self, name, info=None): path = self.dir / f"{name}.safetensors" - header = json.dumps({"__metadata__": {"name": name}, "w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode() - path.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4) # a whole file (not damaged) + header = json.dumps({"__metadata__": {"name": name}, "lora_unet_a.lora_up.weight": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode() + path.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4) # a whole LoRA file (not damaged) if info is not None: Path(str(path)[:-12] + ".info").write_text(json.dumps(info), encoding="utf-8") return path diff --git a/tests/test_scan_retry.py b/tests/test_scan_retry.py index 1c1e37d..085b423 100644 --- a/tests/test_scan_retry.py +++ b/tests/test_scan_retry.py @@ -11,7 +11,7 @@ sys.path.insert(0, str(PLUGIN_DIR.parents[1])) sys.path.insert(0, str(PLUGIN_DIR)) import scraper -from api import metadata, scan_summary, scanner +from api import metadata, scan_choice, scan_summary, scanner from model_identity import computed_file_identity DIGEST = "a" * 64 @@ -88,6 +88,36 @@ class ScanRetryTests(unittest.TestCase): with mock.patch.object(scan_summary, "get_active_scan_paths", return_value=[self.temp.name]): self.assertEqual(scan_summary._summarize()["damaged"], 1) + def test_models_set_not_to_be_scanned_are_left_out(self): + # A voice model's header names no drawing model kind: the page asks; "skip" leaves it out. + header = json.dumps({"ema_model.transformer.w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode() + voice = Path(self.temp.name, "voice.safetensors") + voice.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4) + summarize = lambda: scan_summary._summarize() + with mock.patch.object(scan_summary, "get_active_scan_paths", return_value=[self.temp.name]): + first = summarize() + self.assertEqual([item["filename"] for item in first["new_models"] if item.get("not_drawing")], ["voice.safetensors"]) + self.assertTrue(metadata._read_metadata(str(voice))["not_drawing"]) + target = [{"type": "checkpoints", "path_idx": 0, "subfolder": "/", "files": ["voice.safetensors", "gone.safetensors"]}] + with mock.patch.object(scan_choice, "resolve_folder_subdir", return_value=(self.temp.name, self.temp.name)): + self.assertEqual(scan_choice._set_scan_choice(target, "skip"), (1, ["gone.safetensors"])) + with self.assertRaises(ValueError): + scan_choice._set_scan_choice(target, "never") + after = summarize() + self.assertEqual((after["excluded"], after["total"], after["not_drawing"]), (1, first["total"] - 1, 0)) + self.assertEqual(metadata._read_metadata(str(voice))["scan_choice"], "skip") + with mock.patch.object(sys, "argv", ["scraper.py", self.temp.name]), \ + mock.patch.object(scraper, "fetch_civitai_info", return_value=None), \ + mock.patch.object(scraper, "calculate_sha256", return_value=DIGEST): + scraper.main() + self.assertFalse(Path(self.temp.name, "voice.info").exists()) + # Picked by hand, it is scanned all the same. + with mock.patch.object(sys, "argv", ["scraper.py", self.temp.name, "--target-files", "voice.safetensors"]), \ + mock.patch.object(scraper, "fetch_civitai_info", return_value=None), \ + mock.patch.object(scraper, "calculate_sha256", return_value=DIGEST): + scraper.main() + self.assertTrue(Path(self.temp.name, "voice.info").exists()) + def test_changed_file_is_hashed_again(self): self.unmatched.write_bytes(b"other bytes, other size") with mock.patch.object(scraper, "write_report") as report: diff --git a/web/modules/locales.js b/web/modules/locales.js index 88c15c3..e332c22 100644 --- a/web/modules/locales.js +++ b/web/modules/locales.js @@ -282,6 +282,22 @@ export const i18n = { scanReasonNotFound: "Civitai 上没有这个文件", scanReasonNetwork: "当时连不上 Civitai", scanReasonOffline: "离线扫描，还没查 Civitai", + scanPageNotDrawing: "有 {count} 个还没扫的模型看起来不是绘图模型（比如语音模型），Civitai 上多半查不到。要扫描它们吗？", + scanPageSkipThem: "都不扫", + scanPageScanThem: "都照常扫", + scanPageSeeThem: "逐个看", + scanPageExcludedNote: "{count} 个模型设为不扫描，看看是哪些", + scanListNotDrawing: "看起来不是绘图模型 · {count}", + scanListNotDrawingHint: "这些模型的文件头不像任何一种绘图模型（大模型、LoRA、VAE…），常见的是语音、语言这类模型，Civitai 上一般查不到。设为不扫描后，扫描全部时会跳过它们，也不再显示“未扫描”；想记下它们的指纹（修复工作流时用），就照常扫描。", + scanListExcluded: "设为不扫描的模型 · {count}", + scanListExcludedHint: "扫描全部时跳过这些模型；单独挑出来扫仍然会扫。改为扫描后，下次扫描就会算上它们。", + scanListExcludedAll: "全部改为扫描", + scanRowSkip: "不扫描", + scanRowKeep: "照常扫描", + scanRowUnskip: "改为扫描", + scanRowNotDrawing: "看起来不是绘图模型", + scanFingerprintKept: "指纹已记下，修复工作流时可以用", + scanChoiceFailed: "有 {count} 个模型没改成（找不到文件，或它的 .anomalous.json 读不了）。", scanReasonDamaged: "文件头损坏：文件可能没下载完整，或被别的工具改坏了，所以 Civitai 认不出它，也读不出底模。建议重新下载这个模型，换好后点「重新查找」", scanReasonUnknown: "Civitai 上没找到", scanErrorNameConflict: "目标文件名已被另一个不同的模型占用，没有改名", @@ -345,6 +361,11 @@ export const i18n = { modelSourceLocalShort: "本地", modelSourceDamagedShort: "文件损坏", modelSourceDamaged: "文件头损坏，建议重新下载", + modelSourceSkipped: "不扫描", + modelSourceSkippedHint: "你设了不扫描：扫描全部时会跳过它（单独挑出来扫仍然会扫）。", + modelSourceNotDrawingHint: "还没扫描。文件头看起来不像绘图模型（可能是语音之类的模型），Civitai 上多半查不到，可以设为不扫描。", + modelSourceSetSkip: "设为不扫描", + modelSourceSetScan: "改为扫描", modelSourceLocalHint: "{reason}，底模是从模型文件推断的。可以在扫描页重新联网查找。", modelSourceNotScanned: "未扫描", modelSourceNotScannedHint: "还没扫描：到扫描页补全封面、触发词和底模。", @@ -2598,6 +2619,22 @@ export const i18n = { scanReasonNotFound: "Civitai does not know this file", scanReasonNetwork: "Civitai could not be reached then", scanReasonOffline: "Offline scan, Civitai not asked yet", + scanPageNotDrawing: "{count} models not scanned yet look like no drawing model (a voice model, say); Civitai most likely has no record of them. Scan them?", + scanPageSkipThem: "Don't scan them", + scanPageScanThem: "Scan them as usual", + scanPageSeeThem: "See each", + scanPageExcludedNote: "{count} models are set to not be scanned; see which", + scanListNotDrawing: "Look like no drawing model · {count}", + scanListNotDrawingHint: "These models' headers look like no drawing model (checkpoint, LoRA, VAE…); usually voice or language models, which Civitai does not list. Set to not be scanned, a scan of everything skips them and they no longer show “Not scanned”; scan them as usual to keep their fingerprint (used to repair workflows).", + scanListExcluded: "Models not scanned · {count}", + scanListExcludedHint: "A scan of everything skips these models; picked by hand they are still scanned. Scan them again and the next scan includes them.", + scanListExcludedAll: "Scan them all again", + scanRowSkip: "Don't scan", + scanRowKeep: "Scan as usual", + scanRowUnskip: "Scan again", + scanRowNotDrawing: "Looks like no drawing model", + scanFingerprintKept: "fingerprint kept, used to repair workflows", + scanChoiceFailed: "{count} models were not changed (file not found, or its .anomalous.json cannot be read).", scanReasonDamaged: "Damaged header: the file may not have downloaded whole, or another tool broke it, so Civitai cannot tell it and its base model cannot be read. Download this model again, then press Look up again", scanReasonUnknown: "Not found on Civitai", scanErrorNameConflict: "a different model already has the target file name; not renamed", @@ -2661,6 +2698,11 @@ export const i18n = { modelSourceLocalShort: "Local", modelSourceDamagedShort: "Damaged", modelSourceDamaged: "Damaged header, download it again", + modelSourceSkipped: "Not scanned (your choice)", + modelSourceSkippedHint: "You set it to not be scanned: a scan of everything skips it (picked by hand it is still scanned).", + modelSourceNotDrawingHint: "Not scanned yet. Its header looks like no drawing model (a voice model, say), which Civitai most likely has no record of; you can set it to not be scanned.", + modelSourceSetSkip: "Don't scan it", + modelSourceSetScan: "Scan it again", modelSourceLocalHint: "{reason}; the base model was inferred from the file. You can look it up again on the scan page.", modelSourceNotScanned: "Not scanned", modelSourceNotScannedHint: "Not scanned yet: the scan page fills in cover, trigger words and base model.", diff --git a/web/modules/model_source.js b/web/modules/model_source.js index 18a68f7..53ce894 100644 --- a/web/modules/model_source.js +++ b/web/modules/model_source.js @@ -1,8 +1,10 @@ /** * Where a model's information came from, shown on cards and in the detail header. * Matched on Civitai is the normal case and looks as before; only the exceptions are - * marked: base model inferred from the file (≈), and not scanned yet. The detail header - * also names the fields the user set (metadata.user_fields, from .anomalous.json). + * marked: base model inferred from the file (≈), a damaged file, not scanned yet, and set to + * "don't scan" (metadata.scan_choice). The detail header also names the fields the user set + * (metadata.user_fields, from .anomalous.json), and for a model not scanned whose header + * looks like no drawing model, or one set to "don't scan", a button to change that. */ import { translate as t } from './locales.js'; @@ -39,9 +41,10 @@ export function renderSourceBadge(model) { badge.textContent = `≈ ${baseModel || t('modelSourceLocalShort')}`; badge.title = t('modelSourceLocalHint', { reason: reasonText(meta.unmatched_reason) }); } else if (!meta.info_source && scannable(model)) { + const skipped = meta.scan_choice === 'skip'; badge.classList.add('is-unscanned'); - badge.textContent = t('modelSourceNotScanned'); - badge.title = t('modelSourceNotScannedHint'); + badge.textContent = t(skipped ? 'modelSourceSkipped' : 'modelSourceNotScanned'); + badge.title = t(skipped ? 'modelSourceSkippedHint' : meta.not_drawing ? 'modelSourceNotDrawingHint' : 'modelSourceNotScannedHint'); } else if (baseModel) { badge.classList.add(colourClass(baseModel)); badge.textContent = baseModel; @@ -55,10 +58,12 @@ export function renderSourceBadge(model) { export function renderSourceLine(span, model, meta) { const parts = [`Size: ${escapeHtml(model.size_mb)} MB`]; if (meta.baseModel) parts.push(`Base: ${meta.info_source === 'local' ? '≈ ' : ''}${escapeHtml(meta.baseModel)}`); + const skipped = meta.scan_choice === 'skip'; const source = meta.info_source === 'civitai' ? t('modelSourceCivitai') : meta.info_source === 'local' ? t('modelSourceLocal') - : scannable(model) ? t('modelSourceNotScanned') : ''; + : scannable(model) ? t(skipped ? 'modelSourceSkipped' : 'modelSourceNotScanned') : ''; if (source) parts.push(`${escapeHtml(t('modelSourceLabel'))}: ${escapeHtml(source)}`); + if (skipped && meta.info_source) parts.push(escapeHtml(t('modelSourceSkipped'))); if (meta.info_source === 'local' && meta.unmatched_reason === 'damaged') { parts.push(`⚠ ${escapeHtml(t('modelSourceDamaged'))}`); } @@ -66,5 +71,36 @@ export function renderSourceLine(span, model, meta) { if (edited.length) parts.push(escapeHtml(t('modelSourceEdited', { fields: edited.join(t('modelSourceListSep')) }))); span.innerHTML = parts.join(' · '); span.title = meta.info_source === 'local' ? t('modelSourceLocalHint', { reason: reasonText(meta.unmatched_reason) }) - : !meta.info_source && scannable(model) ? t('modelSourceNotScannedHint') : ''; + : skipped ? t('modelSourceSkippedHint') + : !meta.info_source && scannable(model) ? t(meta.not_drawing ? 'modelSourceNotDrawingHint' : 'modelSourceNotScannedHint') : ''; + if (scannable(model) && (skipped || (!meta.info_source && meta.not_drawing))) span.append(choiceButton(span, model, meta)); +} + +/** "Don't scan" / "Scan it after all" in the detail header: saves the choice, then draws the line again. */ +function choiceButton(span, model, meta) { + const skip = meta.scan_choice !== 'skip'; + const node = document.createElement('button'); + node.type = 'button'; + node.className = 'anomalous-source-choice'; + node.textContent = t(skip ? 'modelSourceSetSkip' : 'modelSourceSetScan'); + node.onclick = async () => { + node.disabled = true; + const choice = skip ? 'skip' : ''; + try { + const res = await fetch('/anomalous/scan_choice', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ choice, targets: [{ type: model.type, path_idx: model.path_idx, subfolder: model.subfolder || '/', files: [model.filename] }] }), + }); + const data = await res.json().catch(() => ({})); + if (!res.ok || data.status !== 'success' || data.changed !== 1) throw new Error(data.message || `HTTP ${res.status}`); + meta.scan_choice = choice; + renderSourceLine(span, model, meta); + } catch (error) { + console.error('Could not save the scan choice:', error); + node.disabled = false; + alert(t('scanChoiceFailed', { count: 1 })); + } + }; + return node; } diff --git a/web/modules/ui_scan_lists.js b/web/modules/ui_scan_lists.js index 63f6fc6..661229a 100644 --- a/web/modules/ui_scan_lists.js +++ b/web/modules/ui_scan_lists.js @@ -3,7 +3,9 @@ * not scanned yet, the unmatched ones, the model files a scan never reads (other formats), and * each model the last scan changed (GET /anomalous/scan_summary, /anomalous/last_scan). * On the scan page itself the last scan is one summary card with a way to its list. - * Every row opens its model; rows offer "scan" / "look up again" and a Civitai search. + * Every row opens its model; rows offer "scan" / "look up again" and a Civitai search. Two + * more lists: the new models whose header looks like no drawing model ("don't scan" / "scan + * as usual"), and the models set to "don't scan" ("scan it after all"). * The page passes what the buttons do (`actions`); this module only renders. */ @@ -135,12 +137,39 @@ function listOf(view, summary, last, actions) { title: t('scanListNew', { count: items.length }), hint: t('scanListNewHint'), all: items.length ? [t('scanListScanAll', { count: items.length }), () => actions.scan(items)] : null, - render: item => row(item, '', [ + render: item => row(item, item.not_drawing ? t('scanRowNotDrawing') : '', [ [t('scanRowOpen'), () => actions.open(item)], [t('scanRowScan'), () => actions.scan([item]), actions.busy], ]), }; } + if (view === 'notdrawing') { + const items = (summary?.new_models || []).filter(item => item.not_drawing); + return { + items, + title: t('scanListNotDrawing', { count: items.length }), + hint: t('scanListNotDrawingHint'), + all: items.length ? [t('scanPageSkipThem'), () => actions.choose(items, 'skip')] : null, + render: item => row(item, '', [ + [t('scanRowOpen'), () => actions.open(item)], + [t('scanRowSkip'), () => actions.choose([item], 'skip')], + [t('scanRowKeep'), () => actions.choose([item], 'scan')], + ]), + }; + } + if (view === 'excluded') { + const items = summary?.excluded_models || []; + return { + items, + title: t('scanListExcluded', { count: items.length }), + hint: t('scanListExcludedHint'), + all: items.length ? [t('scanListExcludedAll'), () => actions.choose(items, '')] : null, + render: item => row(item, '', [ + [t('scanRowOpen'), () => actions.open(item)], + [t('scanRowUnskip'), () => actions.choose([item], '')], + ]), + }; + } if (view === 'skipped') { const items = summary?.skipped_models || []; return { @@ -171,13 +200,13 @@ function listOf(view, summary, last, actions) { hint: t(actions.offline ? 'scanListUnmatchedHintOffline' : 'scanListUnmatchedHint'), all: items.length && !actions.offline ? [t('scanPageRetry', { count: items.length }), actions.retryAll] : null, render: item => row(item.reason === 'damaged' ? { ...item, status: 'damaged' } : item, - [reasonText(item.reason), guess(item)].filter(Boolean).join(' · '), + [reasonText(item.reason), guess(item), item.reason === 'not_found' ? t('scanFingerprintKept') : ''].filter(Boolean).join(' · '), unmatchedButtons(item, actions), item.reason === 'damaged' ? '⚠' : '≈'), }; } /** - * One list as a page ('new' | 'unmatched' | 'skipped' | 'result'): back to the scan page, what + * One list as a page ('new' | 'unmatched' | 'skipped' | 'result' | 'notdrawing' | 'excluded'): back to the scan page, what * these models are and why they are here, a button for all of them, then the rows. A scan * started here shows its progress in `progressHost`, above the rows. */ diff --git a/web/modules/ui_scan_page.js b/web/modules/ui_scan_page.js index 306a734..c7ef11e 100644 --- a/web/modules/ui_scan_page.js +++ b/web/modules/ui_scan_page.js @@ -3,7 +3,8 @@ * (GET /anomalous/scan_summary), one button to scan, how the next scan goes, progress in * the page and the last scan in one card. Each count and the last scan open their list * as a page of its own (ui_scan_lists.js), the settings card the settings page; each - * has a way back. + * has a way back. New models whose header looks like no drawing model are asked about + * (don't scan them, scan them, see them); models set to "don't scan" have a list of their own. * Scans run through scan_runner.js; the progress box is scan_progress.js's panel, * hosted here while the page is shown. */ @@ -23,7 +24,8 @@ const options = { forceOverwrite: false, autoFix: true, }; -// The view shown: '' the page itself, 'settings', or a list: 'new' | 'unmatched' | 'skipped' | 'result'. +// The view shown: '' the page itself, 'settings', or a list: 'new' | 'unmatched' | 'skipped' | 'result' +// | 'notdrawing' | 'excluded'. // `scroll`: where to put a list shown again after a model's Back. const pageState = { view: '', scroll: 0 }; @@ -216,6 +218,37 @@ async function run(owner, panel, extra = {}) { if (panel.isConnected && panel.style.display !== 'none') renderScanPage(owner, panel); } +/** Saves "don't scan" ('skip'), "scan" or no choice ('') for `items`, then reads the page again. */ +async function setChoice(owner, panel, items, choice) { + try { + const res = await fetch('/anomalous/scan_choice', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ targets: targetsForItems(items), choice }), + }); + const data = await res.json().catch(() => ({})); + if (!res.ok || data.status !== 'success') throw new Error(data.message || `HTTP ${res.status}`); + if (data.failed?.length) alert(t('scanChoiceFailed', { count: data.failed.length })); + } catch (error) { + console.error('Could not save the scan choice:', error); + alert(t('scanChoiceFailed', { count: items.length })); + } + renderScanPage(owner, panel); +} + +/** The question about new models that look like no drawing model: don't scan them, scan them, or see them. */ +function notDrawingCard(owner, panel, items, see) { + const card = el('section', 'anomalous-scan-card anomalous-scan-ask'); + const buttons = el('div', 'anomalous-scan-ask-actions'); + buttons.append( + button('anomalous-scan-secondary anomalous-scan-small-btn', t('scanPageSkipThem'), () => setChoice(owner, panel, items, 'skip')), + button('anomalous-scan-secondary anomalous-scan-small-btn', t('scanPageScanThem'), () => setChoice(owner, panel, items, 'scan')), + button('anomalous-scan-small-btn anomalous-scan-ask-see', t('scanPageSeeThem'), see), + ); + card.append(el('p', 'anomalous-scan-ask-text', t('scanPageNotDrawing', { count: items.length })), buttons); + return card; +} + /** To another view of the page ('' is the scan page itself), with what was read last. */ function go(owner, panel, view) { pageState.view = view; @@ -236,6 +269,7 @@ function listActions(owner, panel, busy) { openListedModel(owner, item, () => openScanPage(owner, view, scroll)); }, scan: (items, { retry = false } = {}) => run(owner, panel, { targets: targetsForItems(items), retryUnmatched: retry }), + choose: (items, choice) => setChoice(owner, panel, items, choice), retryAll: () => run(owner, panel, { retryUnmatched: true }), }; } @@ -250,11 +284,15 @@ function mainPage(owner, panel, { summary, config, last }, busy, progressHost) { stats.append( stat(summary?.total, t('scanPageTotal')), stat(summary?.matched, t('scanPageMatched'), 'ok'), - stat(summary?.unmatched, t('scanPageUnmatched'), summary?.unmatched ? 'warn' : '', summary?.unmatched ? open('unmatched') : null), + // Not on Civitai is no fault (the fingerprint is kept): only damaged files and lookups still to do warn. + stat(summary?.unmatched, t('scanPageUnmatched'), summary?.damaged || summary?.pending ? 'warn' : '', summary?.unmatched ? open('unmatched') : null), stat(summary?.new, t('scanPageNew'), summary?.new ? 'new' : '', summary?.new ? open('new') : null), ); page.append(stats); if (summary?.damaged) page.append(button('anomalous-scan-damaged-note', t('scanPageDamagedNote', { count: summary.damaged }), open('unmatched'))); + const notDrawing = (summary?.new_models || []).filter(item => item.not_drawing); + if (notDrawing.length) page.append(notDrawingCard(owner, panel, notDrawing, open('notdrawing'))); + if (summary?.excluded) page.append(button('anomalous-scan-skipped-note', t('scanPageExcludedNote', { count: summary.excluded }), open('excluded'))); if (summary?.skipped) page.append(button('anomalous-scan-skipped-note', t('scanPageSkippedNote', { count: summary.skipped }), open('skipped'))); if (!summary) page.append(el('p', 'anomalous-scan-muted', t('scanPageSummaryFailed'))); diff --git a/web/styles/17-model-types.css b/web/styles/17-model-types.css index cf74e69..d1a4e96 100644 --- a/web/styles/17-model-types.css +++ b/web/styles/17-model-types.css @@ -104,6 +104,20 @@ font-weight: 600; } +/* "Don't scan it" / "Scan it again" after the detail header's source line. */ +.anomalous-source-choice { + margin-left: 8px; + padding: 1px 8px; + border: 1px solid var(--amb-border-strong); + border-radius: 999px; + background: transparent; + color: var(--amb-text-soft); + font-size: 0.85em; + cursor: pointer; +} + +.anomalous-source-choice:hover { border-color: var(--amb-link); color: var(--amb-text-main); } + .anomalous-source-sep { opacity: 0.5; } diff --git a/web/styles/18-scan-page.css b/web/styles/18-scan-page.css index 4753d0d..0741197 100644 --- a/web/styles/18-scan-page.css +++ b/web/styles/18-scan-page.css @@ -50,6 +50,24 @@ color: var(--amb-text-main); } +/* The question about new models that look like no drawing model, under the counts. */ +.anomalous-scan-ask { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 10px 14px; +} + +.anomalous-scan-ask-text { flex: 1 1 260px; margin: 0; color: var(--amb-text-soft); font-size: 0.9em; line-height: 1.5; } +.anomalous-scan-ask-actions { display: flex; flex-wrap: wrap; gap: 8px; } + +.anomalous-scan-ask-see { + border: none; + background: none; + color: var(--amb-link); + cursor: pointer; +} + /* "N model files are damaged": a warning under the counts, to their list. */ .anomalous-scan-damaged-note { align-self: flex-start;