From 4bc559eaa44b98fe4f054f451e0c04cc9e4145f3 Mon Sep 17 00:00:00 2001 From: DemonGatanjieu <207458273+DemonGatanjieu@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:39:15 +0800 Subject: [PATCH] Scan page: quick with many models or a network drive; says it is counting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a cloud server with 1041 models not scanned yet, on the platform's network model disk, GET /anomalous/scan_summary took 79 s, each time the page opened, and the page stayed blank meanwhile: it read each new model's header (for "looks like no drawing model") one after another. The headers are now read 16 at a time (HEAD_READERS) and through get_metadata, whose cache keeps each answer until the file changes, so the page opened again reads none of them and the picker after it finds them there. Opened afresh, a count slower than 400 ms shows "Counting your models…" instead of a blank page. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + api/scan_summary.py | 25 +++++++++++++++++++------ docs/architecture/backend.md | 6 +++++- docs/architecture/frontend.md | 2 +- tests/test_scan_retry.py | 18 ++++++++++++++++++ web/modules/locales.js | 4 ++++ web/modules/ui_scan_page.js | 25 ++++++++++++++++++++++--- web/styles.css | 2 +- web/styles/18-scan-page.css | 33 +++++++++++++++++++++++++++++++++ 9 files changed, 104 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9771d38..d10f564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,7 @@ - **Components on their own (基础组件)**: VAE, text encoders, CLIP, CLIP Vision and upscalers get a chip of their own, "7 components · not found is fine", whose list (found, not found, not scanned yet) says that many of them are not on Civitai at all and that this does not matter (the fingerprint is still kept for Model Check). The counts above, and the Not found and Not scanned yet lists, leave them out; their own list shows all of them. - **The model picker is new**: every model a scan reads, as a list (name, folder, state, base model, size) or as covers. Chips by state, the same words as the scan page — **Not scanned yet, Not found, Damaged, Found, Never scan** — with counts; a search; the folders on the left with how many are picked in each, two folders of one type told apart by where they are. **Pick these N** picks everything shown (so "the 12 not scanned yet in LoRA" is two clicks), shift-click picks a run, **Picked only** shows the picks. The footer says how many are picked and how big they are. - Only files a scan can read (.safetensors) are listed; the old picker also offered .ckpt and other files a scan never opens. +- **Quick to open with many models, or models on a network drive** (a NAS, a cloud server's model disk): the page reads the headers of models not scanned yet 16 at a time and remembers them until the file changes, so it opens again at once; while it counts, it says **Counting your models…** instead of staying blank. On a cloud server with 1041 unscanned models it had taken 79 seconds each time. ### ⚙️ Settings in tabs (设置分组) - The settings page is split into tabs: **Look, Models, Workflows, Prompts, Backup, Help**. "Change where downloads go" opens Models directly. diff --git a/api/scan_summary.py b/api/scan_summary.py index e6834eb..f9c2921 100644 --- a/api/scan_summary.py +++ b/api/scan_summary.py @@ -10,6 +10,7 @@ a scan reads with its name, size, cover and state.""" import asyncio import os +from concurrent.futures import ThreadPoolExecutor from aiohttp import web @@ -20,10 +21,8 @@ from .scan_report import locate_root try: from ..model_identity import is_unmatched, scan_choice, sidecar_info, unmatched_reason - from ..model_kind import inspect_file except ImportError: # loaded outside the package (tests) from model_identity import is_unmatched, scan_choice, sidecar_info, unmatched_reason - from model_kind import inspect_file MAX_LISTED = 2000 # models listed per kind; the counts stay exact MAX_PICKABLE = 5000 # models the picker lists; its total stays exact @@ -34,6 +33,11 @@ OTHER_MODEL_EXTENSIONS = (".ckpt", ".pt", ".pth", ".bin", ".gguf", ".onnx", ".sf COMPONENT_TYPES = ("vae", "clip", "text_encoders", "clip_vision", "upscale_models") # What the picker needs of a model's metadata: its card badge (model_source.js) and its name. PICKER_FIELDS = ("info_source", "unmatched_reason", "scan_choice", "not_drawing", "baseModel", "base_guess", "custom_name") +# New models' headers are read this many at a time: on a network drive (a NAS, a cloud server's +# model disk) each read waits on the network, and a thousand one after another take minutes. +# They are read through get_metadata, whose cache keeps them until the file changes (the page +# opened again reads none, and the picker after it finds them there). +HEAD_READERS = 16 def _walk(): @@ -57,6 +61,11 @@ def _walk(): yield located, base_dir, path, other +def _looks_not_drawing(path): + """Whether the header of a new model reads but names no drawing model kind.""" + return bool(get_metadata(path).get("not_drawing")) + + def _summarize(): """Counts the files the scanner handles (.safetensors in the active scan folders), each once. `pending`: unmatched models Civitai was never asked about (offline, or unreachable then), @@ -67,7 +76,7 @@ def _summarize(): are listed again, matched ones as well, in `component_models` with their `state` (matched | unmatched | new); `components` counts them. `covers`: see `_covers`.""" total = matched = 0 - unmatched, new, skipped, excluded, components, scanned = [], [], [], [], [], [] + unmatched, new, skipped, excluded, components, scanned, to_read = [], [], [], [], [], [], [] for located, base_dir, path, other in _walk(): item = {"type": located[0], "path_idx": located[1], "filename": os.path.basename(path), "rel": os.path.relpath(path, base_dir).replace(os.sep, "/")} @@ -91,14 +100,18 @@ def _summarize(): item["state"] = "matched" continue if info is None: - found = inspect_file(path) - if not choice and found["format"] == "safetensors" and not found["kind"]: - item["not_drawing"] = True + if not choice: + to_read.append((item, path)) item["state"] = "new" new.append(item) else: item.update(state="unmatched", reason=unmatched_reason(info), base=info.get("baseModel") or "") unmatched.append(item) + if to_read: + with ThreadPoolExecutor(max_workers=HEAD_READERS) as pool: + for (item, _path), not_drawing in zip(to_read, pool.map(_looks_not_drawing, [path for _item, path in to_read])): + if not_drawing: + item["not_drawing"] = True return { "total": total, "matched": matched, "unmatched": len(unmatched), "new": len(new), "pending": sum(item["reason"] in ("offline", "network") for item in unmatched), diff --git a/docs/architecture/backend.md b/docs/architecture/backend.md index 6b587a1..3f0f88f 100644 --- a/docs/architecture/backend.md +++ b/docs/architecture/backend.md @@ -171,7 +171,11 @@ folder skips it and the scan summary counts it apart (`excluded`), not in the ot picked by hand (`targets`) it is scanned all the same. `"scan": "scan"` records that the user was asked and kept it. Nothing decides this alone: the summary only marks new models whose safetensors header reads but names no drawing model kind (`model_kind.inspect_file`, kind "", -a voice model say) as `not_drawing`, and the scan page asks. `POST /anomalous/scan_choice` +a voice model say) as `not_drawing`, and the scan page asks. Those headers are read +`HEAD_READERS` (16) at a time, since on a network drive each read waits on the network (1041 +new models on a cloud server's model disk took 79 s one after another), through +`get_metadata`, whose cache keeps each answer until the file changes: the page opened again +reads none of them, and the picker after it finds them there. `POST /anomalous/scan_choice` (`api/scan_choice.py`, this computer only) takes the picked models as scan `targets`, at most 2000, and writes only `.safetensors` files' layers that are readable. diff --git a/docs/architecture/frontend.md b/docs/architecture/frontend.md index fe3f55f..fa59e76 100644 --- a/docs/architecture/frontend.md +++ b/docs/architecture/frontend.md @@ -75,7 +75,7 @@ The main surfaces are: - Grid and model detail: model browsing plus coordinated detail display in `ui_detail.js` (its head with the actions in `model_detail_head.js`), metadata editing in `ui_model_editor.js`; `model_widget_path.js` puts a - model path into a loader's widget. The scan page's model picker is `ui_scan_picker.js`, its settings card `ui_scan_settings.js`. + model path into a loader's widget. The scan page's model picker is `ui_scan_picker.js`, its settings card `ui_scan_settings.js`; opened afresh, a count slower than 400 ms shows "counting your models" instead of a blank page. - Gallery: generated outputs (pictures and videos; All / Pictures / Videos beside the search and the hidden formats in Settings, `gallery_filter.js` + `ui_gallery_filter.js`) and the full-screen Image Detail Studio Workbench, for pictures only. diff --git a/tests/test_scan_retry.py b/tests/test_scan_retry.py index 085b423..fbbce61 100644 --- a/tests/test_scan_retry.py +++ b/tests/test_scan_retry.py @@ -88,6 +88,24 @@ class ScanRetryTests(unittest.TestCase): with mock.patch.object(scan_summary, "get_active_scan_paths", return_value=[self.temp.name]): self.assertEqual(scan_summary._summarize()["damaged"], 1) + def test_new_models_headers_are_read_once_until_the_file_changes(self): + # On a network drive each read waits: the page opened again reads none of them. + header = json.dumps({"ema_model.transformer.w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode() + voice = Path(self.temp.name, "voice.safetensors") + voice.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4) + reads = [] + real_inspect = metadata.inspect_file + metadata.clear_metadata_cache() + with mock.patch.object(scan_summary, "get_active_scan_paths", return_value=[self.temp.name]), \ + mock.patch.object(metadata, "inspect_file", side_effect=lambda path: reads.append(path) or real_inspect(path)): + first = scan_summary._summarize() + again = scan_summary._summarize() + self.assertEqual(len(reads), 2) # new.safetensors and the voice model, each once + self.assertEqual(again["not_drawing"], first["not_drawing"]) + voice.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 8) # changed: read again + scan_summary._summarize() + self.assertEqual([Path(path).name for path in reads].count("voice.safetensors"), 2) + def test_models_set_not_to_be_scanned_are_left_out(self): # A voice model's header names no drawing model kind: the page asks; "skip" leaves it out. header = json.dumps({"ema_model.transformer.w": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}).encode() diff --git a/web/modules/locales.js b/web/modules/locales.js index 44be484..8457291 100644 --- a/web/modules/locales.js +++ b/web/modules/locales.js @@ -239,6 +239,8 @@ export const i18n = { scanPagePickModels: "挑选模型扫描…", scanPickBackToAll: "清空，改回扫描全部", scanPageLeadShort: "给模型配上封面、说明和底模；扫过的不会重复下载。", + scanCountingTitle: "正在清点你的模型…", + scanCountingHint: "模型很多，或者放在网络盘上时，第一次要等一会儿；之后再打开会快很多。", scanPageAboutLink: "扫描是怎么回事？", scanHeroWaiting: "还有 {count} 个模型待查", scanHeroPicked: "已选 {count} 个模型", @@ -2813,6 +2815,8 @@ export const i18n = { scanPagePickModels: "Pick models to scan…", scanPickBackToAll: "Clear, scan everything again", scanPageLeadShort: "Gives your models a cover, a description and a base model; models already scanned are not downloaded again.", + scanCountingTitle: "Counting your models…", + scanCountingHint: "With many models, or models on a network drive, the first time takes a while; opening the page again is much quicker.", scanPageAboutLink: "How scanning works", scanHeroWaiting: "{count} {count|model|models} still to look up", scanHeroPicked: "{count} {count|model|models} picked", diff --git a/web/modules/ui_scan_page.js b/web/modules/ui_scan_page.js index 0246fce..a3e356f 100644 --- a/web/modules/ui_scan_page.js +++ b/web/modules/ui_scan_page.js @@ -12,6 +12,8 @@ * like no drawing model (don't scan them, scan them, see them). Then the settings (ui_scan_settings.js) and the last scan in one line. * Lists are pages of their own (ui_scan_lists.js) with a way back. Scans run through * scan_runner.js; the progress box is scan_progress.js's panel, hosted here while the page is shown. + * Opened afresh and the counts slow to come (many models, or on a network drive), it says it is + * counting instead of staying blank. */ import { translate as t } from './locales.js'; @@ -24,6 +26,7 @@ import { renderScanSettings, scanOptions as options } from './ui_scan_settings.j const SHOWN_PICKS = 12; // picked models named on the page; the rest are counted const BAND_TILES = 48; // pictures put in the scan card's band (as many as fit show) +const COUNTING_AFTER_MS = 400; // the counts not there by then: "counting your models" shows // The view shown: '' the page itself, or a list: 'new' | 'unmatched' | 'skipped' | 'result' // | 'notdrawing' | 'excluded' | 'components'. `scroll`: where to put a list shown again after a // model's Back. `factsOpen`, `moreOpen`: the scan's full counts, the rarer settings unfolded. @@ -393,12 +396,28 @@ function paint(owner, panel, data) { setScanProgressHost(progressHost); } -/** Renders the page into `panel` (again after each scan, so the counts are fresh). */ -export async function renderScanPage(owner, panel) { +/** What the page shows while the counts are on their way. */ +function countingPage() { + const page = el('div', 'anomalous-scan-page'); + const box = el('div', 'anomalous-scan-counting'); + box.setAttribute('role', 'status'); + box.append(scanIcon('scan', 'anomalous-scan-counting-icon'), el('strong', '', t('scanCountingTitle')), + el('p', '', t('scanCountingHint'))); + page.append(box); + return page; +} + +/** Renders the page into `panel` (again after each scan, so the counts are fresh). `fresh`: the + * page was just opened, so a slow count shows "counting" instead of the page from before. */ +export async function renderScanPage(owner, panel, fresh = false) { const token = (panel._scanRender = (panel._scanRender || 0) + 1); + const counting = fresh && setTimeout(() => { + if (token === panel._scanRender) panel.replaceChildren(countingPage()); + }, COUNTING_AFTER_MS); const [summary, config, last] = await Promise.all([ fetchJson('/anomalous/scan_summary'), fetchJson('/anomalous/config'), fetchJson('/anomalous/last_scan'), ]); + clearTimeout(counting); if (token !== panel._scanRender) return; // Closed or left meanwhile: the progress box keeps floating instead of hiding in the page. if (panel.style.display === 'none' || !owner.modal?.classList.contains('visible')) return; @@ -422,7 +441,7 @@ export function openScanPage(owner, view = '', scroll = 0) { owner.enterToolPage?.('scan'); owner.hideAllPanels(); owner.scanPanel.style.display = 'flex'; - renderScanPage(owner, owner.scanPanel); + renderScanPage(owner, owner.scanPanel, true); } /** Leaving the page: the progress box floats again. */ diff --git a/web/styles.css b/web/styles.css index 24918f6..6d01d01 100644 --- a/web/styles.css +++ b/web/styles.css @@ -16,7 +16,7 @@ @import url("./styles/15-shell-rail.css?v=20261015-firstrun-workspace"); @import url("./styles/16-activity.css?v=20261007-theme-2"); @import url("./styles/17-model-types.css?v=20261007-theme-2"); -@import url("./styles/18-scan-page.css?v=20261011-band-1"); +@import url("./styles/18-scan-page.css?v=20261016-counting"); @import url("./styles/19-model-doctor.css?v=20261007-theme-2"); @import url("./styles/20-current-node.css?v=20261011-last"); @import url("./styles/21-prompt-studio.css?v=20261007-theme-2"); diff --git a/web/styles/18-scan-page.css b/web/styles/18-scan-page.css index 5207013..8783f44 100644 --- a/web/styles/18-scan-page.css +++ b/web/styles/18-scan-page.css @@ -129,6 +129,39 @@ .anomalous-scan-band.is-busy .anomalous-scan-band-tile.is-empty { animation: none; } } +/* The counts on their way (many models, or a network drive). */ +.anomalous-scan-counting { + display: flex; + flex-direction: column; + align-items: center; + gap: 8px; + padding: 72px 24px; + text-align: center; + color: var(--amb-text-main); +} + +.anomalous-scan-counting p { + margin: 0; + max-width: 420px; + color: var(--amb-text-soft); + line-height: 1.5; +} + +.anomalous-scan-counting-icon { + width: 40px; + height: 40px; + color: var(--amb-link); + animation: anomalous-scan-counting 1.4s ease-in-out infinite; +} + +@keyframes anomalous-scan-counting { + 50% { opacity: 0.35; } +} + +@media (prefers-reduced-motion: reduce) { + .anomalous-scan-counting-icon { animation: none; } +} + .anomalous-scan-hero-body { display: flex; flex: 1 1 auto;