diff --git a/app/ui/features/generate/warn.js b/app/ui/features/generate/warn.js index c525cf5e..597c14a0 100644 --- a/app/ui/features/generate/warn.js +++ b/app/ui/features/generate/warn.js @@ -1,4 +1,5 @@ -// The chips beside Generate: the pipeline has nowhere to put the scene text; a learning feature is on with no Taste key. +// The chips beside Generate: the pipeline has nowhere to put the scene text; a learning feature is on with no Taste key; +// this torch build runs int8 models slowly. import { composer as c } from "../../composer/composer.js"; export default { @@ -20,7 +21,13 @@ export default { const twice = c.button.sm({ label: "⚠ Two sparse attentions", tone: "ghost", disabled: true, title: "SLA and ComfyUI's Model Sparse Attention node are both in this pipeline. Both sparsify attention; remove one." }).node; twice.hidden = true; - host.append(chip, plain, keyless, twice); + // A torch build that halves int8 speed on this GPU (cu12x on Blackwell). Asked once: the build cannot change while ComfyUI runs. + const slow = c.button.sm({ label: "⚠ Slow torch for int8", tone: "ghost", disabled: true, title: "" }).node; + slow.hidden = true; + host.append(chip, plain, keyless, twice, slow); + Promise.resolve().then(() => app.api?.torchBuild?.()).then((r) => { + if (r && r.slow_int8) { slow.title = r.slow_int8; slow.hidden = false; } + }).catch(() => {}); // unknown build: the loader still says it in the log when an int8 model loads const draw = (slots) => { chip.hidden = !(slots && !slots.some((s) => (s.roles || []).some((r) => r.at === "generation.prompt"))); }; const drawTwice = (slots) => { const native = (slots || []).some((s) => /SparseAttention/.test(String(s.node || ""))); diff --git a/app/ui/features/generate/warn.test.js b/app/ui/features/generate/warn.test.js index be70ee33..62fd4c14 100644 --- a/app/ui/features/generate/warn.test.js +++ b/app/ui/features/generate/warn.test.js @@ -35,3 +35,23 @@ test("the two-sparse-attentions chip shows only when SLA and a native sparse nod listener(); assert.equal(twice().hidden, true, "SLA off: the native node alone is not a clash"); }); + +test("the slow-torch chip shows the server's sentence when this build runs int8 slowly, and stays hidden otherwise", async () => { + const pipeline = { slots: () => [], allOff: () => false, modulesById: () => ({}), activeModules: () => [], useful: () => false, + currentValues: () => ({}), subscribe: () => () => {} }; + const slowChip = (host) => [...host.querySelectorAll("button")].find((b) => /Slow torch/.test(b.textContent)); + const tick = () => new Promise((r) => setTimeout(r, 0)); + let host = document.createElement("div"); + warn.setup({ host, app: { pipeline, api: { torchBuild: async () => ({ slow_int8: "torch is built for CUDA 12.8: reinstall" }) } } }); + await tick(); + assert.equal(slowChip(host).hidden, false); + assert.match(slowChip(host).title, /CUDA 12\.8/); + host = document.createElement("div"); + warn.setup({ host, app: { pipeline, api: { torchBuild: async () => ({ slow_int8: null }) } } }); + await tick(); + assert.equal(slowChip(host).hidden, true, "a good build says nothing"); + host = document.createElement("div"); + warn.setup({ host, app: { pipeline, api: { torchBuild: async () => { throw new Error("offline"); } } } }); + await tick(); + assert.equal(slowChip(host).hidden, true, "an unreadable build is not a claim"); +}); diff --git a/app/ui/shell/api.js b/app/ui/shell/api.js index 0e64440b..4160892f 100644 --- a/app/ui/shell/api.js +++ b/app/ui/shell/api.js @@ -54,6 +54,7 @@ export const api = { shotMemory: () => call("GET", "/api/m/conditioning_shot_camera/memory"), forgetShotMemory: (kind, name) => call("POST", "/api/m/conditioning_shot_camera/forget", { kind, name }), tasteKeys: () => call("GET", "/api/m/taste/keys"), + torchBuild: () => call("GET", "/api/m/loader_diffusion_model/torch"), blockInfluence: (key) => call("GET", `/api/m/block_influence/status?key=${encodeURIComponent(key || "default")}`), blockInfluenceGroups: (key) => call("GET", `/api/m/block_influence/groups?key=${encodeURIComponent(key || "default")}`), clearBlockInfluence: (key) => call("POST", "/api/m/block_influence/clear", { key: key || "default" }), diff --git a/modules/loaders/common.py b/modules/loaders/common.py index 2c9ab3bd..6dfc6428 100644 --- a/modules/loaders/common.py +++ b/modules/loaders/common.py @@ -49,6 +49,42 @@ def unsupported_dtypes(model, weight_dtype, compute_dtype): if picked in ("fp16", "bf16", "fp32") and supported and dtype_of(picked) not in supported] +_SHORT = {"bfloat16": "bf16", "float16": "fp16", "float32": "fp32"} + + +def stored_weights(state_dict): + """What a file's weights are stored as, by element count: "bf16", or "int8+bf16" for a + mixed file. Read from the tensors, not the weight_dtype widget: a pre-quantized file + loaded with weight_dtype bf16 still runs int8 layers, and the log must say so.""" + sizes = {} + for t in state_dict.values(): + if torch.is_tensor(t) and t.dim() >= 2: + name = str(t.dtype).replace("torch.", "") + sizes[_SHORT.get(name, name)] = sizes.get(_SHORT.get(name, name), 0) + t.numel() + total = sum(sizes.values()) + return "+".join(n for n in sorted(sizes, key=sizes.get, reverse=True) if sizes[n] > total * 0.1) or None + + +def slow_int8_build(): + """Why int8 models sample slowly on this torch build, or None. + + Measured on an RTX PRO 6000 Blackwell, same 4-step H3 run: torch 2.10+cu128 sampled the + int8 model ~2x SLOWER than bf16; the same torch from the cu130 index made int8 ~25% faster. + """ + cuda = getattr(torch.version, "cuda", None) + try: + if not cuda or not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] < 10: + return None + if int(cuda.split(".")[0]) >= 13: + return None + except (RuntimeError, ValueError): + return None + # --force-reinstall is required: pip counts 2.10.0+cu128 as already satisfying torch==2.10.0. + return (f"torch is built for CUDA {cuda}: on this Blackwell GPU int8 models sample about 2x slower " + f"than on a CUDA 13 build. Stop ComfyUI and reinstall torch, torchvision and torchaudio " + f"(same versions) with --force-reinstall --index-url https://download.pytorch.org/whl/cu130") + + def weight_model_options(weight_dtype): """comfy `model_options` for a weight dtype. The mapping core's UNETLoader uses.""" options = {} diff --git a/modules/loaders/diffusion_model/__init__.py b/modules/loaders/diffusion_model/__init__.py index 09f8cd34..0f1761d8 100644 --- a/modules/loaders/diffusion_model/__init__.py +++ b/modules/loaders/diffusion_model/__init__.py @@ -9,3 +9,14 @@ CATEGORY = "system" STATUS = "proven" NODES = [FunPackDiffusionModelLoader] + + +def routes(table, base, web): + @table.get(base + "/torch") + async def _torch(_req): + """Why int8 would run slow on this torch build (None when it would not), for the chip beside Generate.""" + from ..common import slow_int8_build + return web.json_response({"slow_int8": slow_int8_build()}) + + +PROVIDES = {"routes": routes} diff --git a/modules/loaders/diffusion_model/nodes.py b/modules/loaders/diffusion_model/nodes.py index 89506183..71144510 100644 --- a/modules/loaders/diffusion_model/nodes.py +++ b/modules/loaders/diffusion_model/nodes.py @@ -16,7 +16,7 @@ from .. import gguf_support, sla_attention from .. import int8_convrot as int8_convrot_module from ..common import (COMPUTE_DTYPES, WEIGHT_DTYPES, attention_choices, unsupported_dtypes, attention_override, dtype_of, set_fp16_accumulation, - weight_model_options) + slow_int8_build, stored_weights, weight_model_options) def _model_file_choices(): @@ -230,6 +230,13 @@ class FunPackDiffusionModelLoader(io.ComfyNode): notes.append(note) if compute_dtype != "default": notes.append("int8_convrot: compute dtype is forced, so the matmuls run in that dtype -- memory is saved, speed is not") + # Read before the load: comfy may consume the dict. + gguf = gguf_support.is_gguf(model_name) or misnamed + stored = None if gguf else stored_weights(state_dict) + slow = slow_int8_build() if stored and "int8" in stored else None + if slow: + log.alert("FunPack Diffusion Model Loader", slow) + notes.append(slow) model = comfy.sd.load_diffusion_model_state_dict( state_dict, model_options=model_options, metadata=metadata) if model is None: @@ -290,8 +297,11 @@ class FunPackDiffusionModelLoader(io.ComfyNode): kind = type(getattr(model, "model", model)).__name__ applied_attention = "sla_h3" if installed else ( attention if override is not None else "default (as launched)") + # The file's own storage, not the widget: an int8 file loaded at "bf16" still runs int8. + weights = weight_dtype if not stored else ( + stored if weight_dtype in ("default", stored) else f"{stored} in file ({weight_dtype} asked)") log.info("FunPack Diffusion Model Loader", f"{model_name} loaded as {kind}, weights " - f"{weight_dtype}, compute {compute_dtype}, attention " + f"{weights}, compute {compute_dtype}, attention " f"{applied_attention}") return io.NodeOutput(model, "\n".join(notes)) diff --git a/modules/loaders/tests/test_weight_report.py b/modules/loaders/tests/test_weight_report.py new file mode 100644 index 00000000..86b4652c --- /dev/null +++ b/modules/loaders/tests/test_weight_report.py @@ -0,0 +1,73 @@ +"""The loader says what the FILE holds, and when this torch build makes int8 slow.""" + +import pytest +import torch + + +@pytest.fixture(autouse=True) +def _needs_comfy(comfyui): + """The loader test imports a node module, which imports comfy.""" + + +def test_stored_weights_reads_the_tensors_not_the_widget(): + from modules.loaders.common import stored_weights + mixed = {"a": torch.zeros(64, 64, dtype=torch.int8), "b": torch.zeros(32, 32, dtype=torch.bfloat16), + "scale": torch.zeros(64, dtype=torch.float32), "tiny": torch.zeros(2, 2, dtype=torch.float32)} + assert stored_weights(mixed) == "int8+bf16", "1-D scales and a <10% sliver are not the format" + assert stored_weights({"a": torch.zeros(8, 8, dtype=torch.bfloat16)}) == "bf16" + assert stored_weights({"n": 1}) is None + + +@pytest.mark.parametrize("cuda,cap,gpu,slow", [ + ("12.8", (12, 0), True, True), # the rental that ran int8 2x slower + ("13.0", (12, 0), True, False), + ("12.8", (8, 9), True, False), # not Blackwell: not measured, not claimed + ("12.8", (12, 0), False, False), + (None, (12, 0), True, False), # CPU / MPS build +]) +def test_slow_int8_build_names_only_cu12_on_blackwell(monkeypatch, cuda, cap, gpu, slow): + from modules.loaders import common + monkeypatch.setattr(torch.version, "cuda", cuda) + monkeypatch.setattr(torch.cuda, "is_available", lambda: gpu) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: cap) + msg = common.slow_int8_build() + assert bool(msg) is slow + if slow: + assert "--force-reinstall" in msg and "cu130" in msg, "without the flag pip keeps the cu128 build" + + +def _load(monkeypatch, sd, weight_dtype, slow=None): + import comfy.sd + import comfy.utils + import folder_paths + from modules.loaders import gguf_support + from modules.loaders.diffusion_model import nodes + monkeypatch.setattr(folder_paths, "get_full_path_or_raise", lambda kind, name: "/fake/path") + monkeypatch.setattr(gguf_support, "has_gguf_magic", lambda path: False) + monkeypatch.setattr(comfy.utils, "load_torch_file", lambda path, return_metadata: (sd, {})) + monkeypatch.setattr(comfy.sd, "load_diffusion_model_state_dict", lambda sd, model_options, metadata: object()) + monkeypatch.setattr(nodes, "slow_int8_build", lambda: slow) + said = [] + monkeypatch.setattr(nodes.log, "info", lambda src, msg: said.append(("info", msg))) + monkeypatch.setattr(nodes.log, "alert", lambda src, msg: said.append(("alert", msg))) + nodes.FunPackDiffusionModelLoader.execute(model_name="model.safetensors", weight_dtype=weight_dtype, + compute_dtype="bf16", attention="default") + return said + + +def test_an_int8_file_loaded_at_bf16_is_logged_as_int8(monkeypatch): + said = _load(monkeypatch, {"w": torch.zeros(8, 8, dtype=torch.int8)}, "bf16") + line = [m for lvl, m in said if lvl == "info" and "loaded as" in m][0] + assert "weights int8 in file (bf16 asked)" in line + + +def test_a_plain_file_reads_as_before(monkeypatch): + said = _load(monkeypatch, {"w": torch.zeros(8, 8, dtype=torch.bfloat16)}, "bf16", slow="slow build") + line = [m for lvl, m in said if lvl == "info" and "loaded as" in m][0] + assert "weights bf16," in line + assert not [m for lvl, m in said if lvl == "alert"], "the slow-build alert is about int8 files only" + + +def test_an_int8_file_on_a_slow_build_raises_the_alert(monkeypatch): + said = _load(monkeypatch, {"w": torch.zeros(8, 8, dtype=torch.int8)}, "default", slow="slow build") + assert ("alert", "slow build") in said