From 3549907a6a33f4d3771bdca5d65d50623f1ba668 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:17:33 +0000 Subject: [PATCH] 5.1.0: review fixes (interrupt in Noise merging, identity bank probe passes, explorer ratings per model family, NaN guide position), version and changelog Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Tw9fjYZhHqk196LykrWy3T --- CHANGELOG.md | 14 +++++++++++--- README.md | 2 +- modules/conditioning/guide_frames/nodes.py | 6 ++++-- .../guide_frames/tests/test_guide_frames.py | 5 +++++ modules/sampling/attention_bandit/__init__.py | 2 ++ modules/sampling/identity_bank/__init__.py | 7 ++++--- modules/sampling/noise_merge/__init__.py | 3 +++ pyproject.toml | 2 +- 8 files changed, 31 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 632d3fa0..93635123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,23 @@ # Changelog -## [Unreleased] +## [5.1.0] "Classy Chopper" - 2026-10-27 ### Changed +- **On LTX only Noise merging, the Attention explorer and Taste are offered for now.** Every other sampling and guidance enhancement (REINS, Block repeat, Identity bank, Guide frames, Shot memory, Camera move, Temporal style and the rest) is held back there until it has been proven on a real LTX run; H3 is unchanged. A model holds a module back with a `hides:` trait, so taking one off the list in `modules/models/ltx/__init__.py` (`HELD_BACK`) is all it takes to bring it back. Saved settings are kept; the Log says "held back for now". - **Load Media decodes only what it keeps**: a prepared video decodes just the kept frames and shapes each as it comes, and a picture input reads one frame, so a long clip no longer sits whole in memory. - **A clip named in Japanese (or any non-ASCII name) no longer fails a run on a machine whose console is not UTF-8.** - **Restoring a saved pipeline no longer adds another preset's input slots** (an Image to Video loader landing in a Video to Video pipeline made a picture reference fail with "identity: there is no such slot"). - **The ComfyUI log survives a restart**: the console is kept on disk as it prints, and the Log window has a "Previous run" button for the run before the last start. The Log also says how much RAM and GPU memory is in use when a LoRA is applied and when sampling starts, so a run the box kills for memory leaves a trace. - **Shot camera and SLA are H3-only**: models announce `shot_prompts` and `sparse_attention` (only H3 does), shot camera requires the first and the loader's SLA fields the second, so both are hidden when an LTX model is picked. +### Fixed + +- **Cancelling during a Noise merging second pass now cancels** instead of being swallowed as a failed merge. +- **Identity bank's lock** no longer counts probe passes (such as Noise merging's second answer) as steps of the run. +- **Attention explorer ratings made on H3 no longer teach LTX** under the same Taste key: the two model families keep their ratings apart. +- **Guide frames** refuses a NaN position instead of crashing. + ### Added - **Block settings know the model**: the file's header gives its block count, the Blocks hint says "This model has blocks 0-47", and a list is cut to those blocks when you commit it. @@ -22,8 +30,8 @@ - **LTX Image to Video takes its shape from the aspect source**: the picture on the timeline gives the clip its aspect ratio and the project's Width x Height give the pixel budget (new **FunUI Aspect Size** node), as on H3. With no such picture the project's own shape is used. It is not the start frame: that is reference picture 1. - **Flip width and height** button in Project settings, and on the Video input card. - **Video input** (Project and Scene tabs): reference videos (and, on LTX, a video picked as the scene start, fed as frames) from the Media bin can be got ready before the model sees them, like a video loader node: target width and height (fill, fit with bars, or stretch; rounded to a multiple), edge crop in percent, frame rate (project's, the video's own or a number), skip first N frames, cut last N, every Nth frame, frame cap. The sound is cut to match. One set per project and one per scene; a dropdown in Project settings says whose is used (Automatic: the project's when switched on, otherwise the scene's). Off by default, so existing projects load videos as before. Works for any model whose pipeline takes reference videos (MiniMax H3 today). Tested on made-up frames and an encoded file, not yet on a rental. -- **Attention explorer** (needs a Taste key): each run tries a small attention change (sharper, as is or looser) per depth group of blocks, and your ratings decide which changes stay; how far a change goes learns from ratings too (or type your own). The Log names the exact blocks made sharper or looser. Also learns, per run: how strongly each group attends to your prompt ("Prompt focus"), and the clip's four time bands and its sound separately ("Clip regions"). A rating counts more for runs of a similar prompt, and a combination you liked on this prompt is reused (with some groups re-drawn). Zero extra model passes. Untested on a real H3. -- **Noise merging** (Settings > Engine > Sampling): "Merge" can also run a whole second video beside yours from the second seed ("Second run": one extra pass at every step up to the last one named, blended at the steps you name), or mix the run's starting noise with a second seed's (no extra cost, variance kept). Or, at the steps you name, the step is answered a second time on the same picture with different noise (a second seed), and the two pictures are blended at your strength. Merge order picks which one is the base. The second seed is random each run unless you type one. Picture and sound are both merged (the sound can be left out with "Merge sound too"). "Second pass new noise" sets how much of the step's own noise the second pass swaps for the second seed's (1 all, lower keeps more of your own). Costs one extra model pass per merged step. Untested on a real H3. +- **Attention explorer** (needs a Taste key): each run tries a small attention change (sharper, as is or looser) per depth group of blocks, and your ratings decide which changes stay; how far a change goes learns from ratings too (or type your own). The Log names the exact blocks made sharper or looser. Also learns, per run: how strongly each group attends to your prompt ("Prompt focus"), and the clip's four time bands and its sound separately ("Clip regions"). A rating counts more for runs of a similar prompt, and a combination you liked on this prompt is reused (with some groups re-drawn). Zero extra model passes. Verified on H3 and LTX. +- **Noise merging** (Settings > Engine > Sampling): "Merge" can also run a whole second video beside yours from the second seed ("Second run": one extra pass at every step up to the last one named, blended at the steps you name), or mix the run's starting noise with a second seed's (no extra cost, variance kept). Or, at the steps you name, the step is answered a second time on the same picture with different noise (a second seed), and the two pictures are blended at your strength. Merge order picks which one is the base. The second seed is random each run unless you type one. Picture and sound are both merged (the sound can be left out with "Merge sound too"). "Second pass new noise" sets how much of the step's own noise the second pass swaps for the second seed's (1 all, lower keeps more of your own). Costs one extra model pass per merged step. Verified on H3 and LTX. - **SLA rising sparsity** (diffusion model loader, `sla_sparsity_start`): skip fewer attention blocks on early steps and more on late ones, in a straight line up to `sla_sparsity`. Off at 0. One real H3 render (4 steps, 0.70 to 0.90): quality held, perhaps slightly better. Idea from GSRS (arXiv 2610.21129); only the schedule is taken, not its other parts. ## [5.0.3] "Classy Chopper" - 2026-10-25 diff --git a/README.md b/README.md index 628180aa..07a4dbda 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ FunUI uses your existing ComfyUI/PyTorch install. - **MiniMax H3**: the model v5 is built and tested around. - **Stable Diffusion 1.5**: experimental. -- **LTX 2.3**: tested on a real model: Video to Video (direct) and Image to Video (start from References), each falling back to text to video when nothing is fed in; separate model files; LoRAs; Noise merging. Other enhancements are not tested on LTX yet. Please report problems in Issues. +- **LTX 2.3**: tested on a real model: Video to Video (direct) and Image to Video (start from References), each falling back to text to video when nothing is fed in; separate model files; LoRAs; Noise merging. Noise merging and the Attention explorer are the enhancements offered on LTX; the rest are held back there until proven. Please report problems in Issues. ## Documentation diff --git a/modules/conditioning/guide_frames/nodes.py b/modules/conditioning/guide_frames/nodes.py index 7655748b..ba836171 100644 --- a/modules/conditioning/guide_frames/nodes.py +++ b/modules/conditioning/guide_frames/nodes.py @@ -37,8 +37,10 @@ def parse(text): out = [] for item in raw[:MAX_FRAMES]: try: - out.append((str(item["media_id"]), min(max(float(item.get("at", 0.5)), 0.0), 1.0), - min(max(float(item.get("strength", 0.25)), 0.0), 10.0))) + at, strength = float(item.get("at", 0.5)), float(item.get("strength", 0.25)) + if at != at or strength != strength: + raise ValueError("not a number") # JSON allows NaN; a position cannot be one + out.append((str(item["media_id"]), min(max(at, 0.0), 1.0), min(max(strength, 0.0), 10.0))) except (KeyError, TypeError, ValueError, AttributeError): return f"{item!r} is not {{media_id, at, strength}}" return out diff --git a/modules/conditioning/guide_frames/tests/test_guide_frames.py b/modules/conditioning/guide_frames/tests/test_guide_frames.py index 716cc07d..bc5b12b5 100644 --- a/modules/conditioning/guide_frames/tests/test_guide_frames.py +++ b/modules/conditioning/guide_frames/tests/test_guide_frames.py @@ -61,3 +61,8 @@ def test_a_picture_gone_from_the_library_is_left_out_and_bad_lists_are_refused(m for bad in ("not json", '{"a": 1}', '[{"nothing": 1}]'): with pytest.raises(RuntimeError): nodes.FunUIGuideFrames.execute(pos, neg, _Vae(), lat, bad) + + +def test_a_nan_position_is_refused_not_crashed(): + from modules.conditioning.guide_frames import nodes + assert isinstance(nodes.parse('[{"media_id": "a", "at": NaN}]'), str) diff --git a/modules/sampling/attention_bandit/__init__.py b/modules/sampling/attention_bandit/__init__.py index a257c180..a39f716c 100644 --- a/modules/sampling/attention_bandit/__init__.py +++ b/modules/sampling/attention_bandit/__init__.py @@ -368,6 +368,8 @@ def install(patcher, values, key): dm = getattr(patcher.model, "diffusion_model", None) paired = hasattr(dm, "transformer_blocks") # picture and sound attend in separate calls, one stream each video_width, sound_width = getattr(dm, "inner_dim", None), getattr(dm, "audio_inner_dim", None) + if paired: + signature.append(1) # ratings made on a packed-sequence model (H3) must not teach a paired one if focus and paired: focus = False # the prompt's rows are not one run of the sequence here: that part is H3's diff --git a/modules/sampling/identity_bank/__init__.py b/modules/sampling/identity_bank/__init__.py index 7206bcd8..42e9bd52 100644 --- a/modules/sampling/identity_bank/__init__.py +++ b/modules/sampling/identity_bank/__init__.py @@ -158,9 +158,10 @@ def install(patcher, values, key): def apply_model(executor, x, t, *args, **kwargs): out = executor(x, t, *args, **kwargs) - if not locking: - return out - split = streams.video_of(out, streams.model_args(args, kwargs)) + named = streams.model_args(args, kwargs) + if not locking or dit_hooks.probing(named.get("transformer_options") or {}): + return out # a probe pass (noise merging's second answer) is not a step of this run + split = streams.video_of(out, named) if split is None: return out video = split[0].detach().float() diff --git a/modules/sampling/noise_merge/__init__.py b/modules/sampling/noise_merge/__init__.py index efb178d0..8b382953 100644 --- a/modules/sampling/noise_merge/__init__.py +++ b/modules/sampling/noise_merge/__init__.py @@ -23,6 +23,7 @@ import random import time import torch +from comfy.model_management import InterruptProcessingException from comfy.patcher_extension import WrappersMP from ..._core import dit_hooks, log, streams @@ -256,6 +257,8 @@ def install(patcher, values, key): n = merged.shape[-1] - base_tail.shape[-1] merged = torch.cat([merged[..., :n], merge(base_tail, other_tail, strength).to(merged.dtype)], dim=-1) return merged + except InterruptProcessingException: + raise # Cancel stops the run, it is not a failed merge except Exception as exc: # noqa: BLE001 log.broke("FunUI Noise merging", exc, "merging the second answer") return out diff --git a/pyproject.toml b/pyproject.toml index 6cfc1239..505e842f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ComfyUI-FunPack" # Unique identifier for your node. Immutable after creation. description = "An AI movie studio inside ComfyUI: a browser-based studio with timeline, per-scene prompts and one-click montage generation, plus a self-improving engine that learns from your ratings. Every feature is a module, so a broken one is simply absent." -version = "5.0.3" # Custom Node version. Must be semantically versioned. +version = "5.1.0" # Custom Node version. Must be semantically versioned. # Release codename ("Classy Chopper" for 5.x) lives in core/update.py:CODENAMES, keyed by major, shown in Settings > About. license = { text = "GNU GPL v3" } dependencies = [] # Filled in from requirements.txt