From fdf39bbd03d4d0d5b9ef271f38f940bafe7b1690 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:40:23 +0300 Subject: [PATCH] fix: H3 image conditioning reaches the model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent, silent breaks made an H3 i2v scene generate from the prompt alone. 1. The Editor always runs split_by_transitions, and the per-scene conditionings REPLACE the single entry Studio built — but they were re-encoded with no reference_image and no h3_references. Both the frame-0 keyframe pin and the Qwen vision presentation were gone before the sampler looked, even for a one-scene project. The split now carries the anchor onto scene 0 and the references onto every scene. Gated to H3: on LTX the anchor rides in the latent, and adding Gemma3 vision to scene 0 would change existing output. 2. A MiniMax H3 Image to Video node hands its first/last frame pins out on its CONDITIONING output, which had nowhere to go — the sampler's positive comes from Studio. New optional h3_keyframes input on the Chain Sampler salvages them: first frame onto the opening scene, last frame re-indexed onto the closing scene's own final frame (the source node's frame_count is its own, and a stale index would land mid-clip where PackedLayout refuses it). The editor auto-wires it for H3 projects. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 23 ++++++++ conditioning.py | 36 +++++++++++- docs/MovieEditor.md | 20 ++++++- movie_editor/backend/builder.py | 4 ++ movie_editor/backend/pipeline_wiring.py | 23 ++++++-- samplers.py | 73 ++++++++++++++++++++++- tests/test_minimax_h3_sampler.py | 77 +++++++++++++++++++++++++ tests/test_minimax_h3_studio.py | 43 ++++++++++++++ 8 files changed, 289 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a4432d5..8b383df3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ ## [Unreleased] +### Fixed +- **H3 image conditioning actually reaches the model.** Two independent breaks, both silent, + both ending with a generation that ignored the input image entirely and left the prompt to + carry the whole shot. The Movie Editor always splits a run into per-scene conditionings, + and that split re-encoded every scene *without* the anchor image or the ref2va references — + so neither the keyframe pin nor the `` presentation ever reached the sampler, + even for a single-scene project. The split now carries the anchor onto the opening scene + and the references onto every scene. LTX projects are untouched: there the anchor travels + in the latent, and adding vision to scene 1 would change existing output. +- **A wired `MiniMax H3 Image to Video` node no longer throws its image away.** That node + carries its first/last frame pins on its CONDITIONING output, which this pipeline dropped + because the sampler's positive comes from Studio. New optional **Chain Sampler · + h3_keyframes** input (auto-wired for H3 projects) salvages the pins — first frame onto the + opening scene, last frame re-indexed onto the closing scene's own final frame. + +### Changed +- **Guide strengths span the full 0..1 range.** Mid-scene guide, identity pin, prior-scene + guide, the per-guide strength field, and the JoyAI memory floor were clamped to 0.25–0.5. + That band is the measured audio-safe sweet spot, not a physical limit; it stays in the + tooltips and out of the code. +- The H3 `v2a_grad_scale` warning chip is gone. The knob does nothing without JoyAI audio + memory on any model, so flagging it as an H3 limitation blamed the wrong thing. + ## [3.5.0] - 2026-08-04 A second model family. FunPack was built around LTX-2 / LTXAV and now supports MiniMax H3 diff --git a/conditioning.py b/conditioning.py index 7295f8ad..a8160ee4 100644 --- a/conditioning.py +++ b/conditioning.py @@ -6938,7 +6938,25 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): scene_effects.append(effect) return scene_texts, scene_effects - def _v2_transition_scene_conditionings(self, clip, scene_texts, scene_effects=None, encode_cache=None): + def _v2_transition_scene_conditionings(self, clip, scene_texts, scene_effects=None, encode_cache=None, + reference_image=None, h3_references=None): + """Per-scene conditionings for the Chain Sampler, one entry per scene. + + These REPLACE the single-entry conditioning built above, so anything that lived on + that entry has to be re-established here or it never reaches the sampler. That is + why the visual conditioning travels in: + + * ``reference_image`` — the i2v anchor, H3 only. Scene 0 only, matching how the chain + reads it (later scenes continue from the previous scene's output unless they carry + their own anchor). It has to be re-encoded, not copied: on H3 the anchor is presented + to Qwen inside the tokenize call, so a scene encoded without it is textually blind to + the image no matter what metadata is attached afterwards. LTX is left alone — there + the anchor reaches the model through the latent, and adding Gemma3 vision to scene 0 + would change what every existing multi-scene LTX project generates. + * ``h3_references`` — ref2va reference media. EVERY scene: a reference identity holds + across the whole chain, and the ```` ordinals in a scene's prompt only + resolve if that scene's own encode presented them. + """ if clip is None: return None scene_texts = list(scene_texts or []) @@ -6946,6 +6964,12 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): return None scene_effects = list(scene_effects or []) + try: + from . import minimax_h3 as h3mod + except ImportError: + import minimax_h3 as h3mod + anchor_image = reference_image if h3mod.is_h3_clip(clip) else None + conditionings = [] scene_count = len(scene_texts) seen_texts = {} @@ -6953,7 +6977,11 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): seen_count = seen_texts.get(scene_text, 0) seen_texts[scene_text] = seen_count + 1 encode_text = f"Returning to an earlier scene: {scene_text}" if seen_count > 0 else scene_text - cond, meta, _ = self._v2_encode_prompt(clip, encode_text, encode_cache=encode_cache) + scene_image = anchor_image if scene_index == 0 else None + cond, meta, _ = self._v2_encode_prompt( + clip, encode_text, encode_cache=encode_cache, + reference_image=scene_image, h3_references=h3_references, + ) if not isinstance(cond, torch.Tensor): return None scene_meta = dict(meta) if isinstance(meta, dict) else {"pooled_output": None} @@ -12866,6 +12894,10 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): if scene_texts: scene_conditionings = self._v2_transition_scene_conditionings( clip, scene_texts, scene_effects=split_scene_effects, encode_cache=encode_cache, + # The per-scene entries replace `output_conditioning` wholesale, so H3's + # visual conditioning has to be re-established here or the anchor image + # and the ref2va references never reach the sampler at all. + reference_image=source_image, h3_references=_h3_references, ) if scene_conditionings is not None: output_conditioning = self._v2_apply_scene_seed_metadata( diff --git a/docs/MovieEditor.md b/docs/MovieEditor.md index 8d624e9b..385ae1a6 100644 --- a/docs/MovieEditor.md +++ b/docs/MovieEditor.md @@ -681,8 +681,24 @@ without complaint: `minimax_h3_fl2va_*` for text-to-video and anchor/keyframe sc using both at once in one scene means one of them is untrained conditioning. Text encoder and both VAEs are shared. Needs ComfyUI v0.30.0 or newer. -**Inert on H3:** Best-FaceID identity transfer, Bounded Attention, segmented detailing, the -and the second pass's latent op all depend on LTX transformer structure H3 does not have. +**Getting an image into an H3 scene — two routes, both supported:** + +1. **Timeline image → `Studio · source_image`** (the default wiring for an image_processing + loader). Studio presents it to Qwen and hands the pixels to the Chain Sampler, which pins + it as the opening scene's frame-0 keyframe. This is the route the Editor's own image + scenes use. +2. **A `MiniMax H3 Image to Video` node's `first_frame` / `last_frame`.** That node emits its + pins on its **CONDITIONING** output — which used to go nowhere, because the sampler's + positive comes from Studio, so the image was silently discarded. Wire that output to + **Chain Sampler · h3_keyframes** (auto-wired for new projects) and the pins survive: a + first_frame lands on scene 1, a last_frame on the last scene's final frame. Only the pins + are read — write the prompt in Studio as usual. + +The run prints `h3_keyframe_anchor(...)` or `h3_wired_keyframes(...)` in its scene mechanics. +If neither appears, the model never saw the image. + +**Inert on H3:** Best-FaceID identity transfer, Bounded Attention, segmented detailing, and +the second pass's latent op all depend on LTX transformer structure H3 does not have. The Chain Sampler switches them off and says why; the main window warns before the run. **Wrong:** Install stuck → **Cancel install**. Pip security policy errors → install packs manually in terminal. After Manager install, restart before pack install. diff --git a/movie_editor/backend/builder.py b/movie_editor/backend/builder.py index cddca8fc..543e68aa 100644 --- a/movie_editor/backend/builder.py +++ b/movie_editor/backend/builder.py @@ -136,6 +136,10 @@ FAMILIES: dict[str, dict] = { ("audiodec", "vae", "VAE", True), # optional: only needed to encode AUDIO ref2va references ("sampler", "audio_vae", "VAE", False), + # optional: a MiniMax H3 Image to Video node emits its first/last frame pins + # on its CONDITIONING output, which this pipeline otherwise drops (the + # sampler's positive comes from Studio). Wiring it here keeps the pins. + ("sampler", "h3_keyframes", "CONDITIONING", False), ), }, }, diff --git a/movie_editor/backend/pipeline_wiring.py b/movie_editor/backend/pipeline_wiring.py index 85d0527a..7b6d58e2 100644 --- a/movie_editor/backend/pipeline_wiring.py +++ b/movie_editor/backend/pipeline_wiring.py @@ -103,6 +103,10 @@ PORT_TO_OPEN_CORE: dict[str, tuple[str, str]] = { # # Everything not listed is inherited, so a rule added for LTXAV reaches H3 too. _H3_LATENT_PORT = "FunPackLTXAVSceneChainSampler.latent_template" +# A MiniMax H3 Image to Video node in the latent role also emits CONDITIONING carrying its +# first_frame / last_frame pins. Studio owns the sampler's positive, so that conditioning has +# nowhere to go and the image is silently lost; this port is where the pins are salvaged. +_H3_KEYFRAME_PORT = "FunPackLTXAVSceneChainSampler.h3_keyframes" FAMILY_WIRING: dict[str, dict] = { "ltxav": {}, @@ -111,19 +115,27 @@ FAMILY_WIRING: dict[str, dict] = { "audio_vae": [("VAE", None, "VAEDecodeAudio.vae"), ("VAE", None, "FunPackLTXAVSceneChainSampler.audio_vae")], "audio_encoder": [], - "empty_latent": [("LATENT", None, _H3_LATENT_PORT)], - "video_latent": [("LATENT", None, _H3_LATENT_PORT)], + "empty_latent": [("LATENT", None, _H3_LATENT_PORT), + ("CONDITIONING", None, _H3_KEYFRAME_PORT)], + "video_latent": [("LATENT", None, _H3_LATENT_PORT), + ("CONDITIONING", None, _H3_KEYFRAME_PORT)], "image_processing": [("IMAGE", None, "FunPackStudio.source_image")], }, - "type_chain_terminals": {"LATENT": [_H3_LATENT_PORT]}, + "type_chain_terminals": {"LATENT": [_H3_LATENT_PORT], + "CONDITIONING": [_H3_KEYFRAME_PORT]}, "default_wires": { "audio_vae": {"VAE": "port:VAEDecodeAudio.vae"}, "audio_encoder": {}, - "empty_latent": {"LATENT": "port:" + _H3_LATENT_PORT}, - "video_latent": {"LATENT": "port:" + _H3_LATENT_PORT}, + # both outputs of the H3 latent node are wired by default: the AV latent, and the + # keyframe pins that come with it when the node has a first/last frame image + "empty_latent": {"LATENT": "port:" + _H3_LATENT_PORT, + "CONDITIONING": "port:" + _H3_KEYFRAME_PORT}, + "video_latent": {"LATENT": "port:" + _H3_LATENT_PORT, + "CONDITIONING": "port:" + _H3_KEYFRAME_PORT}, }, "port_labels": { _H3_LATENT_PORT: "Chain Sampler · latent_template (H3 AV latent)", + _H3_KEYFRAME_PORT: "Chain Sampler · h3_keyframes (first/last frame pins)", "VAEDecodeAudio.vae": "VAE Decode Audio · vae", "FunPackLTXAVSceneChainSampler.audio_vae": "Chain Sampler · audio_vae (ref2va audio)", }, @@ -134,6 +146,7 @@ FAMILY_WIRING: dict[str, dict] = { "FunPackStudio.source_image": ("studio", "source_image"), "FunPackLTXAVSceneChainSampler.vae": ("sampler", "vae"), _H3_LATENT_PORT: ("sampler", "latent_template"), + _H3_KEYFRAME_PORT: ("sampler", "h3_keyframes"), "FunPackLTXAVSceneChainSampler.audio_vae": ("sampler", "audio_vae"), "VAEDecodeAudio.vae": ("audiodec", "vae"), }, diff --git a/samplers.py b/samplers.py index 5676071b..e3af50c2 100644 --- a/samplers.py +++ b/samplers.py @@ -2351,6 +2351,9 @@ class FunPackLTXAVSceneChainSampler: "audio_vae": ("VAE", { "tooltip": "MiniMax H3 only: the audio VAE, needed to encode AUDIO reference media for ref2va (a voice or ambience clip the generation should sound like). Studio lists the references and bakes their