nodes/olivv-cs-ComfyUI-FunPack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

maint: remove the H3 render gains

Browse Source 
Six widgets (three write gains, prompt loudness, taste push, learned/manual
mode) and the whole rating loop behind them. Never proved out on the rental:
picture gain past +-0.1 adds distortion and latent noise, the rest changed
nothing legible, and the audio gain was never worth testing.

The final-layer picture detail SURVIVES and now reads its own widget — it was
the only one of the set that showed a real effect. Relabelled experimental.

Two things fall out with them. The refinement key no longer holds any sampler
value, so key scoping has nothing left to scope and goes too. And the card's
ownership table now reads dependsVals as well as dependsValue: the Studio rows
use the plural form, so absolute_strength was printing under steer_mode
"relative", where it does nothing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
digital-garbage
2026-08-27 14:30:19 +03:00
co-authored by Claude Opus 5
parent 8374838e46
commit aebc286042
14 changed files with 76 additions and 1960 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conditioning.py
-242
View File
@@ -3073,237 +3073,6 @@ class FunPackVideoRefiner:
# SIGMA REFINEMENT
# =========================================================================
# ── rating-learned H3 render gains ──────────────────────────────────────
#
# Four scalars — how hard each block writes into the video, prompt and audio rows, and
# how loudly the prompt is read. They are learned rather than typed: four dimensions is
# SMALLER than the sigma profile's search space, which already converges on ratings
# alone, so asking the user to hand-tune them would be asking them to do a search the
# machine can do.
#
# Same update as the sigma profile — perturb, generate, rate, move toward the
# perturbation that earned reward — with one difference: the sigma profile is centred on
# 0, so its applied value IS its perturbation. These are centred on 1.0, so the
# perturbation is `applied - value` and that is what the reward is credited to.
#
# Unlike anything in conditioning space these transfer between prompts: 0.85 means the
# same thing on every prompt, so a value learned on one run is worth something on the
# next. That is the property that makes them worth learning at all.
H3_GAIN_KEYS = ("video", "prompt", "audio", "prompt_scale", "refiner_bias",
"video_detail")
# Which rated AXIS each gain answers to. A rating carries three per-axis signals in
# [-1, +1] alongside its overall reward, and they say far more than the scalar does:
# "Missing concept" is reward +0.10 (quality was fine) but concept_signal -1.00. Read as
# the scalar, that rating nudges every gain slightly TOWARD the run that missed the
# prompt. Read per axis, it pushes the two prompt gains hard away from it and leaves the
# audio gain alone, which is what the user actually said.
#
# Signals share the reward's [-1, +1] scale, so they drop straight into the same update.
# An empty tuple means no axis is rated for that gain — audio has none — so it falls back
# to the overall reward rather than inventing a signal for it.
H3_GAIN_AXES = {
"video": ("detail_signal", "quality_signal"),
"prompt": ("concept_signal",),
"prompt_scale": ("concept_signal",),
"audio": (),
"refiner_bias": (),
# The final-layer read strength: the same axes as the block gain it is the
# audio-safe twin of. `prompt_time` is deliberately absent from these keys — its
# off value (0.0) sits at the far end of its own range rather than in the middle,
# so exploring around it would step straight into "the prompt is pure noise".
# It stays a manual dial until a rental run says what a good value looks like.
"video_detail": ("detail_signal", "quality_signal"),
}
H3_GAIN_MIN = 0.60
H3_GAIN_MAX = 1.40
H3_GAIN_LR = 0.08
H3_GAIN_EXPLORE_BASE = 0.05
H3_GAIN_EXPLORE_MIN = 0.012
H3_GAIN_EXPLORE_MAX = 0.09
#: gains that are not multipliers. `refiner_bias` is how far to push the prompt rows
#: along the taste direction learned from liked runs, as a fraction of a typical prompt
#: row's magnitude — so its neutral value is 0.0, not 1.0, and it is signed: a negative
#: value pushes AWAY from what was liked, which the loop must be able to choose.
H3_GAIN_CENTRES = {"refiner_bias": 0.0}
H3_GAIN_BOUNDS = {"refiner_bias": (-0.30, 0.30)}
def _h3_gain_centre(self, key):
return float(self.H3_GAIN_CENTRES.get(key, 1.0))
def _h3_gain_bounds(self, key):
return self.H3_GAIN_BOUNDS.get(key, (self.H3_GAIN_MIN, self.H3_GAIN_MAX))
def _h3_gain_explore(self, key, base):
"""This key's exploration width, scaled to the span it explores.
One stored `explore` drives every gain, so a key with a narrower range has to shrink
it proportionally — otherwise its perturbations clip against the bounds and the
reward gets credited to a move that never fully happened.
"""
low, high = self._h3_gain_bounds(key)
span = float(high) - float(low)
reference = self.H3_GAIN_MAX - self.H3_GAIN_MIN
return float(base) * (span / reference if reference else 1.0)
def _ensure_h3_gain_state(self, global_state: dict):
state = global_state.setdefault("h3_gains", {})
neutral = {k: self._h3_gain_centre(k) for k in self.H3_GAIN_KEYS}
state.setdefault("values", dict(neutral))
state.setdefault("last_applied", dict(neutral))
state.setdefault("iterations", 0)
state.setdefault("reward_ema", 0.0)
state.setdefault("explore", self.H3_GAIN_EXPLORE_BASE)
for key in self.H3_GAIN_KEYS: # a key added later starts neutral
state["values"].setdefault(key, neutral[key])
state["last_applied"].setdefault(key, neutral[key])
return state
def _h3_gain_credit(self, key, rating_profile):
"""The reward THIS gain should be credited with, from the axes it answers to.
Recovering four parameters from one scalar works, but slowly — it relies on the
random perturbations decorrelating over many runs. The per-axis signals are already
on every rating, so each rating can teach the parameter it is actually about.
Falls back to the overall reward when a gain has no rated axis (audio), or when a
profile carries no signals at all.
"""
axes = self.H3_GAIN_AXES.get(key, ())
signals = []
for axis in axes:
value = rating_profile.get(axis)
if value is None:
continue
try:
signals.append(float(value))
except (TypeError, ValueError):
continue
if not signals:
return float(rating_profile.get("reward", 0.0) or 0.0)
# The WORST axis, not the average. These axes are conjunctive — the video gain has to
# satisfy detail AND quality — so a satisfied axis must not mask a failed one.
# Averaging let it: "Missing details" is detail -1.00 with quality +0.85, which
# averages to -0.08 and barely moves the gain the rating is actually about.
return min(signals)
def _v2_update_h3_gains(self, global_state: dict, rating_profile: dict):
"""Credit this rating to the perturbation that was actually rendered."""
if not isinstance(global_state, dict) or not isinstance(rating_profile, dict):
return None
if rating_profile.get("skip_learning"):
return None
state = self._ensure_h3_gain_state(global_state)
reward = float(rating_profile.get("reward", 0.0) or 0.0)
values, applied = state["values"], state["last_applied"]
moved = []
for key in self.H3_GAIN_KEYS:
centre = float(values.get(key, self._h3_gain_centre(key)))
perturbation = float(applied.get(key, centre)) - centre
if perturbation == 0.0:
continue
credit = self._h3_gain_credit(key, rating_profile)
low, high = self._h3_gain_bounds(key)
updated = min(high, max(
low, centre + perturbation * credit * self.H3_GAIN_LR))
if updated != centre:
values[key] = updated
moved.append(f"{key}={updated:.3f}")
state["reward_ema"] = 0.85 * float(state.get("reward_ema", 0.0)) + 0.15 * reward
# Landing well narrows the search; landing badly widens it.
decay = 0.97 if reward > 0 else 1.02
state["explore"] = min(self.H3_GAIN_EXPLORE_MAX, max(
self.H3_GAIN_EXPLORE_MIN, float(state.get("explore", self.H3_GAIN_EXPLORE_BASE)) * decay))
state["iterations"] = int(state.get("iterations", 0)) + 1
return ", ".join(moved) if moved else None
def _v2_h3_gains_for_run(self, global_state: dict, seed=0, explore=True):
"""The gains to render with, and the record of what was rendered.
Exploration is what makes the next rating informative — without a perturbation there
is nothing for the reward to be credited to, and the profile can never move. Seeded
from the run's seed so a repeated run repeats its perturbation.
"""
if not isinstance(global_state, dict):
return {k: self._h3_gain_centre(k) for k in self.H3_GAIN_KEYS}
state = self._ensure_h3_gain_state(global_state)
values = state["values"]
base = float(state.get("explore", self.H3_GAIN_EXPLORE_BASE))
out = {}
if explore:
rng = np.random.RandomState(int(seed) % (2 ** 32))
for key in self.H3_GAIN_KEYS:
centre = float(values.get(key, self._h3_gain_centre(key)))
if explore:
centre += float(rng.normal(0.0, self._h3_gain_explore(key, base)))
low, high = self._h3_gain_bounds(key)
out[key] = min(high, max(low, centre))
state["last_applied"] = dict(out)
return out
#: where the learned taste direction rides to the sampler
H3_TASTE_DIR_META = "funpack_h3_taste_dir"
#: liked runs needed before the direction is worth pushing along — the same threshold
#: the conditioning-space consumers of `liked_dir` already use, not a new one
H3_TASTE_DIR_MIN_COUNT = 3
def _h3_taste_direction(self, global_state):
"""The liked direction, as a UNIT vector the sampler can project into refiner space.
This is the same `liked_dir` that conditioning ascent already steers along — the
difference is where it lands. Steering edits the Qwen tensor, which then goes through
condition_proj and the two refiner blocks and a final RMSNorm: the blocks mix it
across tokens and the norm scales its magnitude away, so the direction that arrives
is not the direction that was learned. Handing the sampler the raw direction lets it
add the same thing AFTER the refiner, where it lands as sent.
Unit, not scaled: the magnitude belongs to the refiner's own space and is decided at
apply time. How far to push is the `refiner_bias` gain, and that is learned.
"""
if not isinstance(global_state, dict):
return None
slot = global_state.get("liked_dir") or {}
if int(slot.get("direction_count", 0)) < self.H3_TASTE_DIR_MIN_COUNT:
return None
try:
vector = serializable_to_tensor(slot.get("direction")).float().reshape(-1)
except Exception: # noqa: BLE001
return None
norm = float(vector.norm().item())
if not norm or norm != norm: # zero, or NaN from a corrupted slot
return None
return (vector / norm).cpu()
def _v2_tag_h3_gains(self, conditioning_list, global_state, seed=0):
"""Attach this run's render gains to entry 0 so the sampler can apply them."""
try:
gains = self._v2_h3_gains_for_run(global_state, seed=seed)
except Exception as error: # noqa: BLE001
_log.failed("FunPackStudio", "H3 render gains", error,
"this run renders at the model's trained strengths")
return conditioning_list
neutral = all(abs(v - self._h3_gain_centre(k)) < 1e-6 for k, v in gains.items())
direction = self._h3_taste_direction(global_state)
if (not gains or neutral) and direction is None:
return conditioning_list
out = []
for index, entry in enumerate(conditioning_list or []):
if index == 0 and isinstance(entry, (list, tuple)) and len(entry) >= 2 \
and isinstance(entry[1], dict):
meta = dict(entry[1])
meta["funpack_h3_gains"] = dict(gains)
if direction is not None:
meta[self.H3_TASTE_DIR_META] = direction
out.append([entry[0], meta])
else:
out.append(entry)
_log.feature(
"FunPackStudio", "Rating-learned render gains", True,
", ".join(f"{k}={gains[k]:.3f}" for k in self.H3_GAIN_KEYS if k in gains)
+ ("; taste direction attached" if direction is not None
else f"; no taste direction yet (needs {self.H3_TASTE_DIR_MIN_COUNT} liked runs)")
+ " (learned from ratings, perturbed to keep learning; the Chain Sampler applies "
"them and its h3_gain_* widgets are ignored unless h3_gain_mode is manual)")
return out
def _ensure_sigma_state_defaults(self, global_adaptive: dict):
global_adaptive.setdefault("sigma_profile", [0.0] * 32)
@@ -4998,7 +4767,6 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
successful-seed-memory status string. Shared by the project key and every non-default
key bound to the scene's shortcuts (multi-key rating)."""
self._v2_update_phrase_memory(target_global, scene_run, profile, iter_num, axis_feedback)
self._v2_update_h3_gains(target_global, profile)
self._v2_update_concept_delta_memory(target_global, scene_run, profile)
self._v2_update_concept_pair_dirs(target_global, scene_run, profile)
seed_memory_status = self._v2_update_successful_seed_memory(
@@ -10136,7 +9904,6 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
value_guidance=True, latent=None, seed_output_connected=False,
steer_mode="relative", absolute_strength=0.6,
h3_phrase_emphasis=False,
h3_render_gains=True,
prompt_enhance=False, prompt_enhance_system="",
prompt_enhance_temperature=0.7, prompt_enhance_top_p=0.92,
prompt_enhance_max_length=400, prompt_enhance_thinking=False,
@@ -11201,14 +10968,6 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
f"entry 1 is steered and the other "
f"{len(positive_conditioning) - 1} pass through unchanged. They used to be "
f"discarded, which cut everything after the first entry out of the prompt.")
# Rating-learned render gains ride the conditioning to the sampler, the same bridge
# H3 token weighting uses: Studio owns the key and the learning, the sampler owns the
# application. Perturbed here, once per run, so the NEXT rating has something to
# credit — a run with no perturbation teaches nothing.
if h3_render_gains:
output_conditioning = self._v2_tag_h3_gains(
output_conditioning, global_state, seed=int(_seed or 0))
# Batch Training packs N variant entries (tagged 'funpack_batch_variant') onto the FINAL
# conditioning just before each return below — so it wraps whatever output_conditioning
# ends up being: single-scene OR multi-scene (transition split). Works WITH transitions.
@@ -13512,7 +13271,6 @@ class FunPackStudio:
refinement_key_input=refinement_key_input,
positive_conditioning=positive_conditioning,
h3_phrase_emphasis=bool(rf.get("h3_phrase_emphasis", False)),
h3_render_gains=bool(rf.get("h3_render_gains", True)),
prompt_enhance=bool(rf.get("prompt_enhance", False)),
prompt_enhance_system=str(rf.get("prompt_enhance_system", "") or ""),
prompt_enhance_temperature=float(rf.get("prompt_enhance_temperature", 0.7)),
minimax_h3.py
+5 -223
View File
@@ -1311,26 +1311,12 @@ def extend_token_tags(meta, added_tokens, tag=1):
return out
# ── per-modality AdaLN gain ──────────────────────────────────────────────────
# ── patching H3 in place ────────────────────────────────────────────────────
#
# Every DiT block carries its own ``AdalnProj(t_dim, hidden, expand=6, modalities=3)``:
# six modulation vectors — shift/scale/gate for attention, the same three for the MLP —
# and THREE copies of each, one per modality. Rows are indexed ``t_row * 3 + tag``, so a
# modality's rows across every timestep are the slice ``[tag::3]``.
#
# The GATE is what scales a block's output as it is written back into a row range. Scaling
# one modality's gate is therefore per-block, per-stream write gain: how much all 50 blocks
# are allowed to say about the prompt rows, or the audio rows, independently of the video.
# At 1.0 nothing is touched and the tuple is returned as it arrived.
#
# WHAT THIS CANNOT SEPARATE: keyframe pins and reference images carry the VIDEO tag (0).
# They differ from the target video by TIMESTEP row, not by tag — and only while the
# condition's noise-augmentation timestep is above the video's, which stops being true
# later in the schedule. So the video gain moves the anchor with it. Splitting them needs
# the layout's row mapping, which AdalnProj never sees.
MODALITY_TAGS = {"video": 0, "text": 1, "audio": 2}
_ADALN_GATE_CHUNKS = (2, 5) # shift_msa scale_msa GATE_msa shift_mlp scale_mlp GATE_mlp
_ADALN_MODALITIES = 3
# Patching a module's FORWARD rather than replacing the module is what keeps the state
# dict intact: a wrapper module renames every weight beneath it, and ComfyUI records
# weight names off the LIVE tree, so an unwrap by any other patcher sharing this model
# would leave a restore walking a path that no longer exists.
def unwrapped_forward(module):
@@ -1360,81 +1346,6 @@ def patch_module_forward(patcher, key, wrapper):
patcher.add_object_patch(f"{key}.forward", wrapper)
class AdalnEdit:
"""Wraps one block's ``adaln_proj.forward`` and scales its gates per modality.
Deliberately NOT an ``nn.Module``. Replacing a module with a wrapper module renames
every weight beneath it in the state dict (``adaln_proj.weight`` becomes
``adaln_proj.inner.weight``), and ComfyUI records weight keys off the LIVE tree while
the patch is applied — so an unwrap by any other patcher sharing this model leaves a
restore walking a ``.inner`` that is no longer there. Patching ``forward`` leaves the
tree and the state dict exactly as the checkpoint built them.
"""
#: marks an already-installed wrapper, so a re-install unwraps instead of nesting
_funpack_wrapper = True
def __init__(self, inner, gains):
self.inner = inner
# The ORIGINAL callable, captured before the patch is applied. Calling
# ``inner.forward`` at call time would re-enter this wrapper once it is installed.
self.original = unwrapped_forward(inner)
self.gains = {int(t): float(g) for t, g in (gains or {}).items() if float(g) != 1.0}
def __call__(self, t_emb):
if getattr(self.inner, "modalities", 1) != _ADALN_MODALITIES:
return self.original(t_emb) # FinalLayer shares the class, not the row meaning
out = self.original(t_emb)
if not self.gains:
return out
out = list(out)
for idx in _ADALN_GATE_CHUNKS:
gate = out[idx]
# chunk() returns views onto one buffer — never scale in place.
scaled = gate.clone()
for tag, factor in self.gains.items():
scaled[tag::_ADALN_MODALITIES] = gate[tag::_ADALN_MODALITIES] * factor
out[idx] = scaled
return tuple(out)
def apply_adaln_edits(model, gains=None):
"""Attach per-modality gate gains to every DiT block. Returns (model, note).
Uses ``add_object_patch`` rather than replacing methods by name: ComfyUI restores
object patches on unpatch, so nothing survives the run. Degradation that outlives a
reset has been a real bug here before, and this is the mechanism that prevents it.
`gains` maps a name in MODALITY_TAGS to a float. 1.0 (or absent) means untouched, and
an all-1.0 request returns the model unchanged rather than cloning it.
"""
live = {MODALITY_TAGS[name]: float(value)
for name, value in (gains or {}).items()
if name in MODALITY_TAGS and float(value) != 1.0}
if not live:
return model, None
try:
blocks = model.get_model_object("diffusion_model.blocks")
except Exception as error: # not H3, or a layout we do not know
return model, f"AdaLN gain skipped — no DiT blocks to patch ({error})."
patched = model.clone()
count = 0
for index in range(len(blocks)):
key = f"diffusion_model.blocks.{index}.adaln_proj"
try:
inner = patched.get_model_object(key)
except Exception:
continue
patch_module_forward(patched, key, AdalnEdit(inner, live))
count += 1
if not count:
return model, "AdaLN gain skipped — no adaln_proj found on the blocks."
named = ", ".join(f"{n}={v:g}" for n, v in sorted((gains or {}).items()) if float(v) != 1.0)
return patched, (f"AdaLN modality gain on {count} blocks ({named}). Keyframe pins and "
f"reference images ride the VIDEO tag, so the video gain moves them too.")
# ── final layer: the one video-only edit ─────────────────────────────────────
#
# Every block shares one attention pass, so anything written into the video rows is read by
@@ -1498,132 +1409,3 @@ def apply_final_video_scale(model, scale):
patch_module_forward(patched, key, FinalLayerVideoScale(inner, scale))
return patched, (f"Final-layer video scale {float(scale):g} — past the last attention, "
f"so the audio stream cannot see it.")
# ── token-refiner output ─────────────────────────────────────────────────────
#
# `context` (Qwen, [1, L, 5120]) reaches the DiT through
# ``condition_proj`` (Linear 5120 -> 5376) then ``token_refiner`` (2 blocks + RMSNorm).
# Everything FunPack steers today happens BEFORE that, on the Qwen tensor, which means the
# refiner then processes the edit — its two attention blocks mix it across tokens and its
# final RMSNorm renormalizes the magnitude away. The direction that lands is not the
# direction that was learned.
#
# Editing the refiner's OUTPUT lands unchanged, and lands in the space the 50 blocks
# actually consume. Two edits are supported:
#
# scale — multiply the rows. The blocks read text through attention, so magnitude is how
# loudly the prompt competes with the picture for attention. Distinct from
# h3_gain_prompt, which controls how much each block WRITES BACK into these rows.
# bias — add a direction. A single [hidden] vector added to every prompt row is
# PROMPT-LENGTH INDEPENDENT, which per-position edits are not (Qwen does not pad,
# so H3's sequence length changes with every prompt edit). That is the shape a
# learned direction has to have here to transfer between prompts.
#
# Both are confined to a row range: the `<Picture N>` label and vision-block rows sit at the
# head of this span and are not ours to steer.
class TokenRefinerEdit:
"""Wraps ``token_refiner.forward`` and edits its output over a row range.
Deliberately NOT an ``nn.Module``. Replacing the module with a wrapper module renames
the checkpoint's own keys — ``token_refiner.blocks.*`` becomes
``token_refiner.inner.blocks.*`` — and ComfyUI's ``patch_model`` applies object patches
BEFORE it loads weights, so every weight gets recorded under the ``.inner`` name. On the
way out it restores weights before it unwraps, and with several patchers alive off one
shared model (one clone per scene, plus one for the AdaLN gains) another patcher can
unwrap first — leaving a restore that walks ``.inner`` on a real ``TokenRefiner``.
Patching the forward leaves the tree and the state dict as the checkpoint built them.
"""
#: marks an already-installed wrapper, so a re-install unwraps instead of nesting
_funpack_wrapper = True
def __init__(self, inner, scale=1.0, bias=None, row_start=0, row_end=None,
bias_relative=False):
self.inner = inner
# The ORIGINAL callable, captured before the patch is applied. Calling
# ``inner.forward`` at call time would re-enter this wrapper once installed.
self.original = unwrapped_forward(inner)
self.scale = float(scale)
self.bias = bias
self.row_start = int(row_start)
self.row_end = row_end
# A RELATIVE bias is a unit vector rescaled, at apply time, to a fraction of the
# span's own mean row norm. A learned direction has no natural magnitude in this
# space — the norms here belong to the checkpoint, not to anything the loop measured
# — so an absolute vector would be either inert or catastrophic with no way to tell
# which in advance. Relative makes the strength mean "this fraction of a typical
# prompt row", which is the same thing on every prompt and every checkpoint.
self.bias_relative = bool(bias_relative)
def __call__(self, x, *args, **kwargs):
out = self.original(x, *args, **kwargs)
try:
rows = out.shape[-2]
start = max(0, min(int(self.row_start), rows))
end = rows if self.row_end is None else max(start, min(int(self.row_end), rows))
if end <= start:
return out
out = out.clone()
span = out[..., start:end, :]
if self.scale != 1.0:
span = span * self.scale
if self.bias is not None:
bias = self.bias.to(device=span.device, dtype=span.dtype)
if self.bias_relative:
scale = span.float().norm(dim=-1).mean()
bias = bias * scale.to(bias.dtype)
span = span + bias
out[..., start:end, :] = span
except Exception: # a refiner shape we do not recognise: leave it alone
return self.original(x, *args, **kwargs)
return out
def project_into_refiner_space(model, direction):
"""Map a conditioning-space (Qwen) direction into refiner space with ``condition_proj``.
The bias, not just the weight, would move the origin — a DIRECTION is a difference of two
points, so only the linear part carries over. Returns None when the projection cannot be
read or the width does not match.
"""
try:
proj = model.get_model_object("diffusion_model.condition_proj")
weight = proj.weight
except Exception:
return None
try:
vector = direction.reshape(-1).to(dtype=weight.dtype, device=weight.device)
if vector.shape[0] != weight.shape[1]:
return None
return torch.mv(weight, vector)
except Exception:
return None
def apply_token_refiner_edit(model, scale=1.0, bias=None, row_start=0, row_end=None,
bias_relative=False):
"""Attach a scale and/or bias to the token refiner's output. Returns (model, note).
``add_object_patch`` again, so ComfyUI restores it on unpatch and nothing outlives the
run. A scale of 1.0 with no bias returns the model unchanged rather than cloning it.
"""
if float(scale) == 1.0 and bias is None:
return model, None
key = "diffusion_model.token_refiner"
try:
inner = model.get_model_object(key)
except Exception as error:
return model, f"token-refiner edit skipped — no token_refiner to patch ({error})."
patched = model.clone()
patch_module_forward(patched, key, TokenRefinerEdit(inner, scale, bias, row_start,
row_end, bias_relative=bias_relative))
parts = []
if float(scale) != 1.0:
parts.append(f"scale={float(scale):g}")
if bias is not None:
parts.append("bias (relative)" if bias_relative else "bias")
where = f"rows {row_start}:{row_end if row_end is not None else 'end'}"
return patched, f"token-refiner edit ({', '.join(parts)}) on {where}."
movie_editor/backend/projects.py
+2 -6
View File
@@ -10,7 +10,7 @@ import time
from typing import Optional
from . import config
from .timeline import Project, Scene, without_key_scoped
from .timeline import Project, Scene
def _path(project_id: str):
@@ -39,11 +39,7 @@ def get(project_id: str) -> Optional[Project]:
p = _path(project_id)
if not p.exists():
return None
stored = json.loads(p.read_text())
# Only here, on the way off disk: the rating-learned strengths come from the refinement
# key, so an opened project must not carry yesterday's copy of them.
stored["sampler_inputs"] = without_key_scoped(stored.get("sampler_inputs"))
return Project.from_dict(stored)
return Project.from_dict(json.loads(p.read_text()))
def save(project: Project) -> Project:
movie_editor/backend/settings_card.py
+18 -24
View File
@@ -158,9 +158,13 @@ def _pass_rows(cfg: dict) -> list:
# Two rules, both the user's: skip a value that equals the widget's own default, and skip a
# value whose feature is off.
#: knob -> (the toggle that owns it, the value of that toggle which switches it on).
#: knob -> (the toggle that owns it, the values of that toggle which switch it on).
#: None means "any truthy value". Mirrored from the Editor's own dependsOn wiring, and a
#: test regenerates this from engine_settings.js so the two cannot drift apart.
#:
#: A tuple, not one value: `absolute_strength` belongs to `steer_mode` and is live for BOTH
#: "absolute" and "both". Reading only the singular `dependsValue` missed the plural
#: `dependsVals` the Studio rows use, so it printed under `relative`, where it does nothing.
#: knobs the Editor renders by hand rather than from its knob table, so they carry no
#: dependsOn for the test below to find. Kept separate so the generated half stays generated.
_OWNED_EXTRA = {
@@ -169,7 +173,7 @@ _OWNED_EXTRA = {
_OWNED_BY = {
**_OWNED_EXTRA,
"absolute_strength": ("steer_mode", None),
"absolute_strength": ("steer_mode", ('absolute', 'both')),
"alg_anchor_sigma_threshold": ("alg_anchor", None),
"alg_anchor_strength": ("alg_anchor", None),
"alg_guide_blur_sigma_threshold": ("alg_blur_guides", None),
@@ -191,12 +195,6 @@ _OWNED_BY = {
"dynashift_threshold": ("dynashift", None),
"embed_guidance_source": ("embed_guidance", None),
"embed_guidance_strength": ("embed_guidance", None),
"h3_gain_audio": ("h3_gain_mode", "manual"),
"h3_gain_prompt": ("h3_gain_mode", "manual"),
"h3_gain_video": ("h3_gain_mode", "manual"),
"h3_prompt_scale": ("h3_gain_mode", "manual"),
"h3_taste_bias": ("h3_gain_mode", "manual"),
"h3_video_detail": ("h3_gain_mode", "manual"),
"id_strength": ("identity_transfer_enabled", None),
"joyai_audio_memory": ("joyai_memory", None),
"joyai_fix_frames": ("joyai_memory", None),
@@ -204,16 +202,16 @@ _OWNED_BY = {
"joyai_memory_size": ("joyai_memory", None),
"joyai_memory_strength": ("joyai_memory", None),
"mid_scene_guide_strength": ("mid_scene_guide", None),
"negative_erase_mode": ("negative_erase", None),
"negative_erase_renorm": ("negative_erase", None),
"negative_erase_strength": ("negative_erase", None),
"negative_erase_mode": ("negative_erase", ('true',)),
"negative_erase_renorm": ("negative_erase", ('true',)),
"negative_erase_strength": ("negative_erase", ('true',)),
"output_guidance_strength": ("output_guidance", None),
"phase_scale": ("identity_transfer_enabled", None),
"prompt_enhance_max_length": ("prompt_enhance", None),
"prompt_enhance_system": ("prompt_enhance", None),
"prompt_enhance_temperature": ("prompt_enhance", None),
"prompt_enhance_thinking": ("prompt_enhance", None),
"prompt_enhance_top_p": ("prompt_enhance", None),
"prompt_enhance_max_length": ("prompt_enhance", ('true',)),
"prompt_enhance_system": ("prompt_enhance", ('true',)),
"prompt_enhance_temperature": ("prompt_enhance", ('true',)),
"prompt_enhance_thinking": ("prompt_enhance", ('true',)),
"prompt_enhance_top_p": ("prompt_enhance", ('true',)),
"score_slider_strength": ("score_slider", None),
"source_id": ("identity_transfer_enabled", None),
"v2a_grad_scale": ("joyai_audio_memory", None),
@@ -262,14 +260,7 @@ _EDITOR_DEFAULTS = {
"embed_guidance_strength": 0.02,
"frame_overlap": 16,
"h3_audio_clock": False,
"h3_gain_audio": 1.0,
"h3_gain_mode": "learned",
"h3_gain_prompt": 1.0,
"h3_gain_video": 1.0,
"h3_phrase_emphasis": False,
"h3_prompt_scale": 1.0,
"h3_render_gains": True,
"h3_taste_bias": 0.0,
"h3_video_detail": 1.0,
"id_strength": 1.0,
"identity_transfer_enabled": False,
@@ -352,7 +343,10 @@ def _switched_on(name: str, values: dict, defaults: dict, _seen=None) -> bool:
return True
seen.add(owner)
current = values.get(owner, defaults.get(owner))
live = bool(current) if wanted is None else str(current) == str(wanted)
# Lowercased both sides: the Editor writes JS literals, so a boolean owner arrives as
# "true" while Python's own str(True) is "True".
live = bool(current) if wanted is None \
else str(current).lower() in {str(w).lower() for w in wanted}
return live and _switched_on(owner, values, defaults, seen)
movie_editor/backend/timeline.py
-31
View File
@@ -544,37 +544,6 @@ class Scene:
return project.frame_rate
#: Sampler knobs whose values belong to the REFINEMENT KEY, not to the project.
#:
#: These are learned from ratings and live in the key's own state. A copy in the project file
#: was a second source of truth that outlived the key it came from: deleting every key left
#: the old values still applying, with no way to clear them but typing the neutral value back
#: in by hand.
#:
#: TWO THINGS THIS IS NOT, both learned the hard way:
#:
#: 1. It is not `h3_gain_mode`. A mode is a choice, and clearing it on open would throw
#: away a setting the user made on purpose, which is not what "untie from the project
#: file" meant.
#: 2. It does not apply in MANUAL mode. There, these are hand-typed settings like any other
#: and must persist; there is no learned state to reset, because nothing learned them.
KEY_SCOPED_SAMPLER_INPUTS = frozenset({
"h3_gain_video", "h3_gain_prompt", "h3_gain_audio",
"h3_prompt_scale", "h3_taste_bias", "h3_video_detail",
})
def without_key_scoped(raw) -> dict:
"""Drop the rating-learned values, so a project opens at whatever the key now says.
Applied when a project is READ FROM DISK — never on the save round-trip. `from_dict`
runs on both, and stripping there meant the editor sent a value, got a project back
without it, and showed the default again: the knobs could not be set at all.
"""
values = dict(raw or {})
if str(values.get("h3_gain_mode", "learned")).strip().lower() == "manual":
return values
return {k: v for k, v in values.items() if k not in KEY_SCOPED_SAMPLER_INPUTS}
@dataclass
movie_editor/frontend/engine_defaults.test.js
-43
View File
@@ -16,7 +16,6 @@ const assert = require("node:assert");
function writeSamplerInput(store, k, value, immediate) {
if (value === k.default) {
store.unsetSamplerInput(k.name);
(k.clearsOnDefault || []).forEach((name) => store.unsetSamplerInput(name));
if (immediate) store.flushSave?.();
return;
}
@@ -93,7 +92,6 @@ test("booleans clear when set back to their default", () => {
assert.deepStrictEqual(s.calls[0], ["unset", "segmented_detailing"]);
});
// ── handing ownership back ──────────────────────────────────────────────────
//
// Reported from a rental as a question, which is the best kind: "save as manual, load it,
@@ -102,50 +100,9 @@ test("booleans clear when set back to their default", () => {
// hide in learned mode), until the next time the project was opened. Correct output, state
// nobody could see. That is the failure this whole rule exists to prevent.
const fs = require("node:fs");
const path = require("node:path");
const MODE = {
name: "h3_gain_mode",
default: "learned",
clearsOnDefault: ["h3_gain_video", "h3_gain_prompt", "h3_gain_audio",
"h3_prompt_scale", "h3_taste_bias", "h3_video_detail"],
};
test("switching H3 render gains back to learned clears the manual values with it", () => {
const s = fakeStore();
writeSamplerInput(s, MODE, "learned", true);
const cleared = s.calls.filter((c) => c[0] === "unset").map((c) => c[1]);
assert.deepStrictEqual(cleared, ["h3_gain_mode", ...MODE.clearsOnDefault]);
});
test("choosing manual clears nothing — those values are about to be used", () => {
const s = fakeStore();
writeSamplerInput(s, MODE, "manual", true);
assert.ok(!s.calls.some((c) => c[0] === "unset"));
assert.deepStrictEqual(s.calls[0], ["setNow", "h3_gain_mode", "manual"]);
});
test("an ordinary knob returning to default clears only itself", () => {
const s = fakeStore();
writeSamplerInput(s, GAIN, 1.0, true);
assert.deepStrictEqual(s.calls.filter((c) => c[0] === "unset"), [["unset", "h3_gain_video"]]);
});
test("the rule under test is the one the renderer actually runs", () => {
// This file re-implements writeSamplerInput so it can run without a DOM. That copy is
// free to drift from the real one, and a green suite would say nothing.
const src = fs.readFileSync(path.join(__dirname, "engine_settings.js"), "utf8");
assert.match(src, /\(k\.clearsOnDefault \|\| \[\]\)\.forEach/);
assert.match(src, /clearsOnDefault: \["h3_gain_video"/);
});
test("the mode clears exactly the values the backend treats as key-scoped", () => {
// Two halves of one rule in two languages: the browser clears them on the switch, the
// server clears them on load. A value in one list and not the other is a leak.
const src = fs.readFileSync(
path.join(__dirname, "..", "backend", "timeline.py"), "utf8");
const block = src.split("KEY_SCOPED_SAMPLER_INPUTS = frozenset({")[1].split("})")[0];
const backend = [...block.matchAll(/"([a-z0-9_]+)"/g)].map((m) => m[1]).sort();
assert.deepStrictEqual(backend, [...MODE.clearsOnDefault].sort());
});
movie_editor/frontend/engine_settings.js
+3 -34
View File
@@ -133,8 +133,6 @@
{ name: "prompt_enhance_thinking", label: "Enhancer thinking mode", kind: "bool", default: false,
dependsOn: "prompt_enhance", dependsVals: [true],
hint: "Let the model reason before answering, if it supports it. Slower, and the reasoning is stripped from the result." },
{ name: "h3_render_gains", label: "Rating-learned render gains (H3)", kind: "bool", default: true,
hint: "Learns six render strengths from your ratings alone: how hard each block writes into the picture, the prompt and the soundtrack, how loudly the prompt is read, how far to push the prompt toward what you have rated well, and how much detail the picture gets. Six numbers is a smaller search than the sigma schedule already learns, so there is nothing to tune by hand — each run is rendered slightly off the learned values so the next rating has something to credit. The taste push needs at least 3 liked runs on the key before it does anything. Free. The learned values live in the refinement key, not in this project — clearing your keys resets them. Off leaves the model at its trained strengths; the Chain Sampler's h3_gain_*, h3_taste_bias and h3_video_detail widgets then do nothing unless its h3_gain_mode is set to manual." }
];
function parseStudioSettings(p) {
@@ -325,31 +323,8 @@
{ name: "h3_audio_clock", label: "H3 audio clock (few-step audio)", kind: "bool", default: false,
hint: "MiniMax H3 only: removes audio distortion on few-step turbo schedules. Free.",
detail: "H3 runs video and audio on different flow schedules but only one sigma grid reaches the sampler, so audio drifts further the bigger the step. This corrects it for one multiply per step. Exact with FunPack Distilled Flow, Hybrid Euler 2S and euler; leave it off with res_multistep / dpmpp_2m and the rest of the multistep family, which already absorb most of the error. No effect when both sigma shifts are equal, or on two-evals-per-step samplers." },
{ name: "h3_gain_mode", label: "H3 render gains", kind: "combo", choices: ["learned", "manual"], default: "learned",
clearsOnDefault: ["h3_gain_video", "h3_gain_prompt", "h3_gain_audio",
"h3_prompt_scale", "h3_taste_bias", "h3_video_detail"],
hint: "MiniMax H3 only. 'learned' takes six render strengths from your ratings and ignores the dials below. 'Prompt is settled' is not one of them and works in either mode. 'manual' uses the dials as typed.",
detail: "Learned needs the Refiner's 'Rating-learned render gains (H3)' left on — with it off, or before anything is rated, learned mode renders at the model's trained strengths and the dials still do nothing. Switch to manual to drive them yourself." },
{ name: "h3_gain_video", label: "Write gain · picture", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01,
dependsOn: "h3_gain_mode", dependsValue: "manual",
hint: "How hard each block writes into the picture rows. Below 1.0 is a calmer, less-committed image; above pushes harder. 1.0 = untouched. Free.",
detail: "Keyframe pins and reference images ride the same modality tag as the video, so this moves the anchor with it." },
{ name: "h3_gain_prompt", label: "Write gain · prompt", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01,
dependsOn: "h3_gain_mode", dependsValue: "manual",
hint: "How much each block rewrites the prompt as it draws. Lower it when a scene slowly stops matching what you asked for; 0.0 freezes the text at its encoded value. 1.0 = untouched. Free." },
{ name: "h3_gain_audio", label: "Write gain · soundtrack", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01,
dependsOn: "h3_gain_mode", dependsValue: "manual",
hint: "How hard each block writes into the audio rows. Below 1.0 builds the soundtrack more conservatively. 1.0 = untouched. Free." },
{ name: "h3_prompt_scale", label: "Prompt loudness", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01,
dependsOn: "h3_gain_mode", dependsValue: "manual",
hint: "How loudly the prompt is READ, as opposed to how much it is rewritten. Above 1.0 it is harder to ignore; below, the anchor or the reference gets more say. 1.0 = untouched. Free." },
{ name: "h3_taste_bias", label: "Taste push", kind: "float", default: 0.0, min: -0.30, max: 0.30, step: 0.01,
dependsOn: "h3_gain_mode", dependsValue: "manual",
hint: "Pushes the prompt toward what you have rated well. Negative pushes away from it. Needs at least 3 liked runs on the refinement key; without them it does nothing and says so. 0.0 = off. Free.",
detail: "The Refiner already steers along your liked direction, but on H3 that edit is mixed and renormalized by the token refiner before the model reads it. This adds the same direction after the refiner, where it lands as sent. Measured as a fraction of a typical prompt row, so one value means the same thing on every prompt." },
{ name: "h3_video_detail", label: "Picture detail (audio-safe)", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01,
dependsOn: "h3_gain_mode", dependsValue: "manual",
hint: "Makes the picture crisper or softer without changing the sound at all. Above 1.0 = more detail and contrast, below is softer. The step from 1.0 is small — reach for 1.4-1.8 before deciding it does nothing. Free.",
{ name: "h3_video_detail", label: "Picture detail (experimental)", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01,
hint: "Makes the picture crisper or softer without changing the sound at all. Above 1.0 = more detail and contrast, below is softer. The step from 1.0 is small — reach for 1.4-1.8 before deciding it does nothing. Free. Experimental, lightly tested.",
detail: "Every block of the model shares one attention pass, so anything that moves the picture also reaches the soundtrack. This runs after the last one, where there is no path left for it to travel — it is the audio-safe twin of Write gain \u00b7 picture." },
{ name: "segmented_detailing", label: "Segmented detailing (region refine)", kind: "bool", default: false,
hint: "Re-renders small regions like hands at higher detail after each scene. Costs about 15%.",
@@ -482,12 +457,6 @@
function writeSamplerInput(k, value, immediate) {
if (value === k.default) {
S.unsetSamplerInput(k.name);
// A knob whose default hands ownership BACK to something else takes the values it
// owned with it. Without this, switching H3 render gains from manual to learned left
// the six hand-typed values behind: inert, invisible (the dials hide in learned mode),
// and not cleared until the project was next opened. Deferred invisible state is the
// whole problem this rule exists to remove.
(k.clearsOnDefault || []).forEach((name) => S.unsetSamplerInput(name));
if (immediate) S.flushSave?.();
return;
}
@@ -557,7 +526,7 @@
chain_timing: ["frame_overlap", "transition_duration", "use_same_seed", "cut_opening_frames"],
chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"],
chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"],
chain_experimental: ["context_windows", "context_window_length", "context_window_overlap", "context_window_schedule", "context_window_fuse", "context_window_freenoise", "context_window_retain_first", "h3_audio_clock", "h3_gain_mode", "h3_gain_video", "h3_gain_prompt", "h3_gain_audio", "h3_prompt_scale", "h3_taste_bias", "h3_video_detail", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_mode", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"],
chain_experimental: ["context_windows", "context_window_length", "context_window_overlap", "context_window_schedule", "context_window_fuse", "context_window_freenoise", "context_window_retain_first", "h3_audio_clock", "h3_video_detail", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_mode", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"],
};
function countChainView(p, id, st) {
movie_editor/frontend/pipeline_caps.js
+2 -8
View File
@@ -171,7 +171,7 @@
// Left on against an LTX pipeline they are just as silently inert, so they get a chip
// by the same rule — the user should not have to spend a generation to find out.
// key -> why it cannot run off H3.
const H3_RENDER_GAIN_REASON =
const H3_ONLY_REASON =
"The render gains scale H3's per-modality AdaLN gates and its token refiner. LTX has "
+ "neither: one modulation path for all streams, and the prompt reaches it through "
+ "cross-attention instead of a packed text span.";
@@ -183,13 +183,7 @@
// The render gains scale H3's per-modality AdaLN gates and its token refiner. LTX has
// neither — one modulation path, cross-attention instead of a packed text span — so
// there is nothing for these to reach on any other family.
h3_gain_mode: H3_RENDER_GAIN_REASON,
h3_gain_video: H3_RENDER_GAIN_REASON,
h3_gain_prompt: H3_RENDER_GAIN_REASON,
h3_gain_audio: H3_RENDER_GAIN_REASON,
h3_prompt_scale: H3_RENDER_GAIN_REASON,
h3_taste_bias: H3_RENDER_GAIN_REASON,
h3_video_detail: H3_RENDER_GAIN_REASON,
h3_video_detail: H3_ONLY_REASON,
};
// Sub-settings of identity transfer: meaningless wherever the feature itself cannot run,
samplers.py
+3 -236
View File
@@ -3147,33 +3147,9 @@ class FunPackLTXAVSceneChainSampler:
"default": False,
"tooltip": "EXPERIMENTAL, MiniMax H3 only: integrate the audio stream on its OWN flow schedule. H3 denoises video and audio on two different schedules (the MiniMax H3 Sigma Shift node's shift_video 12 / shift_audio 3), but only one sigma grid reaches a sampler, so the model reconciles them by scaling the audio velocity by the slope between the two schedules at the START of each step. That is exact for infinitesimal steps and increasingly wrong as steps get bigger: on a 4-step schedule the last step drives audio roughly 2.5x past where its own schedule puts it, which is heard as distortion. This replaces that start-of-step slope with the one that actually spans the step, so audio lands where its schedule says. Costs one scalar multiply per step — no extra model call. Aimed at few-step schedules (turbo/distilled LoRAs, 4-8 steps), where nothing else fixes this. NO-OP when shift_video and shift_audio are EQUAL — the streams are then on one schedule and there is nothing to correct (it says so on the console). SAMPLER MATTERS, measured against a perfect predictor so the numbers are pure schedule error (audio error as a % of the stream's full range, 4/8/20 steps): WORKS BEST — FunPack Distilled Flow and Hybrid Euler 2S (runs inside their step loop, exact), and stock `euler` (85/38/14% -> 0/0/0%, exact at every step count). PERFORMS POORLY — the higher-order multistep family (`res_multistep`, `dpmpp_2m`, `gradient_estimation`, `ipndm`, `lms`, `deis`): they already absorb most of this error themselves, so the clock helps them at 4 steps (69% -> 21%) but HURTS at 20 (1% -> 15%); leave it off there. NO EFFECT — two-evals-per-step samplers (`heun`, `dpm_2`, `dpmpp_2s_ancestral`, `dpmpp_sde`, `seeds_2`): a model call cannot be tied to a step from outside their loop, so the wrapper detects that on the first call and switches itself off with a console note rather than guessing. Ancestral/SDE samplers additionally add noise to the audio stream, which this does not address. The clock never touches the video stream directly, though on H3 the two share one attention sequence, so a changed audio latent can still shift the video slightly.",
}),
"h3_gain_video": ("FLOAT", {
"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
"tooltip": "EXPERIMENTAL, MiniMax H3 only. Per-block write gain on the VIDEO rows. Every DiT block carries three sets of AdaLN modulation — one per modality (video / prompt / audio) — and the GATE is what scales that block's attention and MLP output as it is written back into a row range. This multiplies the video gate on all 50 blocks: below 1.0 each block contributes less to the picture (softer, calmer, less detail and less motion), above 1.0 more (harder, busier, and past ~1.3 it overcooks). 1.0 = untouched, and the model is not even cloned. Free — one small vector multiply per block, no extra model calls. KEYFRAME PINS AND REFERENCE IMAGES RIDE THE VIDEO TAG, so this moves them with the picture; they are separated from the target video by timestep row, not by modality, and only early in the schedule. Attached with add_object_patch, which ComfyUI restores on unpatch, so nothing survives the run.",
}),
"h3_gain_prompt": ("FLOAT", {
"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
"tooltip": "EXPERIMENTAL, MiniMax H3 only. Per-block write gain on the PROMPT rows (see h3_gain_video for the mechanism). H3 keeps the text in the same sequence as the picture and lets every block rewrite it, so the prompt the model reads at block 50 is not the one the encoder produced. 0.0 freezes the text at its encoded value for the whole forward — the video still attends to it, but it can no longer drift toward what is being drawn. Lower values are the lever to try when a scene slowly stops matching what you asked for; higher values let the text move further with the picture. 1.0 = untouched. Free. The <Picture N> vision rows inside the text span carry the VIDEO tag, not this one, so a reference is not affected.",
}),
"h3_gain_audio": ("FLOAT", {
"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
"tooltip": "EXPERIMENTAL, MiniMax H3 only. Per-block write gain on the AUDIO rows (see h3_gain_video for the mechanism). This is the one modality whose gate can be moved without touching the other two at the point of writing — though the streams still share one attention pass, so a changed audio row is read by the video on the NEXT block. Below 1.0 the soundtrack is built more conservatively. Free. 1.0 = untouched.",
}),
"h3_taste_bias": ("FLOAT", {
"default": 0.0, "min": -0.30, "max": 0.30, "step": 0.01,
"tooltip": "EXPERIMENTAL, MiniMax H3 only. Pushes the prompt toward what you have RATED WELL, in the model's own space. The Refiner already learns a 'liked direction' from your ratings and steers the conditioning along it — but that edit then goes through H3's token refiner, whose two attention blocks mix it across tokens and whose final norm scales its magnitude away, so what the model reads is not what was learned. This adds the same direction AFTER the refiner, where it lands as sent. Measured as a fraction of a typical prompt row's magnitude, so it means the same thing on every prompt and every checkpoint. Negative pushes AWAY from what you liked. Needs a refinement key with at least 3 liked runs; without one it does nothing and says so. Free. IGNORED unless h3_gain_mode is 'manual' — in 'learned' mode this is learned from your ratings like the other gains. 0.0 = off.",
}),
"h3_gain_mode": (["learned", "manual"], {
"default": "learned",
"tooltip": "Where the six H3 render strengths come from. 'learned' (default) takes them from the refinement key: Studio learns them from your ratings alone and tags them onto the conditioning, and the six widgets below are IGNORED — nothing to tune by hand, which is the point. Six scalars is a smaller search than the sigma schedule already learns from ratings, so it converges in tens of rated runs. With no key wired, or before the first rating, 'learned' renders at the model's trained strengths (untouched, and the taste push at 0). 'manual' ignores the learned values and uses the widgets below exactly as set — for deliberately probing one strength, or for a graph with no Refiner in it.",
}),
"h3_prompt_scale": ("FLOAT", {
"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
"tooltip": "EXPERIMENTAL, MiniMax H3 only. How LOUD the prompt is when the picture reads it. H3 puts the Qwen text through condition_proj + a 2-block token refiner (ending in an RMSNorm) before the DiT sees it, and the blocks then read the text through attention — so the magnitude of those refined rows is how strongly the prompt competes for attention against the picture and the reference. This multiplies them: above 1.0 the prompt is harder to ignore, below 1.0 it recedes and the reference or the anchor gets more say. Applied AFTER the refiner's final norm, so it lands as set rather than being renormalized away. 1.0 = untouched, and the model is not cloned. Free — one multiply on the text rows, no extra model calls. Confined to the PROMPT rows: the <Picture N> label and vision block sit ahead of them and are left alone (read from minimax_token_tags; if those cannot be read the whole text span is scaled and the console says so). DIFFERENT from h3_gain_prompt, which controls how much each block WRITES BACK into the text rows — this controls how loudly they are READ.",
}),
"h3_video_detail": ("FLOAT", {
"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
"tooltip": "EXPERIMENTAL, MiniMax H3 only. Makes the picture crisper or softer WITHOUT changing the sound at all. Above 1.0 = more detail and contrast, below 1.0 = softer and calmer. The step from 1.0 is small, so reach for 1.4-1.8 before deciding it does nothing. 1.0 = untouched, and the model is not cloned. This is the audio-safe twin of h3_gain_video: everything earlier in the model shares one attention pass, so a change to the picture reaches the soundtrack, but this runs after the last one — there is no path left for it to travel. Free. UNVALIDATED.",
"tooltip": "EXPERIMENTAL, MiniMax H3 only. Makes the picture crisper or softer WITHOUT changing the sound at all. Above 1.0 = more detail and contrast, below 1.0 = softer and calmer. The step from 1.0 is small, so reach for 1.4-1.8 before deciding it does nothing. 1.0 = untouched, and the model is not cloned. Everything earlier in the model shares one attention pass, so any other change to the picture also reaches the soundtrack; this runs after the last one, where there is no path left for it to travel. Free. EXPERIMENTAL and lightly tested: 1.25 read as more detail and better contrast on one rental run, and nothing beyond that has been measured.",
}),
"alg_anchor": ("BOOLEAN", {
"default": False,
@@ -3987,9 +3963,7 @@ class FunPackLTXAVSceneChainSampler:
# Per-modality AdaLN gain. Sampler-side and self-contained: it reads no refinement
# key, no rating and no Studio state, so it works on a graph with the Refiner absent
# entirely. 1.0 on all three does not even clone the model.
model = self._install_h3_adaln_gains(model, positive)
model = self._install_h3_final_layer(model, positive)
model = self._install_h3_token_refiner(model, positive)
try:
sampled = comfy.sample.sample_custom(
@@ -6666,204 +6640,6 @@ class FunPackLTXAVSceneChainSampler:
w_idx = idx % w
return (w_idx >= (w // 2)).long()
#: what Studio tags onto entry 0 when it has learned gains for this key
H3_GAINS_META = "funpack_h3_gains"
#: and the learned taste direction that rides with them
H3_TASTE_DIR_META = "funpack_h3_taste_dir"
#: the value of each render gain that means "untouched". Not all of them are 1.0:
#: refiner_bias is a signed push along a learned direction, so its neutral is 0.0.
H3_GAIN_NEUTRAL = {"video": 1.0, "prompt": 1.0, "audio": 1.0,
"prompt_scale": 1.0, "refiner_bias": 0.0,
"video_detail": 1.0}
def _h3_render_gains(self, positive):
"""The four render strengths for this run: learned from ratings, or the widgets.
Learned is the default because rating is the only input the user wants to give — four
scalars is a smaller search than the sigma profile already runs on ratings alone, so a
hand-tuned value here is the user doing work the loop can do. `manual` is the explicit
override, and it still works with no Refiner in the graph at all.
Reading the CONDITIONING META rather than a refinement key keeps the boundary intact:
Studio owns the key and the learning, the sampler owns the application — the same
bridge H3 token weighting already uses.
"""
manual = {"video": float(getattr(self, "_h3_gain_video", 1.0)),
"prompt": float(getattr(self, "_h3_gain_prompt", 1.0)),
"audio": float(getattr(self, "_h3_gain_audio", 1.0)),
"prompt_scale": float(getattr(self, "_h3_prompt_scale", 1.0)),
"refiner_bias": float(getattr(self, "_h3_taste_bias", 0.0)),
"video_detail": float(getattr(self, "_h3_video_detail", 1.0))}
if str(getattr(self, "_h3_gain_mode", "learned")).lower() == "manual":
return manual
learned = None
try:
entry = positive[0] if isinstance(positive, list) and positive else None
if isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict):
candidate = entry[1].get(self.H3_GAINS_META)
if isinstance(candidate, dict):
learned = {k: float(v) for k, v in candidate.items()}
except Exception: # noqa: BLE001
learned = None
if learned is None:
# No key, or nothing rated yet. Trained strengths — NOT the widgets, which in
# this mode are not what the user is steering with.
if any(value != self.H3_GAIN_NEUTRAL[key] for key, value in manual.items()):
_log.feature(
"FunPackSceneChain", "Rating-learned render gains", False,
"nothing rated on this key yet, so the run uses the model's trained "
"strengths. The h3_gain_* widgets are IGNORED in 'learned' mode — set "
"h3_gain_mode to 'manual' to use them as typed.")
return dict(self.H3_GAIN_NEUTRAL)
return {k: learned.get(k, self.H3_GAIN_NEUTRAL[k]) for k in manual}
def _h3_prompt_rows(self, positive):
"""Where the PROMPT starts inside the text span, from the conditioning's own tags.
Returns (start, note). The `<Picture N>` label and a reference's vision block sit at
the head of the span and are not ours to edit, so everything the refiner edit does is
confined to the rows after them.
"""
try:
from . import h3_token_weights as _tw
except ImportError:
import h3_token_weights as _tw
try:
entry = positive[0] if isinstance(positive, list) and positive else None
meta = entry[1] if isinstance(entry, (list, tuple)) and len(entry) >= 2 else {}
cond = entry[0] if entry is not None else None
cond_len = int(cond.shape[1]) if hasattr(cond, "shape") and cond.dim() >= 2 else 0
region = _tw.prompt_region(meta.get("minimax_token_tags"), cond_len)
if region:
return int(region[0]), None
if cond_len:
return 0, ("the prompt/reference boundary could not be read from the token "
"tags, so the WHOLE text span is edited — including a reference's "
"vision block, if one is wired")
except Exception as error: # noqa: BLE001
_log.failed("FunPackSceneChain", "h3 token-refiner row range", error,
"the whole text span is edited instead of just the prompt")
return 0, None
def _h3_taste_bias_vector(self, model, positive, strength):
"""The learned taste direction, projected into the refiner's own space.
Studio learns `liked_dir` in Qwen space and already steers the conditioning along it.
That edit then goes through condition_proj, two refiner blocks and a final RMSNorm —
which mixes it across tokens and normalizes its magnitude away, so the direction that
reaches the 50 DiT blocks is not the one that was learned. Adding it AFTER the
refiner lands it unchanged, in the space the blocks actually read.
Added to every prompt row equally, which is what makes it transferable: Qwen does not
pad, so the row COUNT changes with every prompt edit and a per-position bias would
mean something different on each run.
Returns a unit vector for `TokenRefinerEdit(bias_relative=True)` to rescale, or None.
"""
if not strength:
return None
try:
from . import minimax_h3 as h3mod
except ImportError:
import minimax_h3 as h3mod
direction = None
try:
entry = positive[0] if isinstance(positive, list) and positive else None
if isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict):
direction = entry[1].get(self.H3_TASTE_DIR_META)
except Exception: # noqa: BLE001
direction = None
if direction is None:
_log.feature(
"FunPackSceneChain", "Taste push", False,
"the conditioning carries no learned direction. "
"It needs a refinement key with at least 3 liked runs on it.")
return None
projected = h3mod.project_into_refiner_space(model, direction)
if projected is None:
_log.feature(
"FunPackSceneChain", "Taste push", False,
"the learned direction does not fit the token refiner's space. "
"Its width does not match condition_proj.")
return None
norm = float(projected.float().norm().item())
if not norm or norm != norm:
return None
return (projected.float() / norm) * float(strength)
def _install_h3_token_refiner(self, model, positive):
"""Edit the token refiner's OUTPUT: prompt loudness, and the learned taste push.
One patch for both. They act on the same rows through the same wrapper, so applying
them separately would nest two wrappers and make the second one's `bias_relative`
rescale read a span the first had already scaled.
"""
gains = self._h3_render_gains(positive)
scale = float(gains.get("prompt_scale", 1.0))
strength = float(gains.get("refiner_bias", 0.0))
if scale == 1.0 and not strength:
return model
try:
from . import minimax_h3 as h3mod
except ImportError:
import minimax_h3 as h3mod
if not h3mod.is_h3_model(model):
_log.feature(
"FunPackSceneChain", "Token-refiner edit", False,
"not a MiniMax H3 model. There is no token refiner to edit.")
return model
start, note = self._h3_prompt_rows(positive)
if note:
_log.feature("FunPackSceneChain", "Token-refiner edit", True, note)
bias = self._h3_taste_bias_vector(model, positive, strength)
if scale == 1.0 and bias is None:
return model
try:
patched, applied = h3mod.apply_token_refiner_edit(
model, scale=scale, bias=bias, row_start=start, bias_relative=bias is not None)
except Exception as error: # noqa: BLE001
_log.failed("FunPackSceneChain", "h3 token-refiner edit", error,
"the prompt is read at its trained strength and unbiased")
return model
if applied:
print(f"[FunPackSceneChain] {applied}")
return patched
def _install_h3_adaln_gains(self, model, positive):
"""Scale every DiT block's AdaLN gates per modality, from the three sampler widgets.
Deliberately NOT part of the refinement path. This is a visual-behaviour op, so it
lives on the sampler (Studio produces conditioning; the sampler decides how the model
renders it) and it reads only its own widgets. Nothing here consults a refinement key,
a rating, or a learned direction — turning conditioning steering off does not turn
this off, and turning this on does not require the Refiner to be wired at all.
Returns `model` untouched when all three gains are 1.0, which is every default run.
"""
gains = self._h3_render_gains(positive)
if all(gains.get(k, 1.0) == 1.0 for k in ("video", "prompt", "audio")):
return model
try:
from . import minimax_h3 as h3mod
except ImportError:
import minimax_h3 as h3mod
if not h3mod.is_h3_model(model):
_log.feature(
"FunPackSceneChain", "AdaLN render gains", False,
"not a MiniMax H3 model. The per-modality gates only exist there.")
return model
# MODALITY_TAGS names the text modality "text"; the widget says "prompt" because that
# is what it is to the person setting it.
tagged = {"video": gains["video"], "text": gains["prompt"], "audio": gains["audio"]}
try:
patched, note = h3mod.apply_adaln_edits(model, tagged)
except Exception as error: # noqa: BLE001
_log.failed("FunPackSceneChain", "AdaLN modality gain", error,
"the blocks write at their trained strength for every modality")
return model
if note:
print(f"[FunPackSceneChain] {note}")
return patched
def _install_h3_final_layer(self, model, positive):
@@ -6873,8 +6649,7 @@ class FunPackLTXAVSceneChainSampler:
blocks share one attention sequence. This one cannot: there is no attention after
the final layer, so the audio branch is untouched by construction.
"""
gains = self._h3_render_gains(positive)
detail = float(gains.get("video_detail", 1.0))
detail = float(getattr(self, "_h3_video_detail", 1.0))
if detail == 1.0:
return model
try:
@@ -7222,8 +6997,6 @@ class FunPackLTXAVSceneChainSampler:
second_pass=False, second_pass_op="none", second_pass_sigmas=None,
second_pass_sampler=None,
h3_audio_clock=False,
h3_gain_video=1.0, h3_gain_prompt=1.0, h3_gain_audio=1.0,
h3_prompt_scale=1.0, h3_taste_bias=0.0, h3_gain_mode="learned",
h3_video_detail=1.0,
audio_vae=None, h3_keyframes=None,
unique_id=None, prompt=None):
@@ -7239,14 +7012,8 @@ class FunPackLTXAVSceneChainSampler:
# happened last run is reported again rather than deduped away forever.
_log.begin_run()
self._is_h3 = self._set_stream_axes(model)
# Read once per run, consumed by _install_h3_adaln_gains at each scene's sample call.
self._h3_gain_video = max(0.0, min(2.0, float(h3_gain_video)))
self._h3_gain_prompt = max(0.0, min(2.0, float(h3_gain_prompt)))
self._h3_gain_audio = max(0.0, min(2.0, float(h3_gain_audio)))
self._h3_prompt_scale = max(0.0, min(2.0, float(h3_prompt_scale)))
self._h3_taste_bias = max(-0.30, min(0.30, float(h3_taste_bias)))
# Read once per run, consumed by _install_h3_final_layer at each scene's sample call.
self._h3_video_detail = max(0.0, min(2.0, float(h3_video_detail)))
self._h3_gain_mode = str(h3_gain_mode or "learned").strip().lower()
# The gate every rating-driven wrapper shares. On H3 it is read off the schedule's
# own base grid; on LTX it stays the absolute-sigma gate it was validated with.
_steer_ramp = _make_steer_ramp(sigmas, self._is_h3)
tests/test_h3_adaln_gain.py
-269
View File
@@ -1,269 +0,0 @@
"""Per-modality AdaLN gain: per-block write strength for video / prompt / audio.
Every H3 DiT block carries `AdalnProj(t_dim, hidden, expand=6, modalities=3)` — shift,
scale and gate for attention and for the MLP, in three copies, one per modality. Rows are
indexed `t_row * 3 + tag` (video 0, text 1, audio 2), so one modality's rows across every
timestep are the slice `[tag::3]`. The GATE scales what a block writes back into a row
range, which makes it per-block, per-stream write gain.
This is a sampler-side visual-behaviour op. It reads no refinement key, no rating and no
learned direction, so it is unaffected by conditioning steering being off.
"""
import inspect
import sys
import types
from pathlib import Path
import pytest
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import minimax_h3 as h3
class FakeProj(torch.nn.Module):
"""Stands in for AdalnProj: returns `expand` tensors of [M*modalities, hidden]."""
def __init__(self, modalities=3, expand=6, timesteps=2, hidden=4):
super().__init__()
self.modalities = modalities
self.expand = expand
rows = timesteps * modalities
# row value = its own index, so a scaled row is trivially identifiable
self.buf = torch.arange(1, rows * expand * hidden + 1,
dtype=torch.float32).view(rows, expand * hidden)
def forward(self, t_emb):
return self.buf.chunk(self.expand, dim=-1)
def gates(out):
return out[2], out[5]
def test_a_gain_scales_only_its_own_modality():
inner = FakeProj()
wrapped = h3.AdalnEdit(inner, {h3.MODALITY_TAGS["audio"]: 0.5})
base_msa, base_mlp = gates(inner(None))
msa, mlp = gates(wrapped(None))
for tag in (0, 1):
assert torch.equal(msa[tag::3], base_msa[tag::3])
assert torch.equal(mlp[tag::3], base_mlp[tag::3])
assert torch.allclose(msa[2::3], base_msa[2::3] * 0.5)
assert torch.allclose(mlp[2::3], base_mlp[2::3] * 0.5)
def test_only_the_gate_chunks_move():
"""shift and scale modulate what a block READS; the gate is what it WRITES."""
inner = FakeProj()
wrapped = h3.AdalnEdit(inner, {0: 0.25})
base, out = inner(None), wrapped(None)
for idx in (0, 1, 3, 4):
assert torch.equal(out[idx], base[idx])
def test_every_timestep_row_of_that_modality_is_scaled():
"""Rows are t_row*3+tag, so a modality appears once per unique timestep."""
inner = FakeProj(timesteps=3)
wrapped = h3.AdalnEdit(inner, {1: 2.0})
base_msa, _ = gates(inner(None))
msa, _ = gates(wrapped(None))
assert msa[1::3].shape[0] == 3
assert torch.allclose(msa[1::3], base_msa[1::3] * 2.0)
def test_the_source_buffer_is_not_mutated():
"""chunk() returns views onto one tensor — scaling in place would corrupt the module."""
inner = FakeProj()
before = inner.buf.clone()
h3.AdalnEdit(inner, {0: 0.1})(None)
h3.AdalnEdit(inner, {0: 0.1})(None)
assert torch.equal(inner.buf, before)
def test_gain_of_one_is_dropped_entirely():
wrapped = h3.AdalnEdit(FakeProj(), {0: 1.0, 1: 1.0})
assert wrapped.gains == {}
def test_a_single_modality_projection_is_left_alone():
"""FinalLayer uses AdalnProj too, with modalities=1 — different row meaning."""
inner = FakeProj(modalities=1, expand=2)
wrapped = h3.AdalnEdit(inner, {0: 0.5})
assert all(torch.equal(a, b) for a, b in zip(wrapped(None), inner(None)))
# ── attaching it to a model ─────────────────────────────────────────────────
class FakeBlock(torch.nn.Module):
def __init__(self):
super().__init__()
self.adaln_proj = FakeProj()
class FakePatcher:
def __init__(self, n=4):
self.objects = {"diffusion_model.blocks": [FakeBlock() for _ in range(n)]}
for i in range(n):
self.objects[f"diffusion_model.blocks.{i}.adaln_proj"] = \
self.objects["diffusion_model.blocks"][i].adaln_proj
self.patched = {}
def get_model_object(self, name):
if name in self.patched:
return self.patched[name]
return self.objects[name]
def clone(self):
other = FakePatcher.__new__(FakePatcher)
other.objects = self.objects
other.patched = dict(self.patched)
return other
def add_object_patch(self, name, obj):
self.patched[name] = obj
def test_all_ones_returns_the_same_model_unpatched():
model = FakePatcher()
out, note = h3.apply_adaln_edits(model, {"video": 1.0, "text": 1.0, "audio": 1.0})
assert out is model and note is None
assert model.patched == {}
def test_every_block_is_wrapped():
model = FakePatcher(n=4)
out, note = h3.apply_adaln_edits(model, {"video": 0.8})
assert out is not model
assert len(out.patched) == 4
assert all(isinstance(v, h3.AdalnEdit) for v in out.patched.values())
assert "4 blocks" in note
def test_it_says_the_anchor_moves_with_the_video():
"""Pins and reference images carry the VIDEO tag — a real limit, stated not hidden."""
_out, note = h3.apply_adaln_edits(FakePatcher(), {"video": 0.8})
assert "reference images ride the VIDEO tag" in note
def test_object_patches_are_used_not_method_replacement():
"""ComfyUI restores object patches on unpatch. Degradation outliving a reset has been a
real bug here; this is the mechanism that stops it."""
src = inspect.getsource(h3.apply_adaln_edits)
assert "add_object_patch" in src
assert "model.clone()" in src
def test_an_unknown_model_declines_with_a_reason():
class NoBlocks:
def get_model_object(self, name):
raise KeyError(name)
out, note = h3.apply_adaln_edits(NoBlocks(), {"video": 0.5})
assert isinstance(out, NoBlocks)
assert "no DiT blocks" in note
def test_unknown_modality_names_are_ignored():
model = FakePatcher()
out, note = h3.apply_adaln_edits(model, {"nonsense": 0.5})
assert out is model and note is None
# ── independence from the refinement path ──────────────────────────────────
def test_the_installer_reads_no_rating_or_key():
"""Explicitly separate from conditioning steering: turning value guidance off must not
turn this off, and this must work with no Refiner in the graph at all."""
import samplers
fn = samplers.FunPackLTXAVSceneChainSampler._install_h3_adaln_gains
src = inspect.getsource(fn)
# The prose says what it does NOT read, so check the code, not the docstring.
doc = inspect.getdoc(fn) or ""
for line in doc.splitlines():
src = src.replace(line, "")
for forbidden in ("refinement_key", "value_fn", "phrase_memory",
"global_state", "conditioning_deltas", "refine_v2"):
assert forbidden not in src, f"{forbidden} leaked into a sampler-side op"
def test_the_installer_takes_the_model_and_the_conditioning_only():
"""The conditioning is the bridge: Studio tags the learned gains onto it. The sampler
never opens a refinement key — that is the Studio/Sampler boundary."""
import samplers
params = inspect.signature(
samplers.FunPackLTXAVSceneChainSampler._install_h3_adaln_gains).parameters
assert list(params) == ["self", "model", "positive"]
def _sampler(mode="learned", **widgets):
import samplers
node = samplers.FunPackLTXAVSceneChainSampler.__new__(
samplers.FunPackLTXAVSceneChainSampler)
node._h3_gain_mode = mode
for key, value in {"video": 1.0, "prompt": 1.0, "audio": 1.0}.items():
setattr(node, f"_h3_gain_{key}", widgets.get(key, value))
node._h3_prompt_scale = widgets.get("prompt_scale", 1.0)
return node
def _tagged(gains):
return [[torch.zeros(1, 4, 8), {"funpack_h3_gains": gains}]]
def test_learned_gains_win_over_the_widgets():
node = _sampler(video=1.3, prompt=0.7)
out = node._h3_render_gains(_tagged({"video": 0.9, "prompt": 1.1,
"audio": 1.0, "prompt_scale": 1.2}))
assert out["video"] == pytest.approx(0.9)
assert out["prompt"] == pytest.approx(1.1)
assert out["prompt_scale"] == pytest.approx(1.2)
def test_manual_mode_uses_the_widgets_and_ignores_the_key():
node = _sampler(mode="manual", video=1.3)
out = node._h3_render_gains(_tagged({"video": 0.5}))
assert out["video"] == pytest.approx(1.3)
def test_manual_mode_works_with_no_conditioning_at_all():
"""The explicit override has to survive a graph with no Refiner in it."""
assert _sampler(mode="manual", video=0.8)._h3_render_gains(None)["video"] == pytest.approx(0.8)
def test_learned_mode_before_any_rating_renders_at_trained_strength():
"""Not the widgets: in learned mode those are not what the user is steering with, and
silently honouring them would make an unrated run look like a learned one."""
node = _sampler(video=1.3, prompt=0.6)
out = node._h3_render_gains([[torch.zeros(1, 4, 8), {}]])
assert out == node.H3_GAIN_NEUTRAL
def test_a_missing_key_in_the_learned_dict_falls_back_to_neutral():
out = _sampler()._h3_render_gains(_tagged({"video": 0.9}))
assert out["video"] == pytest.approx(0.9)
assert out["audio"] == 1.0 and out["prompt_scale"] == 1.0
def test_learned_is_the_default_mode():
import samplers
spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES()
fields = {**spec.get("required", {}), **spec.get("optional", {})}
assert fields["h3_gain_mode"][0] == ["learned", "manual"]
assert fields["h3_gain_mode"][1]["default"] == "learned"
def test_the_widgets_exist_and_default_to_untouched():
import samplers
spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES()
fields = {**spec.get("required", {}), **spec.get("optional", {})}
for name in ("h3_gain_video", "h3_gain_prompt", "h3_gain_audio"):
assert name in fields, name
assert fields[name][1]["default"] == 1.0
def test_sample_accepts_them_with_defaults():
import samplers
params = inspect.signature(samplers.FunPackLTXAVSceneChainSampler.sample).parameters
for name in ("h3_gain_video", "h3_gain_prompt", "h3_gain_audio"):
assert params[name].default == 1.0
tests/test_h3_gain_learning.py
-362
View File
@@ -1,362 +0,0 @@
"""The H3 render gains are learned from ratings, not typed.
A handful of scalars is a SMALLER search than the sigma profile, which already converges on
ratings alone — so a hand-tuned value here is the user doing work the loop can do. Same
update as the sigma profile with one difference: that profile is centred on 0, so its applied
value IS its perturbation; most of these are centred on 1.0, so the perturbation is
`applied - value`.
`refiner_bias` is the exception on both counts: it is centred on 0.0 (a signed push along the
learned taste direction, where 0 means no push) and it explores a narrower band, so its
exploration width scales down to match or its perturbations would clip against the bounds and
the reward would be credited to a move that never fully happened.
"""
import sys
import types
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import conditioning as C
@pytest.fixture
def studio():
return C.FunPackVideoRefinerV2.__new__(C.FunPackVideoRefinerV2)
def profile(reward, **kw):
return dict({"reward": reward, "key": "like" if reward > 0 else "awful"}, **kw)
def test_a_fresh_key_starts_neutral(studio):
state = studio._ensure_h3_gain_state({})
assert set(state["values"]) == set(studio.H3_GAIN_KEYS)
# Not all neutrals are 1.0: refiner_bias is a signed push, so its neutral is 0.0.
assert state["values"] == {k: studio._h3_gain_centre(k) for k in studio.H3_GAIN_KEYS}
def test_a_liked_perturbation_is_moved_toward(studio):
g = {}
studio._ensure_h3_gain_state(g)
g["h3_gains"]["last_applied"]["video"] = 1.10 # rendered above centre
studio._v2_update_h3_gains(g, profile(1.0))
assert g["h3_gains"]["values"]["video"] > 1.0
def test_a_disliked_perturbation_is_moved_away_from(studio):
g = {}
studio._ensure_h3_gain_state(g)
g["h3_gains"]["last_applied"]["video"] = 1.10
studio._v2_update_h3_gains(g, profile(-1.0))
assert g["h3_gains"]["values"]["video"] < 1.0
def test_the_sign_of_the_perturbation_matters_not_just_the_reward(studio):
"""Rendering BELOW centre and liking it must move the value DOWN."""
g = {}
studio._ensure_h3_gain_state(g)
g["h3_gains"]["last_applied"]["video"] = 0.90
studio._v2_update_h3_gains(g, profile(1.0))
assert g["h3_gains"]["values"]["video"] < 1.0
def test_no_perturbation_teaches_nothing(studio):
g = {}
studio._ensure_h3_gain_state(g) # last_applied == values
studio._v2_update_h3_gains(g, profile(1.0))
assert g["h3_gains"]["values"] == {k: studio._h3_gain_centre(k) for k in studio.H3_GAIN_KEYS}
def test_values_stay_inside_the_band(studio):
g = {}
studio._ensure_h3_gain_state(g)
for _ in range(400):
g["h3_gains"]["last_applied"]["video"] = studio.H3_GAIN_MAX
studio._v2_update_h3_gains(g, profile(1.0))
assert g["h3_gains"]["values"]["video"] <= studio.H3_GAIN_MAX
def test_exploration_narrows_on_success_and_widens_on_failure(studio):
g = {}
studio._ensure_h3_gain_state(g)
start = g["h3_gains"]["explore"]
studio._v2_update_h3_gains(g, profile(1.0))
assert g["h3_gains"]["explore"] < start
for _ in range(5):
studio._v2_update_h3_gains(g, profile(-1.0))
assert g["h3_gains"]["explore"] > start * 0.99
def test_a_forget_rating_teaches_nothing(studio):
g = {}
studio._ensure_h3_gain_state(g)
g["h3_gains"]["last_applied"]["video"] = 1.2
studio._v2_update_h3_gains(g, profile(1.0, skip_learning=True))
assert g["h3_gains"]["values"]["video"] == 1.0
assert g["h3_gains"]["iterations"] == 0
# ── what gets rendered ─────────────────────────────────────────────────────
def test_a_run_is_perturbed_off_the_learned_centre(studio):
"""Without a perturbation the next rating has nothing to credit."""
g = {}
out = studio._v2_h3_gains_for_run(g, seed=7)
assert any(v != 1.0 for v in out.values())
def test_the_perturbation_is_recorded_for_the_next_rating(studio):
g = {}
out = studio._v2_h3_gains_for_run(g, seed=7)
assert g["h3_gains"]["last_applied"] == out
def test_the_same_seed_repeats_the_same_perturbation(studio):
a = studio._v2_h3_gains_for_run({}, seed=11)
b = studio._v2_h3_gains_for_run({}, seed=11)
assert a == b
assert studio._v2_h3_gains_for_run({}, seed=12) != a
def test_exploration_can_be_switched_off(studio):
assert (studio._v2_h3_gains_for_run({}, explore=False)
== {k: studio._h3_gain_centre(k) for k in studio.H3_GAIN_KEYS})
def test_rendered_values_stay_inside_the_band(studio):
g = {}
studio._ensure_h3_gain_state(g)
g["h3_gains"]["explore"] = 5.0 # absurd, to prove the clamp
out = studio._v2_h3_gains_for_run(g, seed=3)
for key, value in out.items():
low, high = studio._h3_gain_bounds(key)
assert low <= value <= high
# ── the bridge to the sampler ──────────────────────────────────────────────
def test_the_gains_ride_entry_zero_of_the_conditioning(studio, monkeypatch):
import torch
monkeypatch.setattr(C, "_log", types.SimpleNamespace(
failed=lambda *a, **k: None, note_on_change=lambda *a, **k: None, feature=lambda *a, **k: None), raising=False)
cond = [[torch.zeros(1, 4, 8), {}], [torch.zeros(1, 4, 8), {}]]
out = studio._v2_tag_h3_gains(cond, {}, seed=5)
assert "funpack_h3_gains" in out[0][1]
assert "funpack_h3_gains" not in out[1][1]
def test_neutral_gains_tag_nothing(studio, monkeypatch):
import torch
monkeypatch.setattr(studio, "_v2_h3_gains_for_run",
lambda *a, **k: {key: studio._h3_gain_centre(key)
for key in studio.H3_GAIN_KEYS}, raising=False)
cond = [[torch.zeros(1, 4, 8), {}]]
assert studio._v2_tag_h3_gains(cond, {}, seed=5) is cond
def test_learning_runs_on_every_rating_not_behind_a_toggle():
"""The toggle gates APPLICATION; the key must keep growing either way, the same rule
value_guidance already follows."""
import inspect
src = inspect.getsource(C.FunPackVideoRefinerV2._v2_learn_scene_into_state)
assert "_v2_update_h3_gains(target_global, profile)" in src
# ── axis-aware credit assignment ───────────────────────────────────────────
#
# A rating carries three per-axis signals in [-1, +1] as well as its scalar reward, and
# they disagree with it in exactly the cases that matter. "Missing concept" is reward
# +0.10 — quality was fine — but concept_signal -1.00. Read as the scalar it nudges every
# gain toward the run that missed the prompt; read per axis it pushes the prompt gains
# away from it and leaves the audio gain out of it.
def test_missing_concept_punishes_the_prompt_gains(studio):
prof = C.RATING_PROFILES["Missing concept"]
assert prof["reward"] > 0 # the scalar says "slightly good"
assert studio._h3_gain_credit("prompt", prof) == pytest.approx(-1.0)
assert studio._h3_gain_credit("prompt_scale", prof) == pytest.approx(-1.0)
def test_missing_concept_does_not_drag_the_video_gain_down_with_it(studio):
prof = C.RATING_PROFILES["Missing concept"]
assert studio._h3_gain_credit("video", prof) > 0 # detail and quality were fine
def test_missing_details_punishes_the_video_gain(studio):
prof = C.RATING_PROFILES["Missing details"]
assert studio._h3_gain_credit("video", prof) < 0
assert studio._h3_gain_credit("prompt", prof) > 0 # the prompt landed
def test_audio_has_no_rated_axis_so_it_uses_the_overall_reward(studio):
for label in ("Perfect", "Awful", "Missing concept"):
prof = C.RATING_PROFILES[label]
assert studio._h3_gain_credit("audio", prof) == pytest.approx(prof["reward"])
def test_perfect_credits_everything_fully(studio):
prof = C.RATING_PROFILES["Perfect"]
for key in studio.H3_GAIN_KEYS:
assert studio._h3_gain_credit(key, prof) == pytest.approx(1.0)
def test_awful_punishes_everything(studio):
prof = C.RATING_PROFILES["Awful"]
for key in studio.H3_GAIN_KEYS:
assert studio._h3_gain_credit(key, prof) < 0
def test_a_profile_with_no_signals_falls_back_to_the_scalar(studio):
assert studio._h3_gain_credit("prompt", {"reward": 0.4}) == pytest.approx(0.4)
def test_a_satisfied_axis_cannot_mask_a_failed_one(studio):
"""Averaging let it: detail -1.00 with quality +0.85 came out at -0.08, so a rating that
said details were missing barely moved the gain that governs detail."""
prof = {"reward": 0.0, "detail_signal": -1.0, "quality_signal": 1.0}
assert studio._h3_gain_credit("video", prof) == pytest.approx(-1.0)
def test_missing_details_now_moves_the_video_gain_decisively(studio):
assert studio._h3_gain_credit("video", C.RATING_PROFILES["Missing details"]) \
== pytest.approx(-1.0)
def test_both_axes_satisfied_still_credits_fully(studio):
prof = {"reward": 1.0, "detail_signal": 1.0, "quality_signal": 1.0}
assert studio._h3_gain_credit("video", prof) == pytest.approx(1.0)
def test_axis_credit_actually_drives_the_update(studio):
"""The end-to-end consequence: one rating moves two gains in OPPOSITE directions."""
g = {}
studio._ensure_h3_gain_state(g)
for key in ("prompt", "video"):
g["h3_gains"]["last_applied"][key] = 1.10 # both rendered above centre
studio._v2_update_h3_gains(g, dict(C.RATING_PROFILES["Missing concept"]))
assert g["h3_gains"]["values"]["prompt"] < 1.0 # prompt missed -> back off
assert g["h3_gains"]["values"]["video"] > 1.0 # picture was fine -> keep
def test_every_gain_key_has_an_axis_entry(studio):
assert set(studio.H3_GAIN_AXES) == set(studio.H3_GAIN_KEYS)
# ── the learned taste direction ────────────────────────────────────────────
#
# `refiner_bias` says HOW FAR to push; this is WHICH WAY. The direction is the same
# `liked_dir` conditioning ascent already steers along — the difference is where it lands.
# Studio hands it over as a unit vector and the sampler decides the magnitude, because the
# magnitude belongs to the refiner's space and Studio cannot see it.
def _tensor(values):
import torch
return torch.tensor(values, dtype=torch.float32)
def _liked(count, direction=(3.0, 4.0)):
return {"liked_dir": {"direction": C.tensor_to_serializable(_tensor(direction)),
"direction_count": count}}
def test_no_direction_before_enough_liked_runs(studio):
"""Two liked runs is a coincidence. The threshold is the one liked_dir's existing
conditioning-space consumers already use — not a new number invented here."""
assert studio.H3_TASTE_DIR_MIN_COUNT == 3
assert studio._h3_taste_direction(_liked(2)) is None
def test_the_direction_arrives_once_the_threshold_is_met(studio):
assert studio._h3_taste_direction(_liked(3)) is not None
def test_the_direction_is_a_unit_vector(studio):
out = studio._h3_taste_direction(_liked(5, (3.0, 4.0)))
assert float(out.norm()) == pytest.approx(1.0)
assert out.tolist() == pytest.approx([0.6, 0.8])
def test_a_zero_direction_is_refused(studio):
"""Dividing by its norm would produce NaN and poison every prompt row."""
assert studio._h3_taste_direction(_liked(9, (0.0, 0.0))) is None
def test_an_empty_state_has_no_direction(studio):
assert studio._h3_taste_direction({}) is None
assert studio._h3_taste_direction(None) is None
def test_the_direction_rides_entry_zero_with_the_gains(studio, monkeypatch):
import torch
monkeypatch.setattr(C, "_log", types.SimpleNamespace(
failed=lambda *a, **k: None, note_on_change=lambda *a, **k: None, feature=lambda *a, **k: None), raising=False)
cond = [[torch.zeros(1, 4, 8), {}], [torch.zeros(1, 4, 8), {}]]
out = studio._v2_tag_h3_gains(cond, _liked(4), seed=5)
assert studio.H3_TASTE_DIR_META in out[0][1]
assert studio.H3_TASTE_DIR_META not in out[1][1]
def test_a_direction_alone_is_worth_tagging(studio, monkeypatch):
"""Neutral gains used to mean "tag nothing" — but a direction with a zero push is still
worth carrying, because the sampler's manual mode can supply the push itself."""
import torch
monkeypatch.setattr(C, "_log", types.SimpleNamespace(
failed=lambda *a, **k: None, note_on_change=lambda *a, **k: None, feature=lambda *a, **k: None), raising=False)
monkeypatch.setattr(studio, "_v2_h3_gains_for_run",
lambda *a, **k: {key: studio._h3_gain_centre(key)
for key in studio.H3_GAIN_KEYS}, raising=False)
cond = [[torch.zeros(1, 4, 8), {}]]
out = studio._v2_tag_h3_gains(cond, _liked(4), seed=5)
assert out is not cond and studio.H3_TASTE_DIR_META in out[0][1]
# ── the bias explores its own band ─────────────────────────────────────────
def test_the_bias_is_centred_on_no_push(studio):
assert studio._h3_gain_centre("refiner_bias") == 0.0
assert studio._h3_gain_centre("video") == 1.0
def test_the_bias_can_be_learned_negative(studio):
"""Pushing AWAY from the liked direction must be reachable, or the loop can only agree."""
low, high = studio._h3_gain_bounds("refiner_bias")
assert low < 0.0 < high
def test_exploration_scales_to_the_band_it_explores(studio):
"""One stored `explore` drives every gain; a narrower band has to shrink it or its
perturbations clip against the bounds and the credit goes to a move that never happened."""
wide = studio._h3_gain_explore("video", 0.05)
narrow = studio._h3_gain_explore("refiner_bias", 0.05)
assert narrow < wide
low, high = studio._h3_gain_bounds("refiner_bias")
assert narrow == pytest.approx(0.05 * (high - low) / (studio.H3_GAIN_MAX - studio.H3_GAIN_MIN))
def test_a_liked_bias_perturbation_is_moved_toward(studio):
g = {}
studio._ensure_h3_gain_state(g)
g["h3_gains"]["last_applied"]["refiner_bias"] = 0.10
studio._v2_update_h3_gains(g, profile(1.0))
assert g["h3_gains"]["values"]["refiner_bias"] > 0.0
def test_a_disliked_bias_perturbation_is_moved_away_from(studio):
g = {}
studio._ensure_h3_gain_state(g)
g["h3_gains"]["last_applied"]["refiner_bias"] = 0.10
studio._v2_update_h3_gains(g, profile(-1.0))
assert g["h3_gains"]["values"]["refiner_bias"] < 0.0
def test_the_bias_stays_inside_its_own_band_not_the_shared_one(studio):
g = {}
studio._ensure_h3_gain_state(g)
low, high = studio._h3_gain_bounds("refiner_bias")
for _ in range(400):
g["h3_gains"]["last_applied"]["refiner_bias"] = high
studio._v2_update_h3_gains(g, profile(1.0))
assert low <= g["h3_gains"]["values"]["refiner_bias"] <= high
tests/test_h3_token_refiner_edit.py
-331
View File
@@ -1,331 +0,0 @@
"""Editing the token refiner's OUTPUT rather than the Qwen tensor before it.
H3 takes `context` [1, L, 5120] through condition_proj (-> 5376) and a 2-block token
refiner that ends in an RMSNorm. Everything FunPack steers today happens BEFORE that, so
the refiner processes the edit: its attention mixes it across tokens and its final norm
renormalizes the magnitude away. An edit applied AFTER lands as set, in the space the 50
blocks consume.
Two edits: `scale` (how loudly the prompt is read through attention) and `bias` (a single
[hidden] vector added to every prompt row — prompt-LENGTH-INDEPENDENT, unlike anything
per-position, because Qwen does not pad and H3's sequence length moves with every edit).
"""
import inspect
import sys
from pathlib import Path
import pytest
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import minimax_h3 as h3
HID = 6
class FakeRefiner(torch.nn.Module):
def forward(self, x, *a, **k):
return torch.ones(x.shape[-2], HID)
class FakePatcher:
def __init__(self, proj_weight=None):
self.objects = {"diffusion_model.token_refiner": FakeRefiner()}
if proj_weight is not None:
self.objects["diffusion_model.condition_proj"] = \
torch.nn.Linear(proj_weight.shape[1], proj_weight.shape[0])
with torch.no_grad():
self.objects["diffusion_model.condition_proj"].weight.copy_(proj_weight)
self.patched = {}
def get_model_object(self, name):
if name in self.patched:
return self.patched[name]
if name not in self.objects:
raise KeyError(name)
return self.objects[name]
def clone(self):
other = FakePatcher.__new__(FakePatcher)
other.objects, other.patched = self.objects, dict(self.patched)
return other
def add_object_patch(self, name, obj):
self.patched[name] = obj
def run(edit, rows=8):
return edit(torch.zeros(1, rows, 4))
# ── the edit ────────────────────────────────────────────────────────────────
def test_scale_multiplies_the_rows():
out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=0.5))
assert torch.allclose(out, torch.full((8, HID), 0.5))
def test_rows_before_the_start_are_untouched():
"""The <Picture N> label and vision block sit at the head of this span."""
out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=0.0, row_start=3))
assert torch.all(out[:3] == 1.0)
assert torch.all(out[3:] == 0.0)
def test_a_row_end_bounds_the_other_side():
out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=2.0, row_start=2, row_end=5))
assert torch.all(out[:2] == 1.0) and torch.all(out[5:] == 1.0)
assert torch.all(out[2:5] == 2.0)
def test_a_bias_is_added_to_every_row_in_range():
bias = torch.arange(HID, dtype=torch.float32)
out = run(h3.TokenRefinerEdit(FakeRefiner(), bias=bias, row_start=1))
assert torch.allclose(out[1], 1.0 + bias)
assert torch.allclose(out[5], 1.0 + bias) # same vector on every row
assert torch.all(out[0] == 1.0)
def test_the_bias_is_length_independent():
"""The property that makes it a usable learning target: one vector, any prompt length."""
bias = torch.full((HID,), 0.25)
short = run(h3.TokenRefinerEdit(FakeRefiner(), bias=bias), rows=4)
long = run(h3.TokenRefinerEdit(FakeRefiner(), bias=bias), rows=40)
assert torch.allclose(short[0], long[0])
def test_scale_and_bias_compose_scale_first():
bias = torch.full((HID,), 1.0)
out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=2.0, bias=bias))
assert torch.allclose(out, torch.full((8, HID), 3.0)) # 1*2 + 1
def test_an_out_of_range_start_is_a_no_op_not_a_crash():
out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=0.0, row_start=99))
assert torch.all(out == 1.0)
def test_the_inner_output_is_not_mutated_in_place():
inner = FakeRefiner()
edit = h3.TokenRefinerEdit(inner, scale=0.0)
run(edit)
assert torch.all(inner(torch.zeros(1, 8, 4)) == 1.0)
# ── attaching it ────────────────────────────────────────────────────────────
def test_no_edit_returns_the_model_unpatched():
model = FakePatcher()
out, note = h3.apply_token_refiner_edit(model, scale=1.0, bias=None)
assert out is model and note is None and model.patched == {}
def test_it_patches_the_forward_not_the_module():
"""The module itself must stay in place: swapping it for a wrapper MODULE renames every
weight under it (token_refiner.blocks.* -> token_refiner.inner.blocks.*), and ComfyUI
records weight names off the live tree while the patch is applied. An unwrap by any
other patcher sharing the model then leaves a restore walking a `.inner` that is gone —
'TokenRefiner object has no attribute inner', mid-sample."""
model = FakePatcher()
out, note = h3.apply_token_refiner_edit(model, scale=1.2, row_start=4)
assert "diffusion_model.token_refiner" not in out.patched
assert isinstance(out.patched["diffusion_model.token_refiner.forward"], h3.TokenRefinerEdit)
assert "scale=1.2" in note and "rows 4:end" in note
def test_the_edit_is_not_a_module_so_the_state_dict_is_untouched():
assert not isinstance(h3.TokenRefinerEdit(FakeRefiner()), torch.nn.Module)
assert not isinstance(h3.AdalnEdit(FakeRefiner(), {}), torch.nn.Module)
def test_reinstalling_unwraps_instead_of_nesting():
"""Per scene we install again off a shared model. ComfyUI restores an object patch by
writing the previous value back, so the previous `forward` can be a live wrapper — and
capturing that would stack another scale on every scene."""
inner = FakeRefiner()
first = h3.TokenRefinerEdit(inner, scale=2.0)
inner.forward = first # as patch_model would leave it
second = h3.TokenRefinerEdit(inner, scale=2.0)
assert second.original is not first
assert torch.allclose(second(torch.zeros(1, 4, 4)), torch.full((4, HID), 2.0))
def test_object_patches_so_nothing_outlives_the_run():
src = inspect.getsource(h3.apply_token_refiner_edit)
assert "add_object_patch" in src and "model.clone()" in src
def test_a_model_without_a_refiner_declines_with_a_reason():
class Bare:
def get_model_object(self, name):
raise KeyError(name)
out, note = h3.apply_token_refiner_edit(Bare(), scale=0.5)
assert isinstance(out, Bare) and "no token_refiner" in note
# ── projecting a conditioning-space direction ───────────────────────────────
def test_a_direction_is_projected_by_the_weight_only():
"""A direction is a difference of two points, so the bias would move the origin."""
weight = torch.randn(HID, 4)
model = FakePatcher(proj_weight=weight)
direction = torch.randn(4)
out = h3.project_into_refiner_space(model, direction)
assert torch.allclose(out, weight @ direction, atol=1e-5)
def test_a_mismatched_width_is_refused():
model = FakePatcher(proj_weight=torch.randn(HID, 4))
assert h3.project_into_refiner_space(model, torch.randn(9)) is None
def test_no_projection_available_returns_none():
assert h3.project_into_refiner_space(FakePatcher(), torch.randn(4)) is None
# ── independence from the refinement path ───────────────────────────────────
def test_the_installer_never_reaches_past_the_conditioning():
"""The refiner edit is now partly rating-learned, but the SAMPLER still only applies.
Studio owns the refinement key, the state file and the learning; the sampler reads what
Studio tagged onto the conditioning and nothing else. Reaching for the key here would put
learning on both sides of the bridge, which is the boundary this project keeps.
"""
import samplers
cls = samplers.FunPackLTXAVSceneChainSampler
src = "".join(inspect.getsource(fn) for fn in
(cls._install_h3_token_refiner, cls._h3_taste_bias_vector, cls._h3_prompt_rows))
for forbidden in ("refinement_key", "_v2_load_state", "value_fn", "global_state",
"phrase_memory"):
assert forbidden not in src
def test_the_widget_defaults_to_untouched():
import samplers
spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES()
fields = {**spec.get("required", {}), **spec.get("optional", {})}
assert fields["h3_prompt_scale"][1]["default"] == 1.0
assert "h3_gain_prompt" in fields["h3_prompt_scale"][1]["tooltip"] # says how they differ
# ── the learned taste bias ──────────────────────────────────────────────────
#
# The direction comes from ratings (Studio's `liked_dir`), the magnitude does not. A learned
# direction has no natural scale in the refiner's space — those norms belong to the
# checkpoint — so the bias is a UNIT vector rescaled at apply time to a fraction of the
# span's own mean row norm. "10% of a typical prompt row" means the same thing on every
# prompt and every checkpoint; an absolute vector would be inert or catastrophic with no way
# to tell which in advance.
class ScaledRefiner(torch.nn.Module):
"""Rows with a known norm, so the relative rescale can be checked exactly."""
def __init__(self, value):
super().__init__()
self.value = float(value)
def forward(self, x, *a, **k):
return torch.full((x.shape[-2], HID), self.value)
def test_a_relative_bias_scales_to_the_rows_it_lands_on():
unit = torch.zeros(HID)
unit[0] = 1.0
out = h3.TokenRefinerEdit(ScaledRefiner(2.0), bias=unit * 0.5,
bias_relative=True)(torch.zeros(1, 4, 4))
row_norm = (torch.full((HID,), 2.0)).norm() # every row has this norm
assert out[0, 0] == pytest.approx(2.0 + 0.5 * float(row_norm), rel=1e-5)
assert out[0, 1] == pytest.approx(2.0) # off-direction untouched
def test_the_same_strength_means_the_same_thing_at_a_different_scale():
"""The property the relative mode exists for: transferable between checkpoints."""
unit = torch.zeros(HID)
unit[0] = 1.0
small = h3.TokenRefinerEdit(ScaledRefiner(1.0), bias=unit * 0.25,
bias_relative=True)(torch.zeros(1, 4, 4))
big = h3.TokenRefinerEdit(ScaledRefiner(100.0), bias=unit * 0.25,
bias_relative=True)(torch.zeros(1, 4, 4))
assert (small[0, 0] / 1.0) == pytest.approx(big[0, 0] / 100.0, rel=1e-4)
def test_an_absolute_bias_still_ignores_the_row_scale():
unit = torch.zeros(HID)
unit[0] = 1.0
out = h3.TokenRefinerEdit(ScaledRefiner(5.0), bias=unit * 0.5)(torch.zeros(1, 4, 4))
assert out[0, 0] == pytest.approx(5.5)
def test_a_negative_strength_pushes_the_other_way():
"""Away from what was liked has to be reachable, or the loop can only ever agree."""
unit = torch.zeros(HID)
unit[0] = 1.0
out = h3.TokenRefinerEdit(ScaledRefiner(2.0), bias=unit * -0.5,
bias_relative=True)(torch.zeros(1, 4, 4))
assert out[0, 0] < 2.0
def test_the_note_says_the_bias_is_relative():
model = FakePatcher()
_out, note = h3.apply_token_refiner_edit(model, bias=torch.zeros(HID), bias_relative=True)
assert "bias (relative)" in note
# ── the state-dict hazard that crashed a real run ───────────────────────────
#
# Reported from a rental: manual gain mode with the refinement keys deleted, sampling dies
# with "'TokenRefiner' object has no attribute 'inner'". Learned mode was fine, because with
# no key the gains are neutral and the installer returns before building anything.
#
# The chain: a wrapper MODULE renames every weight under it, ComfyUI's patch_model applies
# object patches BEFORE loading weights (so names are recorded WITH `.inner`), and
# unpatch_model restores weights BEFORE unwrapping. With more than one patcher alive off one
# shared model — one clone per scene, plus one for the AdaLN gains — another patcher can
# unwrap first, and the weight restore then walks `.inner` on a real TokenRefiner.
class _Refiner(torch.nn.Module):
def __init__(self):
super().__init__()
self.blocks = torch.nn.Linear(4, 4)
def forward(self, x, *a, **k):
return torch.ones(x.shape[-2], HID)
class _DiT(torch.nn.Module):
def __init__(self):
super().__init__()
self.token_refiner = _Refiner()
class _Base(torch.nn.Module):
def __init__(self):
super().__init__()
self.diffusion_model = _DiT()
def test_the_patch_never_renames_a_checkpoint_weight():
"""The whole bug in one assertion: no weight key may gain an `.inner` segment."""
model = _Base()
before = list(dict(model.named_parameters()))
edit = h3.TokenRefinerEdit(model.diffusion_model.token_refiner, scale=1.2)
model.diffusion_model.token_refiner.forward = edit # as patch_model installs it
assert list(dict(model.named_parameters())) == before
assert not any(".inner." in k for k in dict(model.named_parameters()))
def test_the_module_itself_stays_in_place():
model = _Base()
original = model.diffusion_model.token_refiner
edit = h3.TokenRefinerEdit(original, scale=1.2)
model.diffusion_model.token_refiner.forward = edit
assert model.diffusion_model.token_refiner is original
assert isinstance(model.diffusion_model.token_refiner, _Refiner)
def test_the_adaln_gain_patches_a_forward_too():
"""Same hazard, same shape: adaln_proj.weight must not become adaln_proj.inner.weight."""
src = inspect.getsource(h3.apply_adaln_edits)
assert "patch_module_forward" in src
tests/test_key_scoped_knobs.py
-142
View File
@@ -1,142 +0,0 @@
"""The learned render strengths belong to the refinement key, not to the project file.
Reported twice from a rental: the H3 gain and taste-bias values could not be cleared. They
were learned from ratings and stored in the key's own state — but the Editor ALSO wrote them
into the project's sampler_inputs, which made a second source of truth that outlived the key
it came from. Deleting every refinement key left the old values still applying.
The first attempt at this stripped them inside `Project.from_dict`, which was wrong in a way
no unit test caught: `from_dict` parses the SAVE payload as well as the file on disk, so the
editor sent a value, got a project back without it, and redrew the default. Every one of
these dials — including the mode that reaches them — was unsettable. Reported from the
rental as "mode automatically switches back to learned".
The rule now, in three parts:
1. Values are dropped when a project is READ FROM DISK, not on the save round-trip.
2. `h3_gain_mode` is NOT in the set. A mode is a deliberate choice, so clearing it
discards user intent.
3. In MANUAL mode nothing is dropped: there is no learned state to reset, because nothing
learned it — those are hand-typed settings like any other.
"""
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from movie_editor.backend.timeline import ( # noqa: E402
KEY_SCOPED_SAMPLER_INPUTS, Project, without_key_scoped)
FRONTEND = Path(__file__).resolve().parents[1] / "movie_editor" / "frontend"
def test_only_the_rating_learned_values_are_key_scoped():
assert KEY_SCOPED_SAMPLER_INPUTS == {
"h3_gain_video", "h3_gain_prompt", "h3_gain_audio",
"h3_prompt_scale", "h3_taste_bias", "h3_video_detail",
}
def test_the_mode_itself_is_never_stripped():
"""Stripping it made manual mode unreachable: pick it, save, and the round trip handed
back `learned` again."""
assert "h3_gain_mode" not in KEY_SCOPED_SAMPLER_INPUTS
assert without_key_scoped({"h3_gain_mode": "manual"})["h3_gain_mode"] == "manual"
# ── what happens on the way off disk ────────────────────────────────────────
def test_a_project_carrying_learned_values_is_cleaned():
"""An existing project has to heal itself; the user cannot be asked to hand-edit JSON."""
assert without_key_scoped(
{"h3_gain_video": 0.7, "h3_taste_bias": -0.2, "cfg": 1.0}) == {"cfg": 1.0}
def test_unrelated_sampler_inputs_survive():
assert without_key_scoped(
{"h3_audio_clock": True, "frame_overlap": 8, "h3_gain_audio": 1.2}) == {
"h3_audio_clock": True, "frame_overlap": 8}
def test_a_missing_block_is_fine():
assert without_key_scoped(None) == {}
def test_manual_mode_keeps_every_value_it_was_given():
"""They are hand-typed there. Clearing them would delete work, not stale learned state."""
stored = {"h3_gain_mode": "manual", "h3_gain_video": 0.7, "h3_taste_bias": -0.2}
assert without_key_scoped(stored) == stored
def test_learned_is_the_assumption_when_the_mode_is_not_recorded():
"""An older project has no mode key. Learned is the default, so treat it as learned."""
assert without_key_scoped({"h3_gain_video": 0.7}) == {}
def test_it_reads_the_disk_load_path():
import inspect
from movie_editor.backend import projects
assert "without_key_scoped" in inspect.getsource(projects.get)
# ── and what must NOT happen on save ────────────────────────────────────────
def test_parsing_a_save_payload_keeps_the_values():
"""`from_dict` runs on the browser's save as well as on the file. This is the exact
regression: strip here and the Editor can never set any of these at all."""
project = Project.from_dict({"sampler_inputs": {"h3_gain_video": 0.7, "cfg": 1.0}})
assert project.sampler_inputs == {"h3_gain_video": 0.7, "cfg": 1.0}
def test_the_mode_survives_a_save_round_trip():
saved = Project.from_dict({"sampler_inputs": {"h3_gain_mode": "manual"}}).to_dict()
assert saved["sampler_inputs"]["h3_gain_mode"] == "manual"
@pytest.mark.parametrize("knob", sorted(KEY_SCOPED_SAMPLER_INPUTS))
def test_every_learned_dial_can_be_set_within_a_session(knob):
"""Applying only on the next open is the whole point: a dial moved now must take effect
now, or manual mode does nothing."""
saved = Project.from_dict({"sampler_inputs": {knob: 0.5}}).to_dict()
assert saved["sampler_inputs"][knob] == 0.5
# ── the Editor keeps offering them ──────────────────────────────────────────
def _engine_settings():
return (FRONTEND / "engine_settings.js").read_text(encoding="utf-8")
@pytest.mark.parametrize("knob", sorted(KEY_SCOPED_SAMPLER_INPUTS | {"h3_gain_mode"}))
def test_every_render_dial_is_still_offered_in_settings(knob):
"""Untying a value from the project file is a storage change, not a reason to take the
dial away."""
import re
assert re.search(r'\{\s*name:\s*"%s"' % re.escape(knob), _engine_settings())
def test_the_settings_hint_says_where_the_values_live():
assert "live in the refinement key" in _engine_settings()
def test_they_are_still_widgets_on_the_node():
"""A raw ComfyUI graph with no Refiner in it still drives them by hand — that is what
h3_gain_mode 'manual' is for."""
import samplers
spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES()
fields = {**spec.get("required", {}), **spec.get("optional", {})}
for knob in KEY_SCOPED_SAMPLER_INPUTS | {"h3_gain_mode"}:
assert knob in fields
def test_no_key_means_neutral_not_a_remembered_value():
"""With the key gone there is nothing to read, so the run must fall back to the model's
trained strengths rather than to whatever was last used."""
import samplers
import torch
node = samplers.FunPackLTXAVSceneChainSampler.__new__(
samplers.FunPackLTXAVSceneChainSampler)
node._h3_gain_mode = "learned"
out = node._h3_render_gains([[torch.zeros(1, 4, 8), {}]])
assert out == node.H3_GAIN_NEUTRAL
tests/test_settings_card_noise.py
+43 -9
View File
@@ -28,7 +28,7 @@ def defaults(monkeypatch):
"segmented_detailing": False, "detail_targets": "hands, face",
"context_windows": False, "context_window_overlap": 40,
"joyai_memory": False, "joyai_audio_memory": False, "v2a_grad_scale": 1.0,
"h3_gain_mode": "learned", "h3_video_detail": 1.0,
"h3_video_detail": 1.0,
"cfg": 1.0, "frame_overlap": 8,
}
declared = frozenset(table) | {"mystery", "adjustments"}
@@ -89,6 +89,29 @@ def test_an_unrecognised_refiner_key_is_kept():
assert "prompt_repair" in got
def test_an_owner_live_for_more_than_one_value_is_matched_on_any_of_them():
"""absolute_strength belongs to steer_mode and is live for BOTH "absolute" and "both".
Reading only the singular dependsValue missed the plural dependsVals the Studio rows
use, so it printed under "relative", where it does nothing."""
assert sc._OWNED_BY["absolute_strength"] == ("steer_mode", ("absolute", "both"))
rows = dict(sc._live_rows({"steer_mode": "relative", "absolute_strength": 0.05},
defaults=sc._EDITOR_DEFAULTS))
assert "absolute_strength" not in rows
for mode in ("absolute", "both"):
got = dict(sc._live_rows({"steer_mode": mode, "absolute_strength": 0.05},
defaults=sc._EDITOR_DEFAULTS))
assert "absolute_strength" in got, mode
def test_a_boolean_owner_survives_the_journey_from_javascript():
"""The Editor writes JS literals, so a boolean owner arrives as "true" while Python's
own str(True) is "True"."""
assert sc._OWNED_BY["negative_erase_mode"] == ("negative_erase", ("true",))
got = dict(sc._live_rows({"negative_erase": True, "negative_erase_mode": "subtract"},
defaults=sc._EDITOR_DEFAULTS))
assert got["negative_erase_mode"] == "subtract"
def test_a_hand_built_control_still_gets_an_owner():
"""identity_projector is rendered by hand in the Editor, so it carries no dependsOn for
the generator to find — and it was printing a model filename under a disabled feature."""
@@ -122,11 +145,13 @@ def test_a_chain_of_owners_is_walked_to_the_end(defaults):
assert "v2a_grad_scale" in got
def test_an_owner_matched_by_value_not_by_truthiness(defaults):
"""h3_gain_mode is a combo: 'manual' switches its dials on, 'learned' does not — and
'learned' is a perfectly truthy string."""
def test_an_owner_can_be_matched_by_value_not_just_truthiness(monkeypatch, defaults):
"""A combo owner switches its dials on for ONE of its values, and the others are
perfectly truthy strings. No shipped knob uses this today; the rule still has to."""
monkeypatch.setitem(sc._OWNED_BY, "h3_video_detail", ("mode", ("manual",)))
monkeypatch.setitem(defaults, "mode", "learned")
assert rows({"h3_video_detail": 1.4}) == {}
assert "h3_video_detail" in rows({"h3_gain_mode": "manual", "h3_video_detail": 1.4})
assert "h3_video_detail" in rows({"mode": "manual", "h3_video_detail": 1.4})
def test_a_cycle_in_the_table_does_not_hang(monkeypatch, defaults):
@@ -142,11 +167,20 @@ def test_the_ownership_table_is_the_editors_own():
copies of one rule in two languages, so the test regenerates this one from that one."""
src = (FRONTEND / "engine_settings.js").read_text(encoding="utf-8")
found = {}
for name, body in re.findall(r'\{\s*name:\s*"([a-z0-9_]+)"(.*?)\n\s*(?=\{ name:|\];)',
for name, body in re.findall(r'\{\s*name:\s*"([a-z0-9_.]+)"(.*?)\n\s*(?=\{ name:|\];)',
src, re.S):
m = re.search(r'dependsOn:\s*"([a-z0-9_]+)"(?:,\s*dependsValue:\s*"([^"]*)")?', body)
if m:
found[name] = (m.group(1), m.group(2))
m = re.search(r'dependsOn:\s*"([a-z0-9_]+)"', body)
if not m:
continue
one = re.search(r'dependsValue:\s*"([^"]*)"', body)
many = re.search(r'dependsVals:\s*\[([^\]]*)\]', body)
if one:
vals = (one.group(1),)
elif many:
vals = tuple(v.strip().strip('"') for v in many.group(1).split(",") if v.strip())
else:
vals = None
found[name] = (m.group(1), vals)
assert found, "no dependsOn rows parsed — the parser, not the table, is what broke"
assert {**found, **sc._OWNED_EXTRA} == sc._OWNED_BY