diff --git a/conditioning.py b/conditioning.py index 62579925..dfe67ed6 100644 --- a/conditioning.py +++ b/conditioning.py @@ -3073,237 +3073,6 @@ class FunPackVideoRefiner: # SIGMA REFINEMENT # ========================================================================= - # ── rating-learned H3 render gains ────────────────────────────────────── - # - # Four scalars — how hard each block writes into the video, prompt and audio rows, and - # how loudly the prompt is read. They are learned rather than typed: four dimensions is - # SMALLER than the sigma profile's search space, which already converges on ratings - # alone, so asking the user to hand-tune them would be asking them to do a search the - # machine can do. - # - # Same update as the sigma profile — perturb, generate, rate, move toward the - # perturbation that earned reward — with one difference: the sigma profile is centred on - # 0, so its applied value IS its perturbation. These are centred on 1.0, so the - # perturbation is `applied - value` and that is what the reward is credited to. - # - # Unlike anything in conditioning space these transfer between prompts: 0.85 means the - # same thing on every prompt, so a value learned on one run is worth something on the - # next. That is the property that makes them worth learning at all. - H3_GAIN_KEYS = ("video", "prompt", "audio", "prompt_scale", "refiner_bias", - "video_detail") - # Which rated AXIS each gain answers to. A rating carries three per-axis signals in - # [-1, +1] alongside its overall reward, and they say far more than the scalar does: - # "Missing concept" is reward +0.10 (quality was fine) but concept_signal -1.00. Read as - # the scalar, that rating nudges every gain slightly TOWARD the run that missed the - # prompt. Read per axis, it pushes the two prompt gains hard away from it and leaves the - # audio gain alone, which is what the user actually said. - # - # Signals share the reward's [-1, +1] scale, so they drop straight into the same update. - # An empty tuple means no axis is rated for that gain — audio has none — so it falls back - # to the overall reward rather than inventing a signal for it. - H3_GAIN_AXES = { - "video": ("detail_signal", "quality_signal"), - "prompt": ("concept_signal",), - "prompt_scale": ("concept_signal",), - "audio": (), - "refiner_bias": (), - # The final-layer read strength: the same axes as the block gain it is the - # audio-safe twin of. `prompt_time` is deliberately absent from these keys — its - # off value (0.0) sits at the far end of its own range rather than in the middle, - # so exploring around it would step straight into "the prompt is pure noise". - # It stays a manual dial until a rental run says what a good value looks like. - "video_detail": ("detail_signal", "quality_signal"), - } - H3_GAIN_MIN = 0.60 - H3_GAIN_MAX = 1.40 - H3_GAIN_LR = 0.08 - H3_GAIN_EXPLORE_BASE = 0.05 - H3_GAIN_EXPLORE_MIN = 0.012 - H3_GAIN_EXPLORE_MAX = 0.09 - #: gains that are not multipliers. `refiner_bias` is how far to push the prompt rows - #: along the taste direction learned from liked runs, as a fraction of a typical prompt - #: row's magnitude — so its neutral value is 0.0, not 1.0, and it is signed: a negative - #: value pushes AWAY from what was liked, which the loop must be able to choose. - H3_GAIN_CENTRES = {"refiner_bias": 0.0} - H3_GAIN_BOUNDS = {"refiner_bias": (-0.30, 0.30)} - - def _h3_gain_centre(self, key): - return float(self.H3_GAIN_CENTRES.get(key, 1.0)) - - def _h3_gain_bounds(self, key): - return self.H3_GAIN_BOUNDS.get(key, (self.H3_GAIN_MIN, self.H3_GAIN_MAX)) - - def _h3_gain_explore(self, key, base): - """This key's exploration width, scaled to the span it explores. - - One stored `explore` drives every gain, so a key with a narrower range has to shrink - it proportionally — otherwise its perturbations clip against the bounds and the - reward gets credited to a move that never fully happened. - """ - low, high = self._h3_gain_bounds(key) - span = float(high) - float(low) - reference = self.H3_GAIN_MAX - self.H3_GAIN_MIN - return float(base) * (span / reference if reference else 1.0) - - def _ensure_h3_gain_state(self, global_state: dict): - state = global_state.setdefault("h3_gains", {}) - neutral = {k: self._h3_gain_centre(k) for k in self.H3_GAIN_KEYS} - state.setdefault("values", dict(neutral)) - state.setdefault("last_applied", dict(neutral)) - state.setdefault("iterations", 0) - state.setdefault("reward_ema", 0.0) - state.setdefault("explore", self.H3_GAIN_EXPLORE_BASE) - for key in self.H3_GAIN_KEYS: # a key added later starts neutral - state["values"].setdefault(key, neutral[key]) - state["last_applied"].setdefault(key, neutral[key]) - return state - - def _h3_gain_credit(self, key, rating_profile): - """The reward THIS gain should be credited with, from the axes it answers to. - - Recovering four parameters from one scalar works, but slowly — it relies on the - random perturbations decorrelating over many runs. The per-axis signals are already - on every rating, so each rating can teach the parameter it is actually about. - - Falls back to the overall reward when a gain has no rated axis (audio), or when a - profile carries no signals at all. - """ - axes = self.H3_GAIN_AXES.get(key, ()) - signals = [] - for axis in axes: - value = rating_profile.get(axis) - if value is None: - continue - try: - signals.append(float(value)) - except (TypeError, ValueError): - continue - if not signals: - return float(rating_profile.get("reward", 0.0) or 0.0) - # The WORST axis, not the average. These axes are conjunctive — the video gain has to - # satisfy detail AND quality — so a satisfied axis must not mask a failed one. - # Averaging let it: "Missing details" is detail -1.00 with quality +0.85, which - # averages to -0.08 and barely moves the gain the rating is actually about. - return min(signals) - - def _v2_update_h3_gains(self, global_state: dict, rating_profile: dict): - """Credit this rating to the perturbation that was actually rendered.""" - if not isinstance(global_state, dict) or not isinstance(rating_profile, dict): - return None - if rating_profile.get("skip_learning"): - return None - state = self._ensure_h3_gain_state(global_state) - reward = float(rating_profile.get("reward", 0.0) or 0.0) - values, applied = state["values"], state["last_applied"] - moved = [] - for key in self.H3_GAIN_KEYS: - centre = float(values.get(key, self._h3_gain_centre(key))) - perturbation = float(applied.get(key, centre)) - centre - if perturbation == 0.0: - continue - credit = self._h3_gain_credit(key, rating_profile) - low, high = self._h3_gain_bounds(key) - updated = min(high, max( - low, centre + perturbation * credit * self.H3_GAIN_LR)) - if updated != centre: - values[key] = updated - moved.append(f"{key}={updated:.3f}") - state["reward_ema"] = 0.85 * float(state.get("reward_ema", 0.0)) + 0.15 * reward - # Landing well narrows the search; landing badly widens it. - decay = 0.97 if reward > 0 else 1.02 - state["explore"] = min(self.H3_GAIN_EXPLORE_MAX, max( - self.H3_GAIN_EXPLORE_MIN, float(state.get("explore", self.H3_GAIN_EXPLORE_BASE)) * decay)) - state["iterations"] = int(state.get("iterations", 0)) + 1 - return ", ".join(moved) if moved else None - - def _v2_h3_gains_for_run(self, global_state: dict, seed=0, explore=True): - """The gains to render with, and the record of what was rendered. - - Exploration is what makes the next rating informative — without a perturbation there - is nothing for the reward to be credited to, and the profile can never move. Seeded - from the run's seed so a repeated run repeats its perturbation. - """ - if not isinstance(global_state, dict): - return {k: self._h3_gain_centre(k) for k in self.H3_GAIN_KEYS} - state = self._ensure_h3_gain_state(global_state) - values = state["values"] - base = float(state.get("explore", self.H3_GAIN_EXPLORE_BASE)) - out = {} - if explore: - rng = np.random.RandomState(int(seed) % (2 ** 32)) - for key in self.H3_GAIN_KEYS: - centre = float(values.get(key, self._h3_gain_centre(key))) - if explore: - centre += float(rng.normal(0.0, self._h3_gain_explore(key, base))) - low, high = self._h3_gain_bounds(key) - out[key] = min(high, max(low, centre)) - state["last_applied"] = dict(out) - return out - - #: where the learned taste direction rides to the sampler - H3_TASTE_DIR_META = "funpack_h3_taste_dir" - #: liked runs needed before the direction is worth pushing along — the same threshold - #: the conditioning-space consumers of `liked_dir` already use, not a new one - H3_TASTE_DIR_MIN_COUNT = 3 - - def _h3_taste_direction(self, global_state): - """The liked direction, as a UNIT vector the sampler can project into refiner space. - - This is the same `liked_dir` that conditioning ascent already steers along — the - difference is where it lands. Steering edits the Qwen tensor, which then goes through - condition_proj and the two refiner blocks and a final RMSNorm: the blocks mix it - across tokens and the norm scales its magnitude away, so the direction that arrives - is not the direction that was learned. Handing the sampler the raw direction lets it - add the same thing AFTER the refiner, where it lands as sent. - - Unit, not scaled: the magnitude belongs to the refiner's own space and is decided at - apply time. How far to push is the `refiner_bias` gain, and that is learned. - """ - if not isinstance(global_state, dict): - return None - slot = global_state.get("liked_dir") or {} - if int(slot.get("direction_count", 0)) < self.H3_TASTE_DIR_MIN_COUNT: - return None - try: - vector = serializable_to_tensor(slot.get("direction")).float().reshape(-1) - except Exception: # noqa: BLE001 - return None - norm = float(vector.norm().item()) - if not norm or norm != norm: # zero, or NaN from a corrupted slot - return None - return (vector / norm).cpu() - - def _v2_tag_h3_gains(self, conditioning_list, global_state, seed=0): - """Attach this run's render gains to entry 0 so the sampler can apply them.""" - try: - gains = self._v2_h3_gains_for_run(global_state, seed=seed) - except Exception as error: # noqa: BLE001 - _log.failed("FunPackStudio", "H3 render gains", error, - "this run renders at the model's trained strengths") - return conditioning_list - neutral = all(abs(v - self._h3_gain_centre(k)) < 1e-6 for k, v in gains.items()) - direction = self._h3_taste_direction(global_state) - if (not gains or neutral) and direction is None: - return conditioning_list - out = [] - for index, entry in enumerate(conditioning_list or []): - if index == 0 and isinstance(entry, (list, tuple)) and len(entry) >= 2 \ - and isinstance(entry[1], dict): - meta = dict(entry[1]) - meta["funpack_h3_gains"] = dict(gains) - if direction is not None: - meta[self.H3_TASTE_DIR_META] = direction - out.append([entry[0], meta]) - else: - out.append(entry) - _log.feature( - "FunPackStudio", "Rating-learned render gains", True, - ", ".join(f"{k}={gains[k]:.3f}" for k in self.H3_GAIN_KEYS if k in gains) - + ("; taste direction attached" if direction is not None - else f"; no taste direction yet (needs {self.H3_TASTE_DIR_MIN_COUNT} liked runs)") - + " (learned from ratings, perturbed to keep learning; the Chain Sampler applies " - "them and its h3_gain_* widgets are ignored unless h3_gain_mode is manual)") - return out def _ensure_sigma_state_defaults(self, global_adaptive: dict): global_adaptive.setdefault("sigma_profile", [0.0] * 32) @@ -4998,7 +4767,6 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): successful-seed-memory status string. Shared by the project key and every non-default key bound to the scene's shortcuts (multi-key rating).""" self._v2_update_phrase_memory(target_global, scene_run, profile, iter_num, axis_feedback) - self._v2_update_h3_gains(target_global, profile) self._v2_update_concept_delta_memory(target_global, scene_run, profile) self._v2_update_concept_pair_dirs(target_global, scene_run, profile) seed_memory_status = self._v2_update_successful_seed_memory( @@ -10136,7 +9904,6 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): value_guidance=True, latent=None, seed_output_connected=False, steer_mode="relative", absolute_strength=0.6, h3_phrase_emphasis=False, - h3_render_gains=True, prompt_enhance=False, prompt_enhance_system="", prompt_enhance_temperature=0.7, prompt_enhance_top_p=0.92, prompt_enhance_max_length=400, prompt_enhance_thinking=False, @@ -11201,14 +10968,6 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): f"entry 1 is steered and the other " f"{len(positive_conditioning) - 1} pass through unchanged. They used to be " f"discarded, which cut everything after the first entry out of the prompt.") - # Rating-learned render gains ride the conditioning to the sampler, the same bridge - # H3 token weighting uses: Studio owns the key and the learning, the sampler owns the - # application. Perturbed here, once per run, so the NEXT rating has something to - # credit — a run with no perturbation teaches nothing. - if h3_render_gains: - output_conditioning = self._v2_tag_h3_gains( - output_conditioning, global_state, seed=int(_seed or 0)) - # Batch Training packs N variant entries (tagged 'funpack_batch_variant') onto the FINAL # conditioning just before each return below — so it wraps whatever output_conditioning # ends up being: single-scene OR multi-scene (transition split). Works WITH transitions. @@ -13512,7 +13271,6 @@ class FunPackStudio: refinement_key_input=refinement_key_input, positive_conditioning=positive_conditioning, h3_phrase_emphasis=bool(rf.get("h3_phrase_emphasis", False)), - h3_render_gains=bool(rf.get("h3_render_gains", True)), prompt_enhance=bool(rf.get("prompt_enhance", False)), prompt_enhance_system=str(rf.get("prompt_enhance_system", "") or ""), prompt_enhance_temperature=float(rf.get("prompt_enhance_temperature", 0.7)), diff --git a/minimax_h3.py b/minimax_h3.py index 36d7891d..0b77504d 100644 --- a/minimax_h3.py +++ b/minimax_h3.py @@ -1311,26 +1311,12 @@ def extend_token_tags(meta, added_tokens, tag=1): return out -# ── per-modality AdaLN gain ────────────────────────────────────────────────── +# ── patching H3 in place ──────────────────────────────────────────────────── # -# Every DiT block carries its own ``AdalnProj(t_dim, hidden, expand=6, modalities=3)``: -# six modulation vectors — shift/scale/gate for attention, the same three for the MLP — -# and THREE copies of each, one per modality. Rows are indexed ``t_row * 3 + tag``, so a -# modality's rows across every timestep are the slice ``[tag::3]``. -# -# The GATE is what scales a block's output as it is written back into a row range. Scaling -# one modality's gate is therefore per-block, per-stream write gain: how much all 50 blocks -# are allowed to say about the prompt rows, or the audio rows, independently of the video. -# At 1.0 nothing is touched and the tuple is returned as it arrived. -# -# WHAT THIS CANNOT SEPARATE: keyframe pins and reference images carry the VIDEO tag (0). -# They differ from the target video by TIMESTEP row, not by tag — and only while the -# condition's noise-augmentation timestep is above the video's, which stops being true -# later in the schedule. So the video gain moves the anchor with it. Splitting them needs -# the layout's row mapping, which AdalnProj never sees. -MODALITY_TAGS = {"video": 0, "text": 1, "audio": 2} -_ADALN_GATE_CHUNKS = (2, 5) # shift_msa scale_msa GATE_msa shift_mlp scale_mlp GATE_mlp -_ADALN_MODALITIES = 3 +# Patching a module's FORWARD rather than replacing the module is what keeps the state +# dict intact: a wrapper module renames every weight beneath it, and ComfyUI records +# weight names off the LIVE tree, so an unwrap by any other patcher sharing this model +# would leave a restore walking a path that no longer exists. def unwrapped_forward(module): @@ -1360,81 +1346,6 @@ def patch_module_forward(patcher, key, wrapper): patcher.add_object_patch(f"{key}.forward", wrapper) -class AdalnEdit: - """Wraps one block's ``adaln_proj.forward`` and scales its gates per modality. - - Deliberately NOT an ``nn.Module``. Replacing a module with a wrapper module renames - every weight beneath it in the state dict (``adaln_proj.weight`` becomes - ``adaln_proj.inner.weight``), and ComfyUI records weight keys off the LIVE tree while - the patch is applied — so an unwrap by any other patcher sharing this model leaves a - restore walking a ``.inner`` that is no longer there. Patching ``forward`` leaves the - tree and the state dict exactly as the checkpoint built them. - """ - - #: marks an already-installed wrapper, so a re-install unwraps instead of nesting - _funpack_wrapper = True - - def __init__(self, inner, gains): - self.inner = inner - # The ORIGINAL callable, captured before the patch is applied. Calling - # ``inner.forward`` at call time would re-enter this wrapper once it is installed. - self.original = unwrapped_forward(inner) - self.gains = {int(t): float(g) for t, g in (gains or {}).items() if float(g) != 1.0} - - def __call__(self, t_emb): - if getattr(self.inner, "modalities", 1) != _ADALN_MODALITIES: - return self.original(t_emb) # FinalLayer shares the class, not the row meaning - out = self.original(t_emb) - if not self.gains: - return out - out = list(out) - for idx in _ADALN_GATE_CHUNKS: - gate = out[idx] - # chunk() returns views onto one buffer — never scale in place. - scaled = gate.clone() - for tag, factor in self.gains.items(): - scaled[tag::_ADALN_MODALITIES] = gate[tag::_ADALN_MODALITIES] * factor - out[idx] = scaled - return tuple(out) - -def apply_adaln_edits(model, gains=None): - """Attach per-modality gate gains to every DiT block. Returns (model, note). - - Uses ``add_object_patch`` rather than replacing methods by name: ComfyUI restores - object patches on unpatch, so nothing survives the run. Degradation that outlives a - reset has been a real bug here before, and this is the mechanism that prevents it. - - `gains` maps a name in MODALITY_TAGS to a float. 1.0 (or absent) means untouched, and - an all-1.0 request returns the model unchanged rather than cloning it. - """ - live = {MODALITY_TAGS[name]: float(value) - for name, value in (gains or {}).items() - if name in MODALITY_TAGS and float(value) != 1.0} - if not live: - return model, None - try: - blocks = model.get_model_object("diffusion_model.blocks") - except Exception as error: # not H3, or a layout we do not know - return model, f"AdaLN gain skipped — no DiT blocks to patch ({error})." - patched = model.clone() - count = 0 - for index in range(len(blocks)): - key = f"diffusion_model.blocks.{index}.adaln_proj" - try: - inner = patched.get_model_object(key) - except Exception: - continue - patch_module_forward(patched, key, AdalnEdit(inner, live)) - count += 1 - if not count: - return model, "AdaLN gain skipped — no adaln_proj found on the blocks." - named = ", ".join(f"{n}={v:g}" for n, v in sorted((gains or {}).items()) if float(v) != 1.0) - return patched, (f"AdaLN modality gain on {count} blocks ({named}). Keyframe pins and " - f"reference images ride the VIDEO tag, so the video gain moves them too.") - - - - # ── final layer: the one video-only edit ───────────────────────────────────── # # Every block shares one attention pass, so anything written into the video rows is read by @@ -1498,132 +1409,3 @@ def apply_final_video_scale(model, scale): patch_module_forward(patched, key, FinalLayerVideoScale(inner, scale)) return patched, (f"Final-layer video scale {float(scale):g} — past the last attention, " f"so the audio stream cannot see it.") - - -# ── token-refiner output ───────────────────────────────────────────────────── -# -# `context` (Qwen, [1, L, 5120]) reaches the DiT through -# ``condition_proj`` (Linear 5120 -> 5376) then ``token_refiner`` (2 blocks + RMSNorm). -# Everything FunPack steers today happens BEFORE that, on the Qwen tensor, which means the -# refiner then processes the edit — its two attention blocks mix it across tokens and its -# final RMSNorm renormalizes the magnitude away. The direction that lands is not the -# direction that was learned. -# -# Editing the refiner's OUTPUT lands unchanged, and lands in the space the 50 blocks -# actually consume. Two edits are supported: -# -# scale — multiply the rows. The blocks read text through attention, so magnitude is how -# loudly the prompt competes with the picture for attention. Distinct from -# h3_gain_prompt, which controls how much each block WRITES BACK into these rows. -# bias — add a direction. A single [hidden] vector added to every prompt row is -# PROMPT-LENGTH INDEPENDENT, which per-position edits are not (Qwen does not pad, -# so H3's sequence length changes with every prompt edit). That is the shape a -# learned direction has to have here to transfer between prompts. -# -# Both are confined to a row range: the `` label and vision-block rows sit at the -# head of this span and are not ours to steer. - - -class TokenRefinerEdit: - """Wraps ``token_refiner.forward`` and edits its output over a row range. - - Deliberately NOT an ``nn.Module``. Replacing the module with a wrapper module renames - the checkpoint's own keys — ``token_refiner.blocks.*`` becomes - ``token_refiner.inner.blocks.*`` — and ComfyUI's ``patch_model`` applies object patches - BEFORE it loads weights, so every weight gets recorded under the ``.inner`` name. On the - way out it restores weights before it unwraps, and with several patchers alive off one - shared model (one clone per scene, plus one for the AdaLN gains) another patcher can - unwrap first — leaving a restore that walks ``.inner`` on a real ``TokenRefiner``. - Patching the forward leaves the tree and the state dict as the checkpoint built them. - """ - - #: marks an already-installed wrapper, so a re-install unwraps instead of nesting - _funpack_wrapper = True - - def __init__(self, inner, scale=1.0, bias=None, row_start=0, row_end=None, - bias_relative=False): - self.inner = inner - # The ORIGINAL callable, captured before the patch is applied. Calling - # ``inner.forward`` at call time would re-enter this wrapper once installed. - self.original = unwrapped_forward(inner) - self.scale = float(scale) - self.bias = bias - self.row_start = int(row_start) - self.row_end = row_end - # A RELATIVE bias is a unit vector rescaled, at apply time, to a fraction of the - # span's own mean row norm. A learned direction has no natural magnitude in this - # space — the norms here belong to the checkpoint, not to anything the loop measured - # — so an absolute vector would be either inert or catastrophic with no way to tell - # which in advance. Relative makes the strength mean "this fraction of a typical - # prompt row", which is the same thing on every prompt and every checkpoint. - self.bias_relative = bool(bias_relative) - - def __call__(self, x, *args, **kwargs): - out = self.original(x, *args, **kwargs) - try: - rows = out.shape[-2] - start = max(0, min(int(self.row_start), rows)) - end = rows if self.row_end is None else max(start, min(int(self.row_end), rows)) - if end <= start: - return out - out = out.clone() - span = out[..., start:end, :] - if self.scale != 1.0: - span = span * self.scale - if self.bias is not None: - bias = self.bias.to(device=span.device, dtype=span.dtype) - if self.bias_relative: - scale = span.float().norm(dim=-1).mean() - bias = bias * scale.to(bias.dtype) - span = span + bias - out[..., start:end, :] = span - except Exception: # a refiner shape we do not recognise: leave it alone - return self.original(x, *args, **kwargs) - return out - - -def project_into_refiner_space(model, direction): - """Map a conditioning-space (Qwen) direction into refiner space with ``condition_proj``. - - The bias, not just the weight, would move the origin — a DIRECTION is a difference of two - points, so only the linear part carries over. Returns None when the projection cannot be - read or the width does not match. - """ - try: - proj = model.get_model_object("diffusion_model.condition_proj") - weight = proj.weight - except Exception: - return None - try: - vector = direction.reshape(-1).to(dtype=weight.dtype, device=weight.device) - if vector.shape[0] != weight.shape[1]: - return None - return torch.mv(weight, vector) - except Exception: - return None - - -def apply_token_refiner_edit(model, scale=1.0, bias=None, row_start=0, row_end=None, - bias_relative=False): - """Attach a scale and/or bias to the token refiner's output. Returns (model, note). - - ``add_object_patch`` again, so ComfyUI restores it on unpatch and nothing outlives the - run. A scale of 1.0 with no bias returns the model unchanged rather than cloning it. - """ - if float(scale) == 1.0 and bias is None: - return model, None - key = "diffusion_model.token_refiner" - try: - inner = model.get_model_object(key) - except Exception as error: - return model, f"token-refiner edit skipped — no token_refiner to patch ({error})." - patched = model.clone() - patch_module_forward(patched, key, TokenRefinerEdit(inner, scale, bias, row_start, - row_end, bias_relative=bias_relative)) - parts = [] - if float(scale) != 1.0: - parts.append(f"scale={float(scale):g}") - if bias is not None: - parts.append("bias (relative)" if bias_relative else "bias") - where = f"rows {row_start}:{row_end if row_end is not None else 'end'}" - return patched, f"token-refiner edit ({', '.join(parts)}) on {where}." diff --git a/movie_editor/backend/projects.py b/movie_editor/backend/projects.py index 611817c2..fd9e9cbe 100644 --- a/movie_editor/backend/projects.py +++ b/movie_editor/backend/projects.py @@ -10,7 +10,7 @@ import time from typing import Optional from . import config -from .timeline import Project, Scene, without_key_scoped +from .timeline import Project, Scene def _path(project_id: str): @@ -39,11 +39,7 @@ def get(project_id: str) -> Optional[Project]: p = _path(project_id) if not p.exists(): return None - stored = json.loads(p.read_text()) - # Only here, on the way off disk: the rating-learned strengths come from the refinement - # key, so an opened project must not carry yesterday's copy of them. - stored["sampler_inputs"] = without_key_scoped(stored.get("sampler_inputs")) - return Project.from_dict(stored) + return Project.from_dict(json.loads(p.read_text())) def save(project: Project) -> Project: diff --git a/movie_editor/backend/settings_card.py b/movie_editor/backend/settings_card.py index 13af4195..d06f4093 100644 --- a/movie_editor/backend/settings_card.py +++ b/movie_editor/backend/settings_card.py @@ -158,9 +158,13 @@ def _pass_rows(cfg: dict) -> list: # Two rules, both the user's: skip a value that equals the widget's own default, and skip a # value whose feature is off. -#: knob -> (the toggle that owns it, the value of that toggle which switches it on). +#: knob -> (the toggle that owns it, the values of that toggle which switch it on). #: None means "any truthy value". Mirrored from the Editor's own dependsOn wiring, and a #: test regenerates this from engine_settings.js so the two cannot drift apart. +#: +#: A tuple, not one value: `absolute_strength` belongs to `steer_mode` and is live for BOTH +#: "absolute" and "both". Reading only the singular `dependsValue` missed the plural +#: `dependsVals` the Studio rows use, so it printed under `relative`, where it does nothing. #: knobs the Editor renders by hand rather than from its knob table, so they carry no #: dependsOn for the test below to find. Kept separate so the generated half stays generated. _OWNED_EXTRA = { @@ -169,7 +173,7 @@ _OWNED_EXTRA = { _OWNED_BY = { **_OWNED_EXTRA, - "absolute_strength": ("steer_mode", None), + "absolute_strength": ("steer_mode", ('absolute', 'both')), "alg_anchor_sigma_threshold": ("alg_anchor", None), "alg_anchor_strength": ("alg_anchor", None), "alg_guide_blur_sigma_threshold": ("alg_blur_guides", None), @@ -191,12 +195,6 @@ _OWNED_BY = { "dynashift_threshold": ("dynashift", None), "embed_guidance_source": ("embed_guidance", None), "embed_guidance_strength": ("embed_guidance", None), - "h3_gain_audio": ("h3_gain_mode", "manual"), - "h3_gain_prompt": ("h3_gain_mode", "manual"), - "h3_gain_video": ("h3_gain_mode", "manual"), - "h3_prompt_scale": ("h3_gain_mode", "manual"), - "h3_taste_bias": ("h3_gain_mode", "manual"), - "h3_video_detail": ("h3_gain_mode", "manual"), "id_strength": ("identity_transfer_enabled", None), "joyai_audio_memory": ("joyai_memory", None), "joyai_fix_frames": ("joyai_memory", None), @@ -204,16 +202,16 @@ _OWNED_BY = { "joyai_memory_size": ("joyai_memory", None), "joyai_memory_strength": ("joyai_memory", None), "mid_scene_guide_strength": ("mid_scene_guide", None), - "negative_erase_mode": ("negative_erase", None), - "negative_erase_renorm": ("negative_erase", None), - "negative_erase_strength": ("negative_erase", None), + "negative_erase_mode": ("negative_erase", ('true',)), + "negative_erase_renorm": ("negative_erase", ('true',)), + "negative_erase_strength": ("negative_erase", ('true',)), "output_guidance_strength": ("output_guidance", None), "phase_scale": ("identity_transfer_enabled", None), - "prompt_enhance_max_length": ("prompt_enhance", None), - "prompt_enhance_system": ("prompt_enhance", None), - "prompt_enhance_temperature": ("prompt_enhance", None), - "prompt_enhance_thinking": ("prompt_enhance", None), - "prompt_enhance_top_p": ("prompt_enhance", None), + "prompt_enhance_max_length": ("prompt_enhance", ('true',)), + "prompt_enhance_system": ("prompt_enhance", ('true',)), + "prompt_enhance_temperature": ("prompt_enhance", ('true',)), + "prompt_enhance_thinking": ("prompt_enhance", ('true',)), + "prompt_enhance_top_p": ("prompt_enhance", ('true',)), "score_slider_strength": ("score_slider", None), "source_id": ("identity_transfer_enabled", None), "v2a_grad_scale": ("joyai_audio_memory", None), @@ -262,14 +260,7 @@ _EDITOR_DEFAULTS = { "embed_guidance_strength": 0.02, "frame_overlap": 16, "h3_audio_clock": False, - "h3_gain_audio": 1.0, - "h3_gain_mode": "learned", - "h3_gain_prompt": 1.0, - "h3_gain_video": 1.0, "h3_phrase_emphasis": False, - "h3_prompt_scale": 1.0, - "h3_render_gains": True, - "h3_taste_bias": 0.0, "h3_video_detail": 1.0, "id_strength": 1.0, "identity_transfer_enabled": False, @@ -352,7 +343,10 @@ def _switched_on(name: str, values: dict, defaults: dict, _seen=None) -> bool: return True seen.add(owner) current = values.get(owner, defaults.get(owner)) - live = bool(current) if wanted is None else str(current) == str(wanted) + # Lowercased both sides: the Editor writes JS literals, so a boolean owner arrives as + # "true" while Python's own str(True) is "True". + live = bool(current) if wanted is None \ + else str(current).lower() in {str(w).lower() for w in wanted} return live and _switched_on(owner, values, defaults, seen) diff --git a/movie_editor/backend/timeline.py b/movie_editor/backend/timeline.py index b4b9e20e..125ef01d 100644 --- a/movie_editor/backend/timeline.py +++ b/movie_editor/backend/timeline.py @@ -544,37 +544,6 @@ class Scene: return project.frame_rate -#: Sampler knobs whose values belong to the REFINEMENT KEY, not to the project. -#: -#: These are learned from ratings and live in the key's own state. A copy in the project file -#: was a second source of truth that outlived the key it came from: deleting every key left -#: the old values still applying, with no way to clear them but typing the neutral value back -#: in by hand. -#: -#: TWO THINGS THIS IS NOT, both learned the hard way: -#: -#: 1. It is not `h3_gain_mode`. A mode is a choice, and clearing it on open would throw -#: away a setting the user made on purpose, which is not what "untie from the project -#: file" meant. -#: 2. It does not apply in MANUAL mode. There, these are hand-typed settings like any other -#: and must persist; there is no learned state to reset, because nothing learned them. -KEY_SCOPED_SAMPLER_INPUTS = frozenset({ - "h3_gain_video", "h3_gain_prompt", "h3_gain_audio", - "h3_prompt_scale", "h3_taste_bias", "h3_video_detail", -}) - - -def without_key_scoped(raw) -> dict: - """Drop the rating-learned values, so a project opens at whatever the key now says. - - Applied when a project is READ FROM DISK — never on the save round-trip. `from_dict` - runs on both, and stripping there meant the editor sent a value, got a project back - without it, and showed the default again: the knobs could not be set at all. - """ - values = dict(raw or {}) - if str(values.get("h3_gain_mode", "learned")).strip().lower() == "manual": - return values - return {k: v for k, v in values.items() if k not in KEY_SCOPED_SAMPLER_INPUTS} @dataclass diff --git a/movie_editor/frontend/engine_defaults.test.js b/movie_editor/frontend/engine_defaults.test.js index 0d9f787e..e1b24509 100644 --- a/movie_editor/frontend/engine_defaults.test.js +++ b/movie_editor/frontend/engine_defaults.test.js @@ -16,7 +16,6 @@ const assert = require("node:assert"); function writeSamplerInput(store, k, value, immediate) { if (value === k.default) { store.unsetSamplerInput(k.name); - (k.clearsOnDefault || []).forEach((name) => store.unsetSamplerInput(name)); if (immediate) store.flushSave?.(); return; } @@ -93,7 +92,6 @@ test("booleans clear when set back to their default", () => { assert.deepStrictEqual(s.calls[0], ["unset", "segmented_detailing"]); }); - // ── handing ownership back ────────────────────────────────────────────────── // // Reported from a rental as a question, which is the best kind: "save as manual, load it, @@ -102,50 +100,9 @@ test("booleans clear when set back to their default", () => { // hide in learned mode), until the next time the project was opened. Correct output, state // nobody could see. That is the failure this whole rule exists to prevent. -const fs = require("node:fs"); -const path = require("node:path"); - -const MODE = { - name: "h3_gain_mode", - default: "learned", - clearsOnDefault: ["h3_gain_video", "h3_gain_prompt", "h3_gain_audio", - "h3_prompt_scale", "h3_taste_bias", "h3_video_detail"], -}; - -test("switching H3 render gains back to learned clears the manual values with it", () => { - const s = fakeStore(); - writeSamplerInput(s, MODE, "learned", true); - const cleared = s.calls.filter((c) => c[0] === "unset").map((c) => c[1]); - assert.deepStrictEqual(cleared, ["h3_gain_mode", ...MODE.clearsOnDefault]); -}); - -test("choosing manual clears nothing — those values are about to be used", () => { - const s = fakeStore(); - writeSamplerInput(s, MODE, "manual", true); - assert.ok(!s.calls.some((c) => c[0] === "unset")); - assert.deepStrictEqual(s.calls[0], ["setNow", "h3_gain_mode", "manual"]); -}); - test("an ordinary knob returning to default clears only itself", () => { const s = fakeStore(); writeSamplerInput(s, GAIN, 1.0, true); assert.deepStrictEqual(s.calls.filter((c) => c[0] === "unset"), [["unset", "h3_gain_video"]]); }); -test("the rule under test is the one the renderer actually runs", () => { - // This file re-implements writeSamplerInput so it can run without a DOM. That copy is - // free to drift from the real one, and a green suite would say nothing. - const src = fs.readFileSync(path.join(__dirname, "engine_settings.js"), "utf8"); - assert.match(src, /\(k\.clearsOnDefault \|\| \[\]\)\.forEach/); - assert.match(src, /clearsOnDefault: \["h3_gain_video"/); -}); - -test("the mode clears exactly the values the backend treats as key-scoped", () => { - // Two halves of one rule in two languages: the browser clears them on the switch, the - // server clears them on load. A value in one list and not the other is a leak. - const src = fs.readFileSync( - path.join(__dirname, "..", "backend", "timeline.py"), "utf8"); - const block = src.split("KEY_SCOPED_SAMPLER_INPUTS = frozenset({")[1].split("})")[0]; - const backend = [...block.matchAll(/"([a-z0-9_]+)"/g)].map((m) => m[1]).sort(); - assert.deepStrictEqual(backend, [...MODE.clearsOnDefault].sort()); -}); diff --git a/movie_editor/frontend/engine_settings.js b/movie_editor/frontend/engine_settings.js index 583fee00..506c0237 100644 --- a/movie_editor/frontend/engine_settings.js +++ b/movie_editor/frontend/engine_settings.js @@ -133,8 +133,6 @@ { name: "prompt_enhance_thinking", label: "Enhancer thinking mode", kind: "bool", default: false, dependsOn: "prompt_enhance", dependsVals: [true], hint: "Let the model reason before answering, if it supports it. Slower, and the reasoning is stripped from the result." }, - { name: "h3_render_gains", label: "Rating-learned render gains (H3)", kind: "bool", default: true, - hint: "Learns six render strengths from your ratings alone: how hard each block writes into the picture, the prompt and the soundtrack, how loudly the prompt is read, how far to push the prompt toward what you have rated well, and how much detail the picture gets. Six numbers is a smaller search than the sigma schedule already learns, so there is nothing to tune by hand — each run is rendered slightly off the learned values so the next rating has something to credit. The taste push needs at least 3 liked runs on the key before it does anything. Free. The learned values live in the refinement key, not in this project — clearing your keys resets them. Off leaves the model at its trained strengths; the Chain Sampler's h3_gain_*, h3_taste_bias and h3_video_detail widgets then do nothing unless its h3_gain_mode is set to manual." } ]; function parseStudioSettings(p) { @@ -325,31 +323,8 @@ { name: "h3_audio_clock", label: "H3 audio clock (few-step audio)", kind: "bool", default: false, hint: "MiniMax H3 only: removes audio distortion on few-step turbo schedules. Free.", detail: "H3 runs video and audio on different flow schedules but only one sigma grid reaches the sampler, so audio drifts further the bigger the step. This corrects it for one multiply per step. Exact with FunPack Distilled Flow, Hybrid Euler 2S and euler; leave it off with res_multistep / dpmpp_2m and the rest of the multistep family, which already absorb most of the error. No effect when both sigma shifts are equal, or on two-evals-per-step samplers." }, - { name: "h3_gain_mode", label: "H3 render gains", kind: "combo", choices: ["learned", "manual"], default: "learned", - clearsOnDefault: ["h3_gain_video", "h3_gain_prompt", "h3_gain_audio", - "h3_prompt_scale", "h3_taste_bias", "h3_video_detail"], - hint: "MiniMax H3 only. 'learned' takes six render strengths from your ratings and ignores the dials below. 'Prompt is settled' is not one of them and works in either mode. 'manual' uses the dials as typed.", - detail: "Learned needs the Refiner's 'Rating-learned render gains (H3)' left on — with it off, or before anything is rated, learned mode renders at the model's trained strengths and the dials still do nothing. Switch to manual to drive them yourself." }, - { name: "h3_gain_video", label: "Write gain · picture", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01, - dependsOn: "h3_gain_mode", dependsValue: "manual", - hint: "How hard each block writes into the picture rows. Below 1.0 is a calmer, less-committed image; above pushes harder. 1.0 = untouched. Free.", - detail: "Keyframe pins and reference images ride the same modality tag as the video, so this moves the anchor with it." }, - { name: "h3_gain_prompt", label: "Write gain · prompt", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01, - dependsOn: "h3_gain_mode", dependsValue: "manual", - hint: "How much each block rewrites the prompt as it draws. Lower it when a scene slowly stops matching what you asked for; 0.0 freezes the text at its encoded value. 1.0 = untouched. Free." }, - { name: "h3_gain_audio", label: "Write gain · soundtrack", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01, - dependsOn: "h3_gain_mode", dependsValue: "manual", - hint: "How hard each block writes into the audio rows. Below 1.0 builds the soundtrack more conservatively. 1.0 = untouched. Free." }, - { name: "h3_prompt_scale", label: "Prompt loudness", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01, - dependsOn: "h3_gain_mode", dependsValue: "manual", - hint: "How loudly the prompt is READ, as opposed to how much it is rewritten. Above 1.0 it is harder to ignore; below, the anchor or the reference gets more say. 1.0 = untouched. Free." }, - { name: "h3_taste_bias", label: "Taste push", kind: "float", default: 0.0, min: -0.30, max: 0.30, step: 0.01, - dependsOn: "h3_gain_mode", dependsValue: "manual", - hint: "Pushes the prompt toward what you have rated well. Negative pushes away from it. Needs at least 3 liked runs on the refinement key; without them it does nothing and says so. 0.0 = off. Free.", - detail: "The Refiner already steers along your liked direction, but on H3 that edit is mixed and renormalized by the token refiner before the model reads it. This adds the same direction after the refiner, where it lands as sent. Measured as a fraction of a typical prompt row, so one value means the same thing on every prompt." }, - { name: "h3_video_detail", label: "Picture detail (audio-safe)", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01, - dependsOn: "h3_gain_mode", dependsValue: "manual", - hint: "Makes the picture crisper or softer without changing the sound at all. Above 1.0 = more detail and contrast, below is softer. The step from 1.0 is small — reach for 1.4-1.8 before deciding it does nothing. Free.", + { name: "h3_video_detail", label: "Picture detail (experimental)", kind: "float", default: 1.0, min: 0.0, max: 2.0, step: 0.01, + hint: "Makes the picture crisper or softer without changing the sound at all. Above 1.0 = more detail and contrast, below is softer. The step from 1.0 is small — reach for 1.4-1.8 before deciding it does nothing. Free. Experimental, lightly tested.", detail: "Every block of the model shares one attention pass, so anything that moves the picture also reaches the soundtrack. This runs after the last one, where there is no path left for it to travel — it is the audio-safe twin of Write gain \u00b7 picture." }, { name: "segmented_detailing", label: "Segmented detailing (region refine)", kind: "bool", default: false, hint: "Re-renders small regions like hands at higher detail after each scene. Costs about 15%.", @@ -482,12 +457,6 @@ function writeSamplerInput(k, value, immediate) { if (value === k.default) { S.unsetSamplerInput(k.name); - // A knob whose default hands ownership BACK to something else takes the values it - // owned with it. Without this, switching H3 render gains from manual to learned left - // the six hand-typed values behind: inert, invisible (the dials hide in learned mode), - // and not cleared until the project was next opened. Deferred invisible state is the - // whole problem this rule exists to remove. - (k.clearsOnDefault || []).forEach((name) => S.unsetSamplerInput(name)); if (immediate) S.flushSave?.(); return; } @@ -557,7 +526,7 @@ chain_timing: ["frame_overlap", "transition_duration", "use_same_seed", "cut_opening_frames"], chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold"], chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"], - chain_experimental: ["context_windows", "context_window_length", "context_window_overlap", "context_window_schedule", "context_window_fuse", "context_window_freenoise", "context_window_retain_first", "h3_audio_clock", "h3_gain_mode", "h3_gain_video", "h3_gain_prompt", "h3_gain_audio", "h3_prompt_scale", "h3_taste_bias", "h3_video_detail", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_mode", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], + chain_experimental: ["context_windows", "context_window_length", "context_window_overlap", "context_window_schedule", "context_window_fuse", "context_window_freenoise", "context_window_retain_first", "h3_audio_clock", "h3_video_detail", "segmented_detailing", "detail_targets", "detail_strength", "detail_threshold", "detail_max_area", "detail_mode", "detail_denoise", "mid_scene_guide", "mid_scene_guide_strength", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"], }; function countChainView(p, id, st) { diff --git a/movie_editor/frontend/pipeline_caps.js b/movie_editor/frontend/pipeline_caps.js index 6cbb31eb..1be019e4 100644 --- a/movie_editor/frontend/pipeline_caps.js +++ b/movie_editor/frontend/pipeline_caps.js @@ -171,7 +171,7 @@ // Left on against an LTX pipeline they are just as silently inert, so they get a chip // by the same rule — the user should not have to spend a generation to find out. // key -> why it cannot run off H3. - const H3_RENDER_GAIN_REASON = + const H3_ONLY_REASON = "The render gains scale H3's per-modality AdaLN gates and its token refiner. LTX has " + "neither: one modulation path for all streams, and the prompt reaches it through " + "cross-attention instead of a packed text span."; @@ -183,13 +183,7 @@ // The render gains scale H3's per-modality AdaLN gates and its token refiner. LTX has // neither — one modulation path, cross-attention instead of a packed text span — so // there is nothing for these to reach on any other family. - h3_gain_mode: H3_RENDER_GAIN_REASON, - h3_gain_video: H3_RENDER_GAIN_REASON, - h3_gain_prompt: H3_RENDER_GAIN_REASON, - h3_gain_audio: H3_RENDER_GAIN_REASON, - h3_prompt_scale: H3_RENDER_GAIN_REASON, - h3_taste_bias: H3_RENDER_GAIN_REASON, - h3_video_detail: H3_RENDER_GAIN_REASON, + h3_video_detail: H3_ONLY_REASON, }; // Sub-settings of identity transfer: meaningless wherever the feature itself cannot run, diff --git a/samplers.py b/samplers.py index aa72aa78..e137d18f 100644 --- a/samplers.py +++ b/samplers.py @@ -3147,33 +3147,9 @@ class FunPackLTXAVSceneChainSampler: "default": False, "tooltip": "EXPERIMENTAL, MiniMax H3 only: integrate the audio stream on its OWN flow schedule. H3 denoises video and audio on two different schedules (the MiniMax H3 Sigma Shift node's shift_video 12 / shift_audio 3), but only one sigma grid reaches a sampler, so the model reconciles them by scaling the audio velocity by the slope between the two schedules at the START of each step. That is exact for infinitesimal steps and increasingly wrong as steps get bigger: on a 4-step schedule the last step drives audio roughly 2.5x past where its own schedule puts it, which is heard as distortion. This replaces that start-of-step slope with the one that actually spans the step, so audio lands where its schedule says. Costs one scalar multiply per step — no extra model call. Aimed at few-step schedules (turbo/distilled LoRAs, 4-8 steps), where nothing else fixes this. NO-OP when shift_video and shift_audio are EQUAL — the streams are then on one schedule and there is nothing to correct (it says so on the console). SAMPLER MATTERS, measured against a perfect predictor so the numbers are pure schedule error (audio error as a % of the stream's full range, 4/8/20 steps): WORKS BEST — FunPack Distilled Flow and Hybrid Euler 2S (runs inside their step loop, exact), and stock `euler` (85/38/14% -> 0/0/0%, exact at every step count). PERFORMS POORLY — the higher-order multistep family (`res_multistep`, `dpmpp_2m`, `gradient_estimation`, `ipndm`, `lms`, `deis`): they already absorb most of this error themselves, so the clock helps them at 4 steps (69% -> 21%) but HURTS at 20 (1% -> 15%); leave it off there. NO EFFECT — two-evals-per-step samplers (`heun`, `dpm_2`, `dpmpp_2s_ancestral`, `dpmpp_sde`, `seeds_2`): a model call cannot be tied to a step from outside their loop, so the wrapper detects that on the first call and switches itself off with a console note rather than guessing. Ancestral/SDE samplers additionally add noise to the audio stream, which this does not address. The clock never touches the video stream directly, though on H3 the two share one attention sequence, so a changed audio latent can still shift the video slightly.", }), - "h3_gain_video": ("FLOAT", { - "default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05, - "tooltip": "EXPERIMENTAL, MiniMax H3 only. Per-block write gain on the VIDEO rows. Every DiT block carries three sets of AdaLN modulation — one per modality (video / prompt / audio) — and the GATE is what scales that block's attention and MLP output as it is written back into a row range. This multiplies the video gate on all 50 blocks: below 1.0 each block contributes less to the picture (softer, calmer, less detail and less motion), above 1.0 more (harder, busier, and past ~1.3 it overcooks). 1.0 = untouched, and the model is not even cloned. Free — one small vector multiply per block, no extra model calls. KEYFRAME PINS AND REFERENCE IMAGES RIDE THE VIDEO TAG, so this moves them with the picture; they are separated from the target video by timestep row, not by modality, and only early in the schedule. Attached with add_object_patch, which ComfyUI restores on unpatch, so nothing survives the run.", - }), - "h3_gain_prompt": ("FLOAT", { - "default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05, - "tooltip": "EXPERIMENTAL, MiniMax H3 only. Per-block write gain on the PROMPT rows (see h3_gain_video for the mechanism). H3 keeps the text in the same sequence as the picture and lets every block rewrite it, so the prompt the model reads at block 50 is not the one the encoder produced. 0.0 freezes the text at its encoded value for the whole forward — the video still attends to it, but it can no longer drift toward what is being drawn. Lower values are the lever to try when a scene slowly stops matching what you asked for; higher values let the text move further with the picture. 1.0 = untouched. Free. The vision rows inside the text span carry the VIDEO tag, not this one, so a reference is not affected.", - }), - "h3_gain_audio": ("FLOAT", { - "default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05, - "tooltip": "EXPERIMENTAL, MiniMax H3 only. Per-block write gain on the AUDIO rows (see h3_gain_video for the mechanism). This is the one modality whose gate can be moved without touching the other two at the point of writing — though the streams still share one attention pass, so a changed audio row is read by the video on the NEXT block. Below 1.0 the soundtrack is built more conservatively. Free. 1.0 = untouched.", - }), - "h3_taste_bias": ("FLOAT", { - "default": 0.0, "min": -0.30, "max": 0.30, "step": 0.01, - "tooltip": "EXPERIMENTAL, MiniMax H3 only. Pushes the prompt toward what you have RATED WELL, in the model's own space. The Refiner already learns a 'liked direction' from your ratings and steers the conditioning along it — but that edit then goes through H3's token refiner, whose two attention blocks mix it across tokens and whose final norm scales its magnitude away, so what the model reads is not what was learned. This adds the same direction AFTER the refiner, where it lands as sent. Measured as a fraction of a typical prompt row's magnitude, so it means the same thing on every prompt and every checkpoint. Negative pushes AWAY from what you liked. Needs a refinement key with at least 3 liked runs; without one it does nothing and says so. Free. IGNORED unless h3_gain_mode is 'manual' — in 'learned' mode this is learned from your ratings like the other gains. 0.0 = off.", - }), - "h3_gain_mode": (["learned", "manual"], { - "default": "learned", - "tooltip": "Where the six H3 render strengths come from. 'learned' (default) takes them from the refinement key: Studio learns them from your ratings alone and tags them onto the conditioning, and the six widgets below are IGNORED — nothing to tune by hand, which is the point. Six scalars is a smaller search than the sigma schedule already learns from ratings, so it converges in tens of rated runs. With no key wired, or before the first rating, 'learned' renders at the model's trained strengths (untouched, and the taste push at 0). 'manual' ignores the learned values and uses the widgets below exactly as set — for deliberately probing one strength, or for a graph with no Refiner in it.", - }), - "h3_prompt_scale": ("FLOAT", { - "default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05, - "tooltip": "EXPERIMENTAL, MiniMax H3 only. How LOUD the prompt is when the picture reads it. H3 puts the Qwen text through condition_proj + a 2-block token refiner (ending in an RMSNorm) before the DiT sees it, and the blocks then read the text through attention — so the magnitude of those refined rows is how strongly the prompt competes for attention against the picture and the reference. This multiplies them: above 1.0 the prompt is harder to ignore, below 1.0 it recedes and the reference or the anchor gets more say. Applied AFTER the refiner's final norm, so it lands as set rather than being renormalized away. 1.0 = untouched, and the model is not cloned. Free — one multiply on the text rows, no extra model calls. Confined to the PROMPT rows: the label and vision block sit ahead of them and are left alone (read from minimax_token_tags; if those cannot be read the whole text span is scaled and the console says so). DIFFERENT from h3_gain_prompt, which controls how much each block WRITES BACK into the text rows — this controls how loudly they are READ.", - }), "h3_video_detail": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05, - "tooltip": "EXPERIMENTAL, MiniMax H3 only. Makes the picture crisper or softer WITHOUT changing the sound at all. Above 1.0 = more detail and contrast, below 1.0 = softer and calmer. The step from 1.0 is small, so reach for 1.4-1.8 before deciding it does nothing. 1.0 = untouched, and the model is not cloned. This is the audio-safe twin of h3_gain_video: everything earlier in the model shares one attention pass, so a change to the picture reaches the soundtrack, but this runs after the last one — there is no path left for it to travel. Free. UNVALIDATED.", + "tooltip": "EXPERIMENTAL, MiniMax H3 only. Makes the picture crisper or softer WITHOUT changing the sound at all. Above 1.0 = more detail and contrast, below 1.0 = softer and calmer. The step from 1.0 is small, so reach for 1.4-1.8 before deciding it does nothing. 1.0 = untouched, and the model is not cloned. Everything earlier in the model shares one attention pass, so any other change to the picture also reaches the soundtrack; this runs after the last one, where there is no path left for it to travel. Free. EXPERIMENTAL and lightly tested: 1.25 read as more detail and better contrast on one rental run, and nothing beyond that has been measured.", }), "alg_anchor": ("BOOLEAN", { "default": False, @@ -3987,9 +3963,7 @@ class FunPackLTXAVSceneChainSampler: # Per-modality AdaLN gain. Sampler-side and self-contained: it reads no refinement # key, no rating and no Studio state, so it works on a graph with the Refiner absent # entirely. 1.0 on all three does not even clone the model. - model = self._install_h3_adaln_gains(model, positive) model = self._install_h3_final_layer(model, positive) - model = self._install_h3_token_refiner(model, positive) try: sampled = comfy.sample.sample_custom( @@ -6666,204 +6640,6 @@ class FunPackLTXAVSceneChainSampler: w_idx = idx % w return (w_idx >= (w // 2)).long() - #: what Studio tags onto entry 0 when it has learned gains for this key - H3_GAINS_META = "funpack_h3_gains" - #: and the learned taste direction that rides with them - H3_TASTE_DIR_META = "funpack_h3_taste_dir" - #: the value of each render gain that means "untouched". Not all of them are 1.0: - #: refiner_bias is a signed push along a learned direction, so its neutral is 0.0. - H3_GAIN_NEUTRAL = {"video": 1.0, "prompt": 1.0, "audio": 1.0, - "prompt_scale": 1.0, "refiner_bias": 0.0, - "video_detail": 1.0} - - def _h3_render_gains(self, positive): - """The four render strengths for this run: learned from ratings, or the widgets. - - Learned is the default because rating is the only input the user wants to give — four - scalars is a smaller search than the sigma profile already runs on ratings alone, so a - hand-tuned value here is the user doing work the loop can do. `manual` is the explicit - override, and it still works with no Refiner in the graph at all. - - Reading the CONDITIONING META rather than a refinement key keeps the boundary intact: - Studio owns the key and the learning, the sampler owns the application — the same - bridge H3 token weighting already uses. - """ - manual = {"video": float(getattr(self, "_h3_gain_video", 1.0)), - "prompt": float(getattr(self, "_h3_gain_prompt", 1.0)), - "audio": float(getattr(self, "_h3_gain_audio", 1.0)), - "prompt_scale": float(getattr(self, "_h3_prompt_scale", 1.0)), - "refiner_bias": float(getattr(self, "_h3_taste_bias", 0.0)), - "video_detail": float(getattr(self, "_h3_video_detail", 1.0))} - if str(getattr(self, "_h3_gain_mode", "learned")).lower() == "manual": - return manual - learned = None - try: - entry = positive[0] if isinstance(positive, list) and positive else None - if isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict): - candidate = entry[1].get(self.H3_GAINS_META) - if isinstance(candidate, dict): - learned = {k: float(v) for k, v in candidate.items()} - except Exception: # noqa: BLE001 - learned = None - if learned is None: - # No key, or nothing rated yet. Trained strengths — NOT the widgets, which in - # this mode are not what the user is steering with. - if any(value != self.H3_GAIN_NEUTRAL[key] for key, value in manual.items()): - _log.feature( - "FunPackSceneChain", "Rating-learned render gains", False, - "nothing rated on this key yet, so the run uses the model's trained " - "strengths. The h3_gain_* widgets are IGNORED in 'learned' mode — set " - "h3_gain_mode to 'manual' to use them as typed.") - return dict(self.H3_GAIN_NEUTRAL) - return {k: learned.get(k, self.H3_GAIN_NEUTRAL[k]) for k in manual} - - def _h3_prompt_rows(self, positive): - """Where the PROMPT starts inside the text span, from the conditioning's own tags. - - Returns (start, note). The `` label and a reference's vision block sit at - the head of the span and are not ours to edit, so everything the refiner edit does is - confined to the rows after them. - """ - try: - from . import h3_token_weights as _tw - except ImportError: - import h3_token_weights as _tw - try: - entry = positive[0] if isinstance(positive, list) and positive else None - meta = entry[1] if isinstance(entry, (list, tuple)) and len(entry) >= 2 else {} - cond = entry[0] if entry is not None else None - cond_len = int(cond.shape[1]) if hasattr(cond, "shape") and cond.dim() >= 2 else 0 - region = _tw.prompt_region(meta.get("minimax_token_tags"), cond_len) - if region: - return int(region[0]), None - if cond_len: - return 0, ("the prompt/reference boundary could not be read from the token " - "tags, so the WHOLE text span is edited — including a reference's " - "vision block, if one is wired") - except Exception as error: # noqa: BLE001 - _log.failed("FunPackSceneChain", "h3 token-refiner row range", error, - "the whole text span is edited instead of just the prompt") - return 0, None - - def _h3_taste_bias_vector(self, model, positive, strength): - """The learned taste direction, projected into the refiner's own space. - - Studio learns `liked_dir` in Qwen space and already steers the conditioning along it. - That edit then goes through condition_proj, two refiner blocks and a final RMSNorm — - which mixes it across tokens and normalizes its magnitude away, so the direction that - reaches the 50 DiT blocks is not the one that was learned. Adding it AFTER the - refiner lands it unchanged, in the space the blocks actually read. - - Added to every prompt row equally, which is what makes it transferable: Qwen does not - pad, so the row COUNT changes with every prompt edit and a per-position bias would - mean something different on each run. - - Returns a unit vector for `TokenRefinerEdit(bias_relative=True)` to rescale, or None. - """ - if not strength: - return None - try: - from . import minimax_h3 as h3mod - except ImportError: - import minimax_h3 as h3mod - direction = None - try: - entry = positive[0] if isinstance(positive, list) and positive else None - if isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict): - direction = entry[1].get(self.H3_TASTE_DIR_META) - except Exception: # noqa: BLE001 - direction = None - if direction is None: - _log.feature( - "FunPackSceneChain", "Taste push", False, - "the conditioning carries no learned direction. " - "It needs a refinement key with at least 3 liked runs on it.") - return None - projected = h3mod.project_into_refiner_space(model, direction) - if projected is None: - _log.feature( - "FunPackSceneChain", "Taste push", False, - "the learned direction does not fit the token refiner's space. " - "Its width does not match condition_proj.") - return None - norm = float(projected.float().norm().item()) - if not norm or norm != norm: - return None - return (projected.float() / norm) * float(strength) - - def _install_h3_token_refiner(self, model, positive): - """Edit the token refiner's OUTPUT: prompt loudness, and the learned taste push. - - One patch for both. They act on the same rows through the same wrapper, so applying - them separately would nest two wrappers and make the second one's `bias_relative` - rescale read a span the first had already scaled. - """ - gains = self._h3_render_gains(positive) - scale = float(gains.get("prompt_scale", 1.0)) - strength = float(gains.get("refiner_bias", 0.0)) - if scale == 1.0 and not strength: - return model - try: - from . import minimax_h3 as h3mod - except ImportError: - import minimax_h3 as h3mod - if not h3mod.is_h3_model(model): - _log.feature( - "FunPackSceneChain", "Token-refiner edit", False, - "not a MiniMax H3 model. There is no token refiner to edit.") - return model - start, note = self._h3_prompt_rows(positive) - if note: - _log.feature("FunPackSceneChain", "Token-refiner edit", True, note) - bias = self._h3_taste_bias_vector(model, positive, strength) - if scale == 1.0 and bias is None: - return model - try: - patched, applied = h3mod.apply_token_refiner_edit( - model, scale=scale, bias=bias, row_start=start, bias_relative=bias is not None) - except Exception as error: # noqa: BLE001 - _log.failed("FunPackSceneChain", "h3 token-refiner edit", error, - "the prompt is read at its trained strength and unbiased") - return model - if applied: - print(f"[FunPackSceneChain] {applied}") - return patched - - def _install_h3_adaln_gains(self, model, positive): - """Scale every DiT block's AdaLN gates per modality, from the three sampler widgets. - - Deliberately NOT part of the refinement path. This is a visual-behaviour op, so it - lives on the sampler (Studio produces conditioning; the sampler decides how the model - renders it) and it reads only its own widgets. Nothing here consults a refinement key, - a rating, or a learned direction — turning conditioning steering off does not turn - this off, and turning this on does not require the Refiner to be wired at all. - - Returns `model` untouched when all three gains are 1.0, which is every default run. - """ - gains = self._h3_render_gains(positive) - if all(gains.get(k, 1.0) == 1.0 for k in ("video", "prompt", "audio")): - return model - try: - from . import minimax_h3 as h3mod - except ImportError: - import minimax_h3 as h3mod - if not h3mod.is_h3_model(model): - _log.feature( - "FunPackSceneChain", "AdaLN render gains", False, - "not a MiniMax H3 model. The per-modality gates only exist there.") - return model - # MODALITY_TAGS names the text modality "text"; the widget says "prompt" because that - # is what it is to the person setting it. - tagged = {"video": gains["video"], "text": gains["prompt"], "audio": gains["audio"]} - try: - patched, note = h3mod.apply_adaln_edits(model, tagged) - except Exception as error: # noqa: BLE001 - _log.failed("FunPackSceneChain", "AdaLN modality gain", error, - "the blocks write at their trained strength for every modality") - return model - if note: - print(f"[FunPackSceneChain] {note}") - return patched def _install_h3_final_layer(self, model, positive): @@ -6873,8 +6649,7 @@ class FunPackLTXAVSceneChainSampler: blocks share one attention sequence. This one cannot: there is no attention after the final layer, so the audio branch is untouched by construction. """ - gains = self._h3_render_gains(positive) - detail = float(gains.get("video_detail", 1.0)) + detail = float(getattr(self, "_h3_video_detail", 1.0)) if detail == 1.0: return model try: @@ -7222,8 +6997,6 @@ class FunPackLTXAVSceneChainSampler: second_pass=False, second_pass_op="none", second_pass_sigmas=None, second_pass_sampler=None, h3_audio_clock=False, - h3_gain_video=1.0, h3_gain_prompt=1.0, h3_gain_audio=1.0, - h3_prompt_scale=1.0, h3_taste_bias=0.0, h3_gain_mode="learned", h3_video_detail=1.0, audio_vae=None, h3_keyframes=None, unique_id=None, prompt=None): @@ -7239,14 +7012,8 @@ class FunPackLTXAVSceneChainSampler: # happened last run is reported again rather than deduped away forever. _log.begin_run() self._is_h3 = self._set_stream_axes(model) - # Read once per run, consumed by _install_h3_adaln_gains at each scene's sample call. - self._h3_gain_video = max(0.0, min(2.0, float(h3_gain_video))) - self._h3_gain_prompt = max(0.0, min(2.0, float(h3_gain_prompt))) - self._h3_gain_audio = max(0.0, min(2.0, float(h3_gain_audio))) - self._h3_prompt_scale = max(0.0, min(2.0, float(h3_prompt_scale))) - self._h3_taste_bias = max(-0.30, min(0.30, float(h3_taste_bias))) + # Read once per run, consumed by _install_h3_final_layer at each scene's sample call. self._h3_video_detail = max(0.0, min(2.0, float(h3_video_detail))) - self._h3_gain_mode = str(h3_gain_mode or "learned").strip().lower() # The gate every rating-driven wrapper shares. On H3 it is read off the schedule's # own base grid; on LTX it stays the absolute-sigma gate it was validated with. _steer_ramp = _make_steer_ramp(sigmas, self._is_h3) diff --git a/tests/test_h3_adaln_gain.py b/tests/test_h3_adaln_gain.py deleted file mode 100644 index ec0de68c..00000000 --- a/tests/test_h3_adaln_gain.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Per-modality AdaLN gain: per-block write strength for video / prompt / audio. - -Every H3 DiT block carries `AdalnProj(t_dim, hidden, expand=6, modalities=3)` — shift, -scale and gate for attention and for the MLP, in three copies, one per modality. Rows are -indexed `t_row * 3 + tag` (video 0, text 1, audio 2), so one modality's rows across every -timestep are the slice `[tag::3]`. The GATE scales what a block writes back into a row -range, which makes it per-block, per-stream write gain. - -This is a sampler-side visual-behaviour op. It reads no refinement key, no rating and no -learned direction, so it is unaffected by conditioning steering being off. -""" -import inspect -import sys -import types -from pathlib import Path - -import pytest -import torch - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -import minimax_h3 as h3 - - -class FakeProj(torch.nn.Module): - """Stands in for AdalnProj: returns `expand` tensors of [M*modalities, hidden].""" - - def __init__(self, modalities=3, expand=6, timesteps=2, hidden=4): - super().__init__() - self.modalities = modalities - self.expand = expand - rows = timesteps * modalities - # row value = its own index, so a scaled row is trivially identifiable - self.buf = torch.arange(1, rows * expand * hidden + 1, - dtype=torch.float32).view(rows, expand * hidden) - - def forward(self, t_emb): - return self.buf.chunk(self.expand, dim=-1) - - -def gates(out): - return out[2], out[5] - - -def test_a_gain_scales_only_its_own_modality(): - inner = FakeProj() - wrapped = h3.AdalnEdit(inner, {h3.MODALITY_TAGS["audio"]: 0.5}) - base_msa, base_mlp = gates(inner(None)) - msa, mlp = gates(wrapped(None)) - for tag in (0, 1): - assert torch.equal(msa[tag::3], base_msa[tag::3]) - assert torch.equal(mlp[tag::3], base_mlp[tag::3]) - assert torch.allclose(msa[2::3], base_msa[2::3] * 0.5) - assert torch.allclose(mlp[2::3], base_mlp[2::3] * 0.5) - - -def test_only_the_gate_chunks_move(): - """shift and scale modulate what a block READS; the gate is what it WRITES.""" - inner = FakeProj() - wrapped = h3.AdalnEdit(inner, {0: 0.25}) - base, out = inner(None), wrapped(None) - for idx in (0, 1, 3, 4): - assert torch.equal(out[idx], base[idx]) - - -def test_every_timestep_row_of_that_modality_is_scaled(): - """Rows are t_row*3+tag, so a modality appears once per unique timestep.""" - inner = FakeProj(timesteps=3) - wrapped = h3.AdalnEdit(inner, {1: 2.0}) - base_msa, _ = gates(inner(None)) - msa, _ = gates(wrapped(None)) - assert msa[1::3].shape[0] == 3 - assert torch.allclose(msa[1::3], base_msa[1::3] * 2.0) - - -def test_the_source_buffer_is_not_mutated(): - """chunk() returns views onto one tensor — scaling in place would corrupt the module.""" - inner = FakeProj() - before = inner.buf.clone() - h3.AdalnEdit(inner, {0: 0.1})(None) - h3.AdalnEdit(inner, {0: 0.1})(None) - assert torch.equal(inner.buf, before) - - -def test_gain_of_one_is_dropped_entirely(): - wrapped = h3.AdalnEdit(FakeProj(), {0: 1.0, 1: 1.0}) - assert wrapped.gains == {} - - -def test_a_single_modality_projection_is_left_alone(): - """FinalLayer uses AdalnProj too, with modalities=1 — different row meaning.""" - inner = FakeProj(modalities=1, expand=2) - wrapped = h3.AdalnEdit(inner, {0: 0.5}) - assert all(torch.equal(a, b) for a, b in zip(wrapped(None), inner(None))) - - -# ── attaching it to a model ───────────────────────────────────────────────── - -class FakeBlock(torch.nn.Module): - def __init__(self): - super().__init__() - self.adaln_proj = FakeProj() - - -class FakePatcher: - def __init__(self, n=4): - self.objects = {"diffusion_model.blocks": [FakeBlock() for _ in range(n)]} - for i in range(n): - self.objects[f"diffusion_model.blocks.{i}.adaln_proj"] = \ - self.objects["diffusion_model.blocks"][i].adaln_proj - self.patched = {} - - def get_model_object(self, name): - if name in self.patched: - return self.patched[name] - return self.objects[name] - - def clone(self): - other = FakePatcher.__new__(FakePatcher) - other.objects = self.objects - other.patched = dict(self.patched) - return other - - def add_object_patch(self, name, obj): - self.patched[name] = obj - - -def test_all_ones_returns_the_same_model_unpatched(): - model = FakePatcher() - out, note = h3.apply_adaln_edits(model, {"video": 1.0, "text": 1.0, "audio": 1.0}) - assert out is model and note is None - assert model.patched == {} - - -def test_every_block_is_wrapped(): - model = FakePatcher(n=4) - out, note = h3.apply_adaln_edits(model, {"video": 0.8}) - assert out is not model - assert len(out.patched) == 4 - assert all(isinstance(v, h3.AdalnEdit) for v in out.patched.values()) - assert "4 blocks" in note - - -def test_it_says_the_anchor_moves_with_the_video(): - """Pins and reference images carry the VIDEO tag — a real limit, stated not hidden.""" - _out, note = h3.apply_adaln_edits(FakePatcher(), {"video": 0.8}) - assert "reference images ride the VIDEO tag" in note - - -def test_object_patches_are_used_not_method_replacement(): - """ComfyUI restores object patches on unpatch. Degradation outliving a reset has been a - real bug here; this is the mechanism that stops it.""" - src = inspect.getsource(h3.apply_adaln_edits) - assert "add_object_patch" in src - assert "model.clone()" in src - - -def test_an_unknown_model_declines_with_a_reason(): - class NoBlocks: - def get_model_object(self, name): - raise KeyError(name) - out, note = h3.apply_adaln_edits(NoBlocks(), {"video": 0.5}) - assert isinstance(out, NoBlocks) - assert "no DiT blocks" in note - - -def test_unknown_modality_names_are_ignored(): - model = FakePatcher() - out, note = h3.apply_adaln_edits(model, {"nonsense": 0.5}) - assert out is model and note is None - - -# ── independence from the refinement path ────────────────────────────────── - -def test_the_installer_reads_no_rating_or_key(): - """Explicitly separate from conditioning steering: turning value guidance off must not - turn this off, and this must work with no Refiner in the graph at all.""" - import samplers - fn = samplers.FunPackLTXAVSceneChainSampler._install_h3_adaln_gains - src = inspect.getsource(fn) - # The prose says what it does NOT read, so check the code, not the docstring. - doc = inspect.getdoc(fn) or "" - for line in doc.splitlines(): - src = src.replace(line, "") - for forbidden in ("refinement_key", "value_fn", "phrase_memory", - "global_state", "conditioning_deltas", "refine_v2"): - assert forbidden not in src, f"{forbidden} leaked into a sampler-side op" - - -def test_the_installer_takes_the_model_and_the_conditioning_only(): - """The conditioning is the bridge: Studio tags the learned gains onto it. The sampler - never opens a refinement key — that is the Studio/Sampler boundary.""" - import samplers - params = inspect.signature( - samplers.FunPackLTXAVSceneChainSampler._install_h3_adaln_gains).parameters - assert list(params) == ["self", "model", "positive"] - - -def _sampler(mode="learned", **widgets): - import samplers - node = samplers.FunPackLTXAVSceneChainSampler.__new__( - samplers.FunPackLTXAVSceneChainSampler) - node._h3_gain_mode = mode - for key, value in {"video": 1.0, "prompt": 1.0, "audio": 1.0}.items(): - setattr(node, f"_h3_gain_{key}", widgets.get(key, value)) - node._h3_prompt_scale = widgets.get("prompt_scale", 1.0) - return node - - -def _tagged(gains): - return [[torch.zeros(1, 4, 8), {"funpack_h3_gains": gains}]] - - -def test_learned_gains_win_over_the_widgets(): - node = _sampler(video=1.3, prompt=0.7) - out = node._h3_render_gains(_tagged({"video": 0.9, "prompt": 1.1, - "audio": 1.0, "prompt_scale": 1.2})) - assert out["video"] == pytest.approx(0.9) - assert out["prompt"] == pytest.approx(1.1) - assert out["prompt_scale"] == pytest.approx(1.2) - - -def test_manual_mode_uses_the_widgets_and_ignores_the_key(): - node = _sampler(mode="manual", video=1.3) - out = node._h3_render_gains(_tagged({"video": 0.5})) - assert out["video"] == pytest.approx(1.3) - - -def test_manual_mode_works_with_no_conditioning_at_all(): - """The explicit override has to survive a graph with no Refiner in it.""" - assert _sampler(mode="manual", video=0.8)._h3_render_gains(None)["video"] == pytest.approx(0.8) - - -def test_learned_mode_before_any_rating_renders_at_trained_strength(): - """Not the widgets: in learned mode those are not what the user is steering with, and - silently honouring them would make an unrated run look like a learned one.""" - node = _sampler(video=1.3, prompt=0.6) - out = node._h3_render_gains([[torch.zeros(1, 4, 8), {}]]) - assert out == node.H3_GAIN_NEUTRAL - - -def test_a_missing_key_in_the_learned_dict_falls_back_to_neutral(): - out = _sampler()._h3_render_gains(_tagged({"video": 0.9})) - assert out["video"] == pytest.approx(0.9) - assert out["audio"] == 1.0 and out["prompt_scale"] == 1.0 - - -def test_learned_is_the_default_mode(): - import samplers - spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES() - fields = {**spec.get("required", {}), **spec.get("optional", {})} - assert fields["h3_gain_mode"][0] == ["learned", "manual"] - assert fields["h3_gain_mode"][1]["default"] == "learned" - - -def test_the_widgets_exist_and_default_to_untouched(): - import samplers - spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES() - fields = {**spec.get("required", {}), **spec.get("optional", {})} - for name in ("h3_gain_video", "h3_gain_prompt", "h3_gain_audio"): - assert name in fields, name - assert fields[name][1]["default"] == 1.0 - - -def test_sample_accepts_them_with_defaults(): - import samplers - params = inspect.signature(samplers.FunPackLTXAVSceneChainSampler.sample).parameters - for name in ("h3_gain_video", "h3_gain_prompt", "h3_gain_audio"): - assert params[name].default == 1.0 diff --git a/tests/test_h3_gain_learning.py b/tests/test_h3_gain_learning.py deleted file mode 100644 index 5248e53e..00000000 --- a/tests/test_h3_gain_learning.py +++ /dev/null @@ -1,362 +0,0 @@ -"""The H3 render gains are learned from ratings, not typed. - -A handful of scalars is a SMALLER search than the sigma profile, which already converges on -ratings alone — so a hand-tuned value here is the user doing work the loop can do. Same -update as the sigma profile with one difference: that profile is centred on 0, so its applied -value IS its perturbation; most of these are centred on 1.0, so the perturbation is -`applied - value`. - -`refiner_bias` is the exception on both counts: it is centred on 0.0 (a signed push along the -learned taste direction, where 0 means no push) and it explores a narrower band, so its -exploration width scales down to match or its perturbations would clip against the bounds and -the reward would be credited to a move that never fully happened. -""" -import sys -import types -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -import conditioning as C - - -@pytest.fixture -def studio(): - return C.FunPackVideoRefinerV2.__new__(C.FunPackVideoRefinerV2) - - -def profile(reward, **kw): - return dict({"reward": reward, "key": "like" if reward > 0 else "awful"}, **kw) - - -def test_a_fresh_key_starts_neutral(studio): - state = studio._ensure_h3_gain_state({}) - assert set(state["values"]) == set(studio.H3_GAIN_KEYS) - # Not all neutrals are 1.0: refiner_bias is a signed push, so its neutral is 0.0. - assert state["values"] == {k: studio._h3_gain_centre(k) for k in studio.H3_GAIN_KEYS} - - -def test_a_liked_perturbation_is_moved_toward(studio): - g = {} - studio._ensure_h3_gain_state(g) - g["h3_gains"]["last_applied"]["video"] = 1.10 # rendered above centre - studio._v2_update_h3_gains(g, profile(1.0)) - assert g["h3_gains"]["values"]["video"] > 1.0 - - -def test_a_disliked_perturbation_is_moved_away_from(studio): - g = {} - studio._ensure_h3_gain_state(g) - g["h3_gains"]["last_applied"]["video"] = 1.10 - studio._v2_update_h3_gains(g, profile(-1.0)) - assert g["h3_gains"]["values"]["video"] < 1.0 - - -def test_the_sign_of_the_perturbation_matters_not_just_the_reward(studio): - """Rendering BELOW centre and liking it must move the value DOWN.""" - g = {} - studio._ensure_h3_gain_state(g) - g["h3_gains"]["last_applied"]["video"] = 0.90 - studio._v2_update_h3_gains(g, profile(1.0)) - assert g["h3_gains"]["values"]["video"] < 1.0 - - -def test_no_perturbation_teaches_nothing(studio): - g = {} - studio._ensure_h3_gain_state(g) # last_applied == values - studio._v2_update_h3_gains(g, profile(1.0)) - assert g["h3_gains"]["values"] == {k: studio._h3_gain_centre(k) for k in studio.H3_GAIN_KEYS} - - -def test_values_stay_inside_the_band(studio): - g = {} - studio._ensure_h3_gain_state(g) - for _ in range(400): - g["h3_gains"]["last_applied"]["video"] = studio.H3_GAIN_MAX - studio._v2_update_h3_gains(g, profile(1.0)) - assert g["h3_gains"]["values"]["video"] <= studio.H3_GAIN_MAX - - -def test_exploration_narrows_on_success_and_widens_on_failure(studio): - g = {} - studio._ensure_h3_gain_state(g) - start = g["h3_gains"]["explore"] - studio._v2_update_h3_gains(g, profile(1.0)) - assert g["h3_gains"]["explore"] < start - for _ in range(5): - studio._v2_update_h3_gains(g, profile(-1.0)) - assert g["h3_gains"]["explore"] > start * 0.99 - - -def test_a_forget_rating_teaches_nothing(studio): - g = {} - studio._ensure_h3_gain_state(g) - g["h3_gains"]["last_applied"]["video"] = 1.2 - studio._v2_update_h3_gains(g, profile(1.0, skip_learning=True)) - assert g["h3_gains"]["values"]["video"] == 1.0 - assert g["h3_gains"]["iterations"] == 0 - - -# ── what gets rendered ───────────────────────────────────────────────────── - -def test_a_run_is_perturbed_off_the_learned_centre(studio): - """Without a perturbation the next rating has nothing to credit.""" - g = {} - out = studio._v2_h3_gains_for_run(g, seed=7) - assert any(v != 1.0 for v in out.values()) - - -def test_the_perturbation_is_recorded_for_the_next_rating(studio): - g = {} - out = studio._v2_h3_gains_for_run(g, seed=7) - assert g["h3_gains"]["last_applied"] == out - - -def test_the_same_seed_repeats_the_same_perturbation(studio): - a = studio._v2_h3_gains_for_run({}, seed=11) - b = studio._v2_h3_gains_for_run({}, seed=11) - assert a == b - assert studio._v2_h3_gains_for_run({}, seed=12) != a - - -def test_exploration_can_be_switched_off(studio): - assert (studio._v2_h3_gains_for_run({}, explore=False) - == {k: studio._h3_gain_centre(k) for k in studio.H3_GAIN_KEYS}) - - -def test_rendered_values_stay_inside_the_band(studio): - g = {} - studio._ensure_h3_gain_state(g) - g["h3_gains"]["explore"] = 5.0 # absurd, to prove the clamp - out = studio._v2_h3_gains_for_run(g, seed=3) - for key, value in out.items(): - low, high = studio._h3_gain_bounds(key) - assert low <= value <= high - - -# ── the bridge to the sampler ────────────────────────────────────────────── - -def test_the_gains_ride_entry_zero_of_the_conditioning(studio, monkeypatch): - import torch - monkeypatch.setattr(C, "_log", types.SimpleNamespace( - failed=lambda *a, **k: None, note_on_change=lambda *a, **k: None, feature=lambda *a, **k: None), raising=False) - cond = [[torch.zeros(1, 4, 8), {}], [torch.zeros(1, 4, 8), {}]] - out = studio._v2_tag_h3_gains(cond, {}, seed=5) - assert "funpack_h3_gains" in out[0][1] - assert "funpack_h3_gains" not in out[1][1] - - -def test_neutral_gains_tag_nothing(studio, monkeypatch): - import torch - monkeypatch.setattr(studio, "_v2_h3_gains_for_run", - lambda *a, **k: {key: studio._h3_gain_centre(key) - for key in studio.H3_GAIN_KEYS}, raising=False) - cond = [[torch.zeros(1, 4, 8), {}]] - assert studio._v2_tag_h3_gains(cond, {}, seed=5) is cond - - -def test_learning_runs_on_every_rating_not_behind_a_toggle(): - """The toggle gates APPLICATION; the key must keep growing either way, the same rule - value_guidance already follows.""" - import inspect - src = inspect.getsource(C.FunPackVideoRefinerV2._v2_learn_scene_into_state) - assert "_v2_update_h3_gains(target_global, profile)" in src - - -# ── axis-aware credit assignment ─────────────────────────────────────────── -# -# A rating carries three per-axis signals in [-1, +1] as well as its scalar reward, and -# they disagree with it in exactly the cases that matter. "Missing concept" is reward -# +0.10 — quality was fine — but concept_signal -1.00. Read as the scalar it nudges every -# gain toward the run that missed the prompt; read per axis it pushes the prompt gains -# away from it and leaves the audio gain out of it. - -def test_missing_concept_punishes_the_prompt_gains(studio): - prof = C.RATING_PROFILES["Missing concept"] - assert prof["reward"] > 0 # the scalar says "slightly good" - assert studio._h3_gain_credit("prompt", prof) == pytest.approx(-1.0) - assert studio._h3_gain_credit("prompt_scale", prof) == pytest.approx(-1.0) - - -def test_missing_concept_does_not_drag_the_video_gain_down_with_it(studio): - prof = C.RATING_PROFILES["Missing concept"] - assert studio._h3_gain_credit("video", prof) > 0 # detail and quality were fine - - -def test_missing_details_punishes_the_video_gain(studio): - prof = C.RATING_PROFILES["Missing details"] - assert studio._h3_gain_credit("video", prof) < 0 - assert studio._h3_gain_credit("prompt", prof) > 0 # the prompt landed - - -def test_audio_has_no_rated_axis_so_it_uses_the_overall_reward(studio): - for label in ("Perfect", "Awful", "Missing concept"): - prof = C.RATING_PROFILES[label] - assert studio._h3_gain_credit("audio", prof) == pytest.approx(prof["reward"]) - - -def test_perfect_credits_everything_fully(studio): - prof = C.RATING_PROFILES["Perfect"] - for key in studio.H3_GAIN_KEYS: - assert studio._h3_gain_credit(key, prof) == pytest.approx(1.0) - - -def test_awful_punishes_everything(studio): - prof = C.RATING_PROFILES["Awful"] - for key in studio.H3_GAIN_KEYS: - assert studio._h3_gain_credit(key, prof) < 0 - - -def test_a_profile_with_no_signals_falls_back_to_the_scalar(studio): - assert studio._h3_gain_credit("prompt", {"reward": 0.4}) == pytest.approx(0.4) - - -def test_a_satisfied_axis_cannot_mask_a_failed_one(studio): - """Averaging let it: detail -1.00 with quality +0.85 came out at -0.08, so a rating that - said details were missing barely moved the gain that governs detail.""" - prof = {"reward": 0.0, "detail_signal": -1.0, "quality_signal": 1.0} - assert studio._h3_gain_credit("video", prof) == pytest.approx(-1.0) - - -def test_missing_details_now_moves_the_video_gain_decisively(studio): - assert studio._h3_gain_credit("video", C.RATING_PROFILES["Missing details"]) \ - == pytest.approx(-1.0) - - -def test_both_axes_satisfied_still_credits_fully(studio): - prof = {"reward": 1.0, "detail_signal": 1.0, "quality_signal": 1.0} - assert studio._h3_gain_credit("video", prof) == pytest.approx(1.0) - - -def test_axis_credit_actually_drives_the_update(studio): - """The end-to-end consequence: one rating moves two gains in OPPOSITE directions.""" - g = {} - studio._ensure_h3_gain_state(g) - for key in ("prompt", "video"): - g["h3_gains"]["last_applied"][key] = 1.10 # both rendered above centre - studio._v2_update_h3_gains(g, dict(C.RATING_PROFILES["Missing concept"])) - assert g["h3_gains"]["values"]["prompt"] < 1.0 # prompt missed -> back off - assert g["h3_gains"]["values"]["video"] > 1.0 # picture was fine -> keep - - -def test_every_gain_key_has_an_axis_entry(studio): - assert set(studio.H3_GAIN_AXES) == set(studio.H3_GAIN_KEYS) - - -# ── the learned taste direction ──────────────────────────────────────────── -# -# `refiner_bias` says HOW FAR to push; this is WHICH WAY. The direction is the same -# `liked_dir` conditioning ascent already steers along — the difference is where it lands. -# Studio hands it over as a unit vector and the sampler decides the magnitude, because the -# magnitude belongs to the refiner's space and Studio cannot see it. - -def _tensor(values): - import torch - return torch.tensor(values, dtype=torch.float32) - - -def _liked(count, direction=(3.0, 4.0)): - return {"liked_dir": {"direction": C.tensor_to_serializable(_tensor(direction)), - "direction_count": count}} - - -def test_no_direction_before_enough_liked_runs(studio): - """Two liked runs is a coincidence. The threshold is the one liked_dir's existing - conditioning-space consumers already use — not a new number invented here.""" - assert studio.H3_TASTE_DIR_MIN_COUNT == 3 - assert studio._h3_taste_direction(_liked(2)) is None - - -def test_the_direction_arrives_once_the_threshold_is_met(studio): - assert studio._h3_taste_direction(_liked(3)) is not None - - -def test_the_direction_is_a_unit_vector(studio): - out = studio._h3_taste_direction(_liked(5, (3.0, 4.0))) - assert float(out.norm()) == pytest.approx(1.0) - assert out.tolist() == pytest.approx([0.6, 0.8]) - - -def test_a_zero_direction_is_refused(studio): - """Dividing by its norm would produce NaN and poison every prompt row.""" - assert studio._h3_taste_direction(_liked(9, (0.0, 0.0))) is None - - -def test_an_empty_state_has_no_direction(studio): - assert studio._h3_taste_direction({}) is None - assert studio._h3_taste_direction(None) is None - - -def test_the_direction_rides_entry_zero_with_the_gains(studio, monkeypatch): - import torch - monkeypatch.setattr(C, "_log", types.SimpleNamespace( - failed=lambda *a, **k: None, note_on_change=lambda *a, **k: None, feature=lambda *a, **k: None), raising=False) - cond = [[torch.zeros(1, 4, 8), {}], [torch.zeros(1, 4, 8), {}]] - out = studio._v2_tag_h3_gains(cond, _liked(4), seed=5) - assert studio.H3_TASTE_DIR_META in out[0][1] - assert studio.H3_TASTE_DIR_META not in out[1][1] - - -def test_a_direction_alone_is_worth_tagging(studio, monkeypatch): - """Neutral gains used to mean "tag nothing" — but a direction with a zero push is still - worth carrying, because the sampler's manual mode can supply the push itself.""" - import torch - monkeypatch.setattr(C, "_log", types.SimpleNamespace( - failed=lambda *a, **k: None, note_on_change=lambda *a, **k: None, feature=lambda *a, **k: None), raising=False) - monkeypatch.setattr(studio, "_v2_h3_gains_for_run", - lambda *a, **k: {key: studio._h3_gain_centre(key) - for key in studio.H3_GAIN_KEYS}, raising=False) - cond = [[torch.zeros(1, 4, 8), {}]] - out = studio._v2_tag_h3_gains(cond, _liked(4), seed=5) - assert out is not cond and studio.H3_TASTE_DIR_META in out[0][1] - - -# ── the bias explores its own band ───────────────────────────────────────── - -def test_the_bias_is_centred_on_no_push(studio): - assert studio._h3_gain_centre("refiner_bias") == 0.0 - assert studio._h3_gain_centre("video") == 1.0 - - -def test_the_bias_can_be_learned_negative(studio): - """Pushing AWAY from the liked direction must be reachable, or the loop can only agree.""" - low, high = studio._h3_gain_bounds("refiner_bias") - assert low < 0.0 < high - - -def test_exploration_scales_to_the_band_it_explores(studio): - """One stored `explore` drives every gain; a narrower band has to shrink it or its - perturbations clip against the bounds and the credit goes to a move that never happened.""" - wide = studio._h3_gain_explore("video", 0.05) - narrow = studio._h3_gain_explore("refiner_bias", 0.05) - assert narrow < wide - low, high = studio._h3_gain_bounds("refiner_bias") - assert narrow == pytest.approx(0.05 * (high - low) / (studio.H3_GAIN_MAX - studio.H3_GAIN_MIN)) - - -def test_a_liked_bias_perturbation_is_moved_toward(studio): - g = {} - studio._ensure_h3_gain_state(g) - g["h3_gains"]["last_applied"]["refiner_bias"] = 0.10 - studio._v2_update_h3_gains(g, profile(1.0)) - assert g["h3_gains"]["values"]["refiner_bias"] > 0.0 - - -def test_a_disliked_bias_perturbation_is_moved_away_from(studio): - g = {} - studio._ensure_h3_gain_state(g) - g["h3_gains"]["last_applied"]["refiner_bias"] = 0.10 - studio._v2_update_h3_gains(g, profile(-1.0)) - assert g["h3_gains"]["values"]["refiner_bias"] < 0.0 - - -def test_the_bias_stays_inside_its_own_band_not_the_shared_one(studio): - g = {} - studio._ensure_h3_gain_state(g) - low, high = studio._h3_gain_bounds("refiner_bias") - for _ in range(400): - g["h3_gains"]["last_applied"]["refiner_bias"] = high - studio._v2_update_h3_gains(g, profile(1.0)) - assert low <= g["h3_gains"]["values"]["refiner_bias"] <= high diff --git a/tests/test_h3_token_refiner_edit.py b/tests/test_h3_token_refiner_edit.py deleted file mode 100644 index 1c85b362..00000000 --- a/tests/test_h3_token_refiner_edit.py +++ /dev/null @@ -1,331 +0,0 @@ -"""Editing the token refiner's OUTPUT rather than the Qwen tensor before it. - -H3 takes `context` [1, L, 5120] through condition_proj (-> 5376) and a 2-block token -refiner that ends in an RMSNorm. Everything FunPack steers today happens BEFORE that, so -the refiner processes the edit: its attention mixes it across tokens and its final norm -renormalizes the magnitude away. An edit applied AFTER lands as set, in the space the 50 -blocks consume. - -Two edits: `scale` (how loudly the prompt is read through attention) and `bias` (a single -[hidden] vector added to every prompt row — prompt-LENGTH-INDEPENDENT, unlike anything -per-position, because Qwen does not pad and H3's sequence length moves with every edit). -""" -import inspect -import sys -from pathlib import Path - -import pytest -import torch - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -import minimax_h3 as h3 - -HID = 6 - - -class FakeRefiner(torch.nn.Module): - def forward(self, x, *a, **k): - return torch.ones(x.shape[-2], HID) - - -class FakePatcher: - def __init__(self, proj_weight=None): - self.objects = {"diffusion_model.token_refiner": FakeRefiner()} - if proj_weight is not None: - self.objects["diffusion_model.condition_proj"] = \ - torch.nn.Linear(proj_weight.shape[1], proj_weight.shape[0]) - with torch.no_grad(): - self.objects["diffusion_model.condition_proj"].weight.copy_(proj_weight) - self.patched = {} - - def get_model_object(self, name): - if name in self.patched: - return self.patched[name] - if name not in self.objects: - raise KeyError(name) - return self.objects[name] - - def clone(self): - other = FakePatcher.__new__(FakePatcher) - other.objects, other.patched = self.objects, dict(self.patched) - return other - - def add_object_patch(self, name, obj): - self.patched[name] = obj - - -def run(edit, rows=8): - return edit(torch.zeros(1, rows, 4)) - - -# ── the edit ──────────────────────────────────────────────────────────────── - -def test_scale_multiplies_the_rows(): - out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=0.5)) - assert torch.allclose(out, torch.full((8, HID), 0.5)) - - -def test_rows_before_the_start_are_untouched(): - """The label and vision block sit at the head of this span.""" - out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=0.0, row_start=3)) - assert torch.all(out[:3] == 1.0) - assert torch.all(out[3:] == 0.0) - - -def test_a_row_end_bounds_the_other_side(): - out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=2.0, row_start=2, row_end=5)) - assert torch.all(out[:2] == 1.0) and torch.all(out[5:] == 1.0) - assert torch.all(out[2:5] == 2.0) - - -def test_a_bias_is_added_to_every_row_in_range(): - bias = torch.arange(HID, dtype=torch.float32) - out = run(h3.TokenRefinerEdit(FakeRefiner(), bias=bias, row_start=1)) - assert torch.allclose(out[1], 1.0 + bias) - assert torch.allclose(out[5], 1.0 + bias) # same vector on every row - assert torch.all(out[0] == 1.0) - - -def test_the_bias_is_length_independent(): - """The property that makes it a usable learning target: one vector, any prompt length.""" - bias = torch.full((HID,), 0.25) - short = run(h3.TokenRefinerEdit(FakeRefiner(), bias=bias), rows=4) - long = run(h3.TokenRefinerEdit(FakeRefiner(), bias=bias), rows=40) - assert torch.allclose(short[0], long[0]) - - -def test_scale_and_bias_compose_scale_first(): - bias = torch.full((HID,), 1.0) - out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=2.0, bias=bias)) - assert torch.allclose(out, torch.full((8, HID), 3.0)) # 1*2 + 1 - - -def test_an_out_of_range_start_is_a_no_op_not_a_crash(): - out = run(h3.TokenRefinerEdit(FakeRefiner(), scale=0.0, row_start=99)) - assert torch.all(out == 1.0) - - -def test_the_inner_output_is_not_mutated_in_place(): - inner = FakeRefiner() - edit = h3.TokenRefinerEdit(inner, scale=0.0) - run(edit) - assert torch.all(inner(torch.zeros(1, 8, 4)) == 1.0) - - -# ── attaching it ──────────────────────────────────────────────────────────── - -def test_no_edit_returns_the_model_unpatched(): - model = FakePatcher() - out, note = h3.apply_token_refiner_edit(model, scale=1.0, bias=None) - assert out is model and note is None and model.patched == {} - - -def test_it_patches_the_forward_not_the_module(): - """The module itself must stay in place: swapping it for a wrapper MODULE renames every - weight under it (token_refiner.blocks.* -> token_refiner.inner.blocks.*), and ComfyUI - records weight names off the live tree while the patch is applied. An unwrap by any - other patcher sharing the model then leaves a restore walking a `.inner` that is gone — - 'TokenRefiner object has no attribute inner', mid-sample.""" - model = FakePatcher() - out, note = h3.apply_token_refiner_edit(model, scale=1.2, row_start=4) - assert "diffusion_model.token_refiner" not in out.patched - assert isinstance(out.patched["diffusion_model.token_refiner.forward"], h3.TokenRefinerEdit) - assert "scale=1.2" in note and "rows 4:end" in note - - -def test_the_edit_is_not_a_module_so_the_state_dict_is_untouched(): - assert not isinstance(h3.TokenRefinerEdit(FakeRefiner()), torch.nn.Module) - assert not isinstance(h3.AdalnEdit(FakeRefiner(), {}), torch.nn.Module) - - -def test_reinstalling_unwraps_instead_of_nesting(): - """Per scene we install again off a shared model. ComfyUI restores an object patch by - writing the previous value back, so the previous `forward` can be a live wrapper — and - capturing that would stack another scale on every scene.""" - inner = FakeRefiner() - first = h3.TokenRefinerEdit(inner, scale=2.0) - inner.forward = first # as patch_model would leave it - second = h3.TokenRefinerEdit(inner, scale=2.0) - assert second.original is not first - assert torch.allclose(second(torch.zeros(1, 4, 4)), torch.full((4, HID), 2.0)) - - -def test_object_patches_so_nothing_outlives_the_run(): - src = inspect.getsource(h3.apply_token_refiner_edit) - assert "add_object_patch" in src and "model.clone()" in src - - -def test_a_model_without_a_refiner_declines_with_a_reason(): - class Bare: - def get_model_object(self, name): - raise KeyError(name) - out, note = h3.apply_token_refiner_edit(Bare(), scale=0.5) - assert isinstance(out, Bare) and "no token_refiner" in note - - -# ── projecting a conditioning-space direction ─────────────────────────────── - -def test_a_direction_is_projected_by_the_weight_only(): - """A direction is a difference of two points, so the bias would move the origin.""" - weight = torch.randn(HID, 4) - model = FakePatcher(proj_weight=weight) - direction = torch.randn(4) - out = h3.project_into_refiner_space(model, direction) - assert torch.allclose(out, weight @ direction, atol=1e-5) - - -def test_a_mismatched_width_is_refused(): - model = FakePatcher(proj_weight=torch.randn(HID, 4)) - assert h3.project_into_refiner_space(model, torch.randn(9)) is None - - -def test_no_projection_available_returns_none(): - assert h3.project_into_refiner_space(FakePatcher(), torch.randn(4)) is None - - -# ── independence from the refinement path ─────────────────────────────────── - -def test_the_installer_never_reaches_past_the_conditioning(): - """The refiner edit is now partly rating-learned, but the SAMPLER still only applies. - - Studio owns the refinement key, the state file and the learning; the sampler reads what - Studio tagged onto the conditioning and nothing else. Reaching for the key here would put - learning on both sides of the bridge, which is the boundary this project keeps. - """ - import samplers - cls = samplers.FunPackLTXAVSceneChainSampler - src = "".join(inspect.getsource(fn) for fn in - (cls._install_h3_token_refiner, cls._h3_taste_bias_vector, cls._h3_prompt_rows)) - for forbidden in ("refinement_key", "_v2_load_state", "value_fn", "global_state", - "phrase_memory"): - assert forbidden not in src - - -def test_the_widget_defaults_to_untouched(): - import samplers - spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES() - fields = {**spec.get("required", {}), **spec.get("optional", {})} - assert fields["h3_prompt_scale"][1]["default"] == 1.0 - assert "h3_gain_prompt" in fields["h3_prompt_scale"][1]["tooltip"] # says how they differ - - -# ── the learned taste bias ────────────────────────────────────────────────── -# -# The direction comes from ratings (Studio's `liked_dir`), the magnitude does not. A learned -# direction has no natural scale in the refiner's space — those norms belong to the -# checkpoint — so the bias is a UNIT vector rescaled at apply time to a fraction of the -# span's own mean row norm. "10% of a typical prompt row" means the same thing on every -# prompt and every checkpoint; an absolute vector would be inert or catastrophic with no way -# to tell which in advance. - -class ScaledRefiner(torch.nn.Module): - """Rows with a known norm, so the relative rescale can be checked exactly.""" - - def __init__(self, value): - super().__init__() - self.value = float(value) - - def forward(self, x, *a, **k): - return torch.full((x.shape[-2], HID), self.value) - - -def test_a_relative_bias_scales_to_the_rows_it_lands_on(): - unit = torch.zeros(HID) - unit[0] = 1.0 - out = h3.TokenRefinerEdit(ScaledRefiner(2.0), bias=unit * 0.5, - bias_relative=True)(torch.zeros(1, 4, 4)) - row_norm = (torch.full((HID,), 2.0)).norm() # every row has this norm - assert out[0, 0] == pytest.approx(2.0 + 0.5 * float(row_norm), rel=1e-5) - assert out[0, 1] == pytest.approx(2.0) # off-direction untouched - - -def test_the_same_strength_means_the_same_thing_at_a_different_scale(): - """The property the relative mode exists for: transferable between checkpoints.""" - unit = torch.zeros(HID) - unit[0] = 1.0 - small = h3.TokenRefinerEdit(ScaledRefiner(1.0), bias=unit * 0.25, - bias_relative=True)(torch.zeros(1, 4, 4)) - big = h3.TokenRefinerEdit(ScaledRefiner(100.0), bias=unit * 0.25, - bias_relative=True)(torch.zeros(1, 4, 4)) - assert (small[0, 0] / 1.0) == pytest.approx(big[0, 0] / 100.0, rel=1e-4) - - -def test_an_absolute_bias_still_ignores_the_row_scale(): - unit = torch.zeros(HID) - unit[0] = 1.0 - out = h3.TokenRefinerEdit(ScaledRefiner(5.0), bias=unit * 0.5)(torch.zeros(1, 4, 4)) - assert out[0, 0] == pytest.approx(5.5) - - -def test_a_negative_strength_pushes_the_other_way(): - """Away from what was liked has to be reachable, or the loop can only ever agree.""" - unit = torch.zeros(HID) - unit[0] = 1.0 - out = h3.TokenRefinerEdit(ScaledRefiner(2.0), bias=unit * -0.5, - bias_relative=True)(torch.zeros(1, 4, 4)) - assert out[0, 0] < 2.0 - - -def test_the_note_says_the_bias_is_relative(): - model = FakePatcher() - _out, note = h3.apply_token_refiner_edit(model, bias=torch.zeros(HID), bias_relative=True) - assert "bias (relative)" in note - - -# ── the state-dict hazard that crashed a real run ─────────────────────────── -# -# Reported from a rental: manual gain mode with the refinement keys deleted, sampling dies -# with "'TokenRefiner' object has no attribute 'inner'". Learned mode was fine, because with -# no key the gains are neutral and the installer returns before building anything. -# -# The chain: a wrapper MODULE renames every weight under it, ComfyUI's patch_model applies -# object patches BEFORE loading weights (so names are recorded WITH `.inner`), and -# unpatch_model restores weights BEFORE unwrapping. With more than one patcher alive off one -# shared model — one clone per scene, plus one for the AdaLN gains — another patcher can -# unwrap first, and the weight restore then walks `.inner` on a real TokenRefiner. - -class _Refiner(torch.nn.Module): - def __init__(self): - super().__init__() - self.blocks = torch.nn.Linear(4, 4) - - def forward(self, x, *a, **k): - return torch.ones(x.shape[-2], HID) - - -class _DiT(torch.nn.Module): - def __init__(self): - super().__init__() - self.token_refiner = _Refiner() - - -class _Base(torch.nn.Module): - def __init__(self): - super().__init__() - self.diffusion_model = _DiT() - - -def test_the_patch_never_renames_a_checkpoint_weight(): - """The whole bug in one assertion: no weight key may gain an `.inner` segment.""" - model = _Base() - before = list(dict(model.named_parameters())) - edit = h3.TokenRefinerEdit(model.diffusion_model.token_refiner, scale=1.2) - model.diffusion_model.token_refiner.forward = edit # as patch_model installs it - assert list(dict(model.named_parameters())) == before - assert not any(".inner." in k for k in dict(model.named_parameters())) - - -def test_the_module_itself_stays_in_place(): - model = _Base() - original = model.diffusion_model.token_refiner - edit = h3.TokenRefinerEdit(original, scale=1.2) - model.diffusion_model.token_refiner.forward = edit - assert model.diffusion_model.token_refiner is original - assert isinstance(model.diffusion_model.token_refiner, _Refiner) - - -def test_the_adaln_gain_patches_a_forward_too(): - """Same hazard, same shape: adaln_proj.weight must not become adaln_proj.inner.weight.""" - src = inspect.getsource(h3.apply_adaln_edits) - assert "patch_module_forward" in src diff --git a/tests/test_key_scoped_knobs.py b/tests/test_key_scoped_knobs.py deleted file mode 100644 index 950b5084..00000000 --- a/tests/test_key_scoped_knobs.py +++ /dev/null @@ -1,142 +0,0 @@ -"""The learned render strengths belong to the refinement key, not to the project file. - -Reported twice from a rental: the H3 gain and taste-bias values could not be cleared. They -were learned from ratings and stored in the key's own state — but the Editor ALSO wrote them -into the project's sampler_inputs, which made a second source of truth that outlived the key -it came from. Deleting every refinement key left the old values still applying. - -The first attempt at this stripped them inside `Project.from_dict`, which was wrong in a way -no unit test caught: `from_dict` parses the SAVE payload as well as the file on disk, so the -editor sent a value, got a project back without it, and redrew the default. Every one of -these dials — including the mode that reaches them — was unsettable. Reported from the -rental as "mode automatically switches back to learned". - -The rule now, in three parts: - 1. Values are dropped when a project is READ FROM DISK, not on the save round-trip. - 2. `h3_gain_mode` is NOT in the set. A mode is a deliberate choice, so clearing it - discards user intent. - 3. In MANUAL mode nothing is dropped: there is no learned state to reset, because nothing - learned it — those are hand-typed settings like any other. -""" -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from movie_editor.backend.timeline import ( # noqa: E402 - KEY_SCOPED_SAMPLER_INPUTS, Project, without_key_scoped) - -FRONTEND = Path(__file__).resolve().parents[1] / "movie_editor" / "frontend" - - -def test_only_the_rating_learned_values_are_key_scoped(): - assert KEY_SCOPED_SAMPLER_INPUTS == { - "h3_gain_video", "h3_gain_prompt", "h3_gain_audio", - "h3_prompt_scale", "h3_taste_bias", "h3_video_detail", - } - - -def test_the_mode_itself_is_never_stripped(): - """Stripping it made manual mode unreachable: pick it, save, and the round trip handed - back `learned` again.""" - assert "h3_gain_mode" not in KEY_SCOPED_SAMPLER_INPUTS - assert without_key_scoped({"h3_gain_mode": "manual"})["h3_gain_mode"] == "manual" - - -# ── what happens on the way off disk ──────────────────────────────────────── - -def test_a_project_carrying_learned_values_is_cleaned(): - """An existing project has to heal itself; the user cannot be asked to hand-edit JSON.""" - assert without_key_scoped( - {"h3_gain_video": 0.7, "h3_taste_bias": -0.2, "cfg": 1.0}) == {"cfg": 1.0} - - -def test_unrelated_sampler_inputs_survive(): - assert without_key_scoped( - {"h3_audio_clock": True, "frame_overlap": 8, "h3_gain_audio": 1.2}) == { - "h3_audio_clock": True, "frame_overlap": 8} - - -def test_a_missing_block_is_fine(): - assert without_key_scoped(None) == {} - - -def test_manual_mode_keeps_every_value_it_was_given(): - """They are hand-typed there. Clearing them would delete work, not stale learned state.""" - stored = {"h3_gain_mode": "manual", "h3_gain_video": 0.7, "h3_taste_bias": -0.2} - assert without_key_scoped(stored) == stored - - -def test_learned_is_the_assumption_when_the_mode_is_not_recorded(): - """An older project has no mode key. Learned is the default, so treat it as learned.""" - assert without_key_scoped({"h3_gain_video": 0.7}) == {} - - -def test_it_reads_the_disk_load_path(): - import inspect - from movie_editor.backend import projects - assert "without_key_scoped" in inspect.getsource(projects.get) - - -# ── and what must NOT happen on save ──────────────────────────────────────── - -def test_parsing_a_save_payload_keeps_the_values(): - """`from_dict` runs on the browser's save as well as on the file. This is the exact - regression: strip here and the Editor can never set any of these at all.""" - project = Project.from_dict({"sampler_inputs": {"h3_gain_video": 0.7, "cfg": 1.0}}) - assert project.sampler_inputs == {"h3_gain_video": 0.7, "cfg": 1.0} - - -def test_the_mode_survives_a_save_round_trip(): - saved = Project.from_dict({"sampler_inputs": {"h3_gain_mode": "manual"}}).to_dict() - assert saved["sampler_inputs"]["h3_gain_mode"] == "manual" - - -@pytest.mark.parametrize("knob", sorted(KEY_SCOPED_SAMPLER_INPUTS)) -def test_every_learned_dial_can_be_set_within_a_session(knob): - """Applying only on the next open is the whole point: a dial moved now must take effect - now, or manual mode does nothing.""" - saved = Project.from_dict({"sampler_inputs": {knob: 0.5}}).to_dict() - assert saved["sampler_inputs"][knob] == 0.5 - - -# ── the Editor keeps offering them ────────────────────────────────────────── - -def _engine_settings(): - return (FRONTEND / "engine_settings.js").read_text(encoding="utf-8") - - -@pytest.mark.parametrize("knob", sorted(KEY_SCOPED_SAMPLER_INPUTS | {"h3_gain_mode"})) -def test_every_render_dial_is_still_offered_in_settings(knob): - """Untying a value from the project file is a storage change, not a reason to take the - dial away.""" - import re - assert re.search(r'\{\s*name:\s*"%s"' % re.escape(knob), _engine_settings()) - - -def test_the_settings_hint_says_where_the_values_live(): - assert "live in the refinement key" in _engine_settings() - - -def test_they_are_still_widgets_on_the_node(): - """A raw ComfyUI graph with no Refiner in it still drives them by hand — that is what - h3_gain_mode 'manual' is for.""" - import samplers - spec = samplers.FunPackLTXAVSceneChainSampler.INPUT_TYPES() - fields = {**spec.get("required", {}), **spec.get("optional", {})} - for knob in KEY_SCOPED_SAMPLER_INPUTS | {"h3_gain_mode"}: - assert knob in fields - - -def test_no_key_means_neutral_not_a_remembered_value(): - """With the key gone there is nothing to read, so the run must fall back to the model's - trained strengths rather than to whatever was last used.""" - import samplers - import torch - node = samplers.FunPackLTXAVSceneChainSampler.__new__( - samplers.FunPackLTXAVSceneChainSampler) - node._h3_gain_mode = "learned" - out = node._h3_render_gains([[torch.zeros(1, 4, 8), {}]]) - assert out == node.H3_GAIN_NEUTRAL diff --git a/tests/test_settings_card_noise.py b/tests/test_settings_card_noise.py index b9dfcb24..2e4c7739 100644 --- a/tests/test_settings_card_noise.py +++ b/tests/test_settings_card_noise.py @@ -28,7 +28,7 @@ def defaults(monkeypatch): "segmented_detailing": False, "detail_targets": "hands, face", "context_windows": False, "context_window_overlap": 40, "joyai_memory": False, "joyai_audio_memory": False, "v2a_grad_scale": 1.0, - "h3_gain_mode": "learned", "h3_video_detail": 1.0, + "h3_video_detail": 1.0, "cfg": 1.0, "frame_overlap": 8, } declared = frozenset(table) | {"mystery", "adjustments"} @@ -89,6 +89,29 @@ def test_an_unrecognised_refiner_key_is_kept(): assert "prompt_repair" in got +def test_an_owner_live_for_more_than_one_value_is_matched_on_any_of_them(): + """absolute_strength belongs to steer_mode and is live for BOTH "absolute" and "both". + Reading only the singular dependsValue missed the plural dependsVals the Studio rows + use, so it printed under "relative", where it does nothing.""" + assert sc._OWNED_BY["absolute_strength"] == ("steer_mode", ("absolute", "both")) + rows = dict(sc._live_rows({"steer_mode": "relative", "absolute_strength": 0.05}, + defaults=sc._EDITOR_DEFAULTS)) + assert "absolute_strength" not in rows + for mode in ("absolute", "both"): + got = dict(sc._live_rows({"steer_mode": mode, "absolute_strength": 0.05}, + defaults=sc._EDITOR_DEFAULTS)) + assert "absolute_strength" in got, mode + + +def test_a_boolean_owner_survives_the_journey_from_javascript(): + """The Editor writes JS literals, so a boolean owner arrives as "true" while Python's + own str(True) is "True".""" + assert sc._OWNED_BY["negative_erase_mode"] == ("negative_erase", ("true",)) + got = dict(sc._live_rows({"negative_erase": True, "negative_erase_mode": "subtract"}, + defaults=sc._EDITOR_DEFAULTS)) + assert got["negative_erase_mode"] == "subtract" + + def test_a_hand_built_control_still_gets_an_owner(): """identity_projector is rendered by hand in the Editor, so it carries no dependsOn for the generator to find — and it was printing a model filename under a disabled feature.""" @@ -122,11 +145,13 @@ def test_a_chain_of_owners_is_walked_to_the_end(defaults): assert "v2a_grad_scale" in got -def test_an_owner_matched_by_value_not_by_truthiness(defaults): - """h3_gain_mode is a combo: 'manual' switches its dials on, 'learned' does not — and - 'learned' is a perfectly truthy string.""" +def test_an_owner_can_be_matched_by_value_not_just_truthiness(monkeypatch, defaults): + """A combo owner switches its dials on for ONE of its values, and the others are + perfectly truthy strings. No shipped knob uses this today; the rule still has to.""" + monkeypatch.setitem(sc._OWNED_BY, "h3_video_detail", ("mode", ("manual",))) + monkeypatch.setitem(defaults, "mode", "learned") assert rows({"h3_video_detail": 1.4}) == {} - assert "h3_video_detail" in rows({"h3_gain_mode": "manual", "h3_video_detail": 1.4}) + assert "h3_video_detail" in rows({"mode": "manual", "h3_video_detail": 1.4}) def test_a_cycle_in_the_table_does_not_hang(monkeypatch, defaults): @@ -142,11 +167,20 @@ def test_the_ownership_table_is_the_editors_own(): copies of one rule in two languages, so the test regenerates this one from that one.""" src = (FRONTEND / "engine_settings.js").read_text(encoding="utf-8") found = {} - for name, body in re.findall(r'\{\s*name:\s*"([a-z0-9_]+)"(.*?)\n\s*(?=\{ name:|\];)', + for name, body in re.findall(r'\{\s*name:\s*"([a-z0-9_.]+)"(.*?)\n\s*(?=\{ name:|\];)', src, re.S): - m = re.search(r'dependsOn:\s*"([a-z0-9_]+)"(?:,\s*dependsValue:\s*"([^"]*)")?', body) - if m: - found[name] = (m.group(1), m.group(2)) + m = re.search(r'dependsOn:\s*"([a-z0-9_]+)"', body) + if not m: + continue + one = re.search(r'dependsValue:\s*"([^"]*)"', body) + many = re.search(r'dependsVals:\s*\[([^\]]*)\]', body) + if one: + vals = (one.group(1),) + elif many: + vals = tuple(v.strip().strip('"') for v in many.group(1).split(",") if v.strip()) + else: + vals = None + found[name] = (m.group(1), vals) assert found, "no dependsOn rows parsed — the parser, not the table, is what broke" assert {**found, **sc._OWNED_EXTRA} == sc._OWNED_BY