feat: confine H3 block repeat/span loop to the last N schedule steps
Every step was being repeated, including the earliest near-pure-noise ones where a block is still deciding layout, not detail -- the likely reason repeat re-rolls the shot instead of sharpening it. New h3_block_repeat_last_steps (0 = every step, unchanged default) windows the repeat to the tail via the exact step index off transformer_options[sample_sigmas/sigmas].
This commit is contained in:
digital-garbage
@@ -316,6 +316,9 @@
|
||||
detail: "H3 puts video, text and audio in one sequence, so a plain repeat doubles the block for the audio rows too — the leading suspect for the unprompted dialogue that band 31-40 produces on every run of it. This restores text and audio rows to their single-pass value afterwards. TWO limits, both measured: it does not undo audio's influence on the second pass (attention had already mixed the twice-processed audio into the video rows before they were restored), and it does not keep audio clean DOWNSTREAM — the blocks after the repeated ones still process audio rows while attending to twice-processed video rows, so audio can be contaminated after the fact. It protects the audio at the exit of the repeated blocks, not all the way to the output, which is why enabling it did not save the audio in span-loop tests. Off by default so the measured sweep results stay reproducible — turn it on and compare against the same seed to see whether the trade goes away." },
|
||||
{ name: "h3_block_repeat_times", label: "Extra passes per repeated block", kind: "int", default: 1, min: 1, max: 4,
|
||||
hint: "1 = the block runs twice in total. Higher pushes further from what the following blocks were trained to receive — measured: everything past ~30 collapses at 2 (including blocks that did nothing at 1), while 11-20 survives it on video but not on audio." },
|
||||
{ name: "h3_block_repeat_last_steps", label: "Only repeat the last N steps", kind: "int", default: 0, min: 0, max: 50, dependsOn: "h3_block_repeat",
|
||||
hint: "0 = repeat on every step (the original behaviour). Above 0, the repeat only fires on the final N denoise steps.",
|
||||
detail: "Every sweep result above was run with this at 0 — the repeat firing on EVERY step, including the earliest, near-pure-noise ones where the block is still deciding layout and motion, not detail. That is the likely reason repeat re-rolls the shot instead of sharpening it: doubling a block's say over what the shot even IS, before it exists yet, is a different operation than doubling its say over how a settled shot looks. Confining the repeat to the tail, after structure is already locked in by the earlier steps, is untested and may behave completely differently from every finding on this page — none of it carries over automatically. Counted in STEPS from the end of the schedule, not sigma, so the window lands on the same point of denoising regardless of step count or sampler." },
|
||||
{ name: "bounded_attention_enabled", label: "Bounded attention (experimental, untested)", kind: "bool", default: false,
|
||||
hint: "Stops two people in one frame swapping each other's features, by letting each half of the frame see only its own sentence. Nearly free, and does nothing unless the scene prompt has two sentences describing two subjects. Built but never run — the left/right split is fixed, so it suits two subjects side by side and nothing else." },
|
||||
{ name: "dynashift", label: "DynaShift (steer off bad gens)", kind: "bool", default: false,
|
||||
@@ -521,7 +524,7 @@
|
||||
chain_timing: ["frame_overlap", "transition_duration", "use_same_seed", "cut_opening_frames"],
|
||||
chain_guidance: ["cfg", "embed_guidance", "embed_guidance_source", "embed_guidance_strength", "score_slider", "score_slider_strength", "taste_nearest_prompt", "output_guidance", "output_guidance_strength", "trajectory_guidance", "trajectory_guidance_strength", "dynashift", "dynashift_strength", "dynashift_threshold", "h3_repr_steering", "h3_repr_steering_strength", "h3_repr_steering_block"],
|
||||
chain_decode: ["decode_noise_scale", "decode_timestep", "decode_tile_size"],
|
||||
chain_experimental: ["context_windows", "context_window_length", "context_window_overlap", "context_window_schedule", "context_window_fuse", "context_window_freenoise", "context_window_retain_first", "h3_video_detail", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "h3_block_repeat", "h3_block_repeat_span_loop", "h3_block_repeat_video_only", "h3_block_repeat_times", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"],
|
||||
chain_experimental: ["context_windows", "context_window_length", "context_window_overlap", "context_window_schedule", "context_window_fuse", "context_window_freenoise", "context_window_retain_first", "h3_video_detail", "joyai_memory", "joyai_memory_size", "joyai_fix_frames", "joyai_frame_select", "joyai_memory_strength", "joyai_audio_memory", "v2a_grad_scale", "alg_blur_guides", "alg_guide_blur_strength", "alg_guide_blur_sigma_threshold", "bounded_attention_enabled", "h3_block_repeat", "h3_block_repeat_span_loop", "h3_block_repeat_video_only", "h3_block_repeat_times", "h3_block_repeat_last_steps", "identity_transfer_enabled", "source_id", "phase_scale", "id_strength", "arcface_mode", "debug_log"],
|
||||
};
|
||||
|
||||
function countChainView(p, id, st) {
|
||||
|
||||
+60
-5
@@ -2986,6 +2986,10 @@ class FunPackLTXAVSceneChainSampler:
|
||||
"default": 1, "min": 1, "max": 4,
|
||||
"tooltip": "How many EXTRA passes each named block gets. 1 = the block runs twice in total. Higher pushes further from what the following blocks were trained to receive.",
|
||||
}),
|
||||
"h3_block_repeat_last_steps": ("INT", {
|
||||
"default": 0, "min": 0, "max": 50,
|
||||
"tooltip": "Only repeat during the FINAL N denoise steps of the schedule (0 = every step, the original behaviour). Early steps set layout/motion/reference from near-pure noise -- doubling a block there is what re-rolls the shot instead of refining it. Confining the repeat to the tail, after that structure is already decided, is the untested alternative. Counted in STEPS, not sigma, so it lands on the same point of the schedule regardless of step count or sampler.",
|
||||
}),
|
||||
# A connection socket, never a widget — safe at the end, and it must stay after
|
||||
# every widget above (see the widgets_values note at the top of this block).
|
||||
"second_pass_sigmas": ("SIGMAS", {
|
||||
@@ -3697,6 +3701,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
h3_repr_steering_block="25",
|
||||
h3_block_repeat="", h3_block_repeat_times=1,
|
||||
h3_block_repeat_video_only=False, h3_block_repeat_span_loop=False,
|
||||
h3_block_repeat_last_steps=0,
|
||||
refinement_key="",):
|
||||
if sampler is None:
|
||||
raise ValueError("sampler input is required.")
|
||||
@@ -3766,7 +3771,8 @@ class FunPackLTXAVSceneChainSampler:
|
||||
_install = (self._install_span_loop if h3_block_repeat_span_loop
|
||||
else self._install_block_repeat)
|
||||
model = _install(model, _repeat_blocks, h3_block_repeat_times,
|
||||
video_only=h3_block_repeat_video_only)
|
||||
video_only=h3_block_repeat_video_only,
|
||||
last_steps=h3_block_repeat_last_steps)
|
||||
# Measurement next, steering last: whatever is installed later chains this probe
|
||||
# as its inner call, so the delta it records is the block's OWN, not one that already
|
||||
# contains another mechanism's injection.
|
||||
@@ -6817,7 +6823,41 @@ class FunPackLTXAVSceneChainSampler:
|
||||
continue
|
||||
return out
|
||||
|
||||
def _install_span_loop(self, model, blocks, times=1, video_only=False):
|
||||
@staticmethod
|
||||
def _in_last_steps(args, last_steps):
|
||||
"""True when the current denoise call is inside the FINAL `last_steps` steps.
|
||||
|
||||
0 = no window, every step (how block repeat behaved before this existed).
|
||||
|
||||
Core puts the whole schedule in transformer_options["sample_sigmas"] and the current
|
||||
step's sigma in ["sigmas"], so the step index is exact -- no sigma threshold to guess
|
||||
per schedule, which matters because the same threshold lands on a different step for
|
||||
beta57@6 than for beta57@8. Index by matching the current sigma against the schedule
|
||||
(comfy/context_windows.py uses the same isclose trick).
|
||||
|
||||
Fails OPEN -- repeats on every step, the pre-window behaviour -- when the schedule is
|
||||
not exposed, and says so once, rather than silently declining to repeat at all."""
|
||||
if int(last_steps) <= 0:
|
||||
return True
|
||||
try:
|
||||
to = args.get("transformer_options") or {}
|
||||
sched, cur = to.get("sample_sigmas"), to.get("sigmas")
|
||||
if sched is None or cur is None:
|
||||
raise KeyError("sample_sigmas/sigmas")
|
||||
cur = cur.reshape(-1)[0].to(sched.dtype)
|
||||
hit = torch.nonzero(torch.isclose(sched, cur, rtol=1e-4))
|
||||
if not len(hit):
|
||||
raise ValueError("current sigma is not on the schedule")
|
||||
# sigmas holds N+1 values for N steps (the trailing 0 is never a denoise call).
|
||||
total = int(sched.shape[0]) - 1
|
||||
return int(hit[0]) >= total - int(last_steps)
|
||||
except Exception as _e: # noqa: BLE001
|
||||
_log.failed("FunPackSceneChain", "H3 block repeat step window", _e,
|
||||
"the repeat runs on EVERY step instead of only the last "
|
||||
f"{int(last_steps)}", key="h3_block_repeat_last_steps")
|
||||
return True
|
||||
|
||||
def _install_span_loop(self, model, blocks, times=1, video_only=False, last_steps=0):
|
||||
"""EXPERIMENTAL (H3): run a CONTIGUOUS SPAN of blocks, then send the result back to the
|
||||
start of the span and run it again -- 31,32..40,31,32..40 -- instead of repeating each
|
||||
block on itself -- 31,31,32,32..40,40.
|
||||
@@ -6869,6 +6909,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
patches_replace = dict(to.get("patches_replace", {}))
|
||||
dit_patches = dict(patches_replace.get("dit", {}))
|
||||
_times, _video_only = int(times), bool(video_only)
|
||||
_last_steps = int(last_steps)
|
||||
if _video_only:
|
||||
try:
|
||||
from . import h3_repr_steering as _rs_mask
|
||||
@@ -6878,6 +6919,13 @@ class FunPackLTXAVSceneChainSampler:
|
||||
|
||||
def _head(args, extra):
|
||||
h = args["img"]
|
||||
if not self._in_last_steps(args, _last_steps):
|
||||
# Outside the window the span still has to RUN -- every other block in it
|
||||
# is a pass-through now -- it just runs once, as it would with this off.
|
||||
for _i in range(lo, hi + 1):
|
||||
h = dit[_i](h, args["t_emb"], args["mod_segments"], args["rope_freqs"],
|
||||
transformer_options=args.get("transformer_options"))
|
||||
return {"img": h}
|
||||
before = h if _video_only else None
|
||||
for _pass in range(_times + 1):
|
||||
for _i in range(lo, hi + 1):
|
||||
@@ -6914,13 +6962,14 @@ class FunPackLTXAVSceneChainSampler:
|
||||
patched.model_options["transformer_options"] = to
|
||||
print(f"[FunPackSceneChain] H3 span loop: blocks {lo}-{hi} run {_times + 1}x as a "
|
||||
f"span (wrap {hi}->{lo}, one seam)"
|
||||
f"{', video rows only' if _video_only else ''}.")
|
||||
f"{', video rows only' if _video_only else ''}"
|
||||
f"{f', on the last {_last_steps} step(s) only' if _last_steps > 0 else ''}.")
|
||||
return patched
|
||||
except Exception as _e: # noqa: BLE001
|
||||
_log.failed("FunPackSceneChain", "H3 span loop", _e, "blocks run once as usual")
|
||||
return model
|
||||
|
||||
def _install_block_repeat(self, model, blocks, times=1, video_only=False):
|
||||
def _install_block_repeat(self, model, blocks, times=1, video_only=False, last_steps=0):
|
||||
"""EXPERIMENTAL (H3): run a block on its OWN output before passing it on.
|
||||
|
||||
`x' = x + f(x)` then `x'' = x' + f(x')`. Because the delta is small, f(x') is close to
|
||||
@@ -6956,6 +7005,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
patches_replace = dict(to.get("patches_replace", {}))
|
||||
dit_patches = dict(patches_replace.get("dit", {}))
|
||||
_times = int(times)
|
||||
_last_steps = int(last_steps)
|
||||
|
||||
_video_only = bool(video_only)
|
||||
if _video_only:
|
||||
@@ -6972,6 +7022,8 @@ class FunPackLTXAVSceneChainSampler:
|
||||
run = (lambda a: extra["original_block"](a)["img"]) if inner is None else \
|
||||
(lambda a: inner(a, extra)["img"])
|
||||
out = run(args)
|
||||
if not self._in_last_steps(args, _last_steps):
|
||||
return {"img": out}
|
||||
first = out if _video_only else None
|
||||
for _ in range(_times):
|
||||
# A fresh args dict each pass: the block reads "img" from it, and
|
||||
@@ -7001,7 +7053,8 @@ class FunPackLTXAVSceneChainSampler:
|
||||
print(f"[FunPackSceneChain] H3 block repeat: block(s) "
|
||||
f"{', '.join(str(b) for b in sorted(blocks))} run {_times + 1}x "
|
||||
f"(+{_times} extra pass each)"
|
||||
f"{', video rows only' if _video_only else ''}.")
|
||||
f"{', video rows only' if _video_only else ''}"
|
||||
f"{f', on the last {_last_steps} step(s) only' if _last_steps > 0 else ''}.")
|
||||
return patched
|
||||
except Exception as _e: # noqa: BLE001
|
||||
_log.failed("FunPackSceneChain", "H3 block repeat", _e, "blocks run once as usual")
|
||||
@@ -7382,6 +7435,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
h3_repr_steering_block="25",
|
||||
h3_block_repeat="", h3_block_repeat_times=1,
|
||||
h3_block_repeat_video_only=False, h3_block_repeat_span_loop=False,
|
||||
h3_block_repeat_last_steps=0,
|
||||
alg_guide_blur_strength=2.0, alg_guide_blur_sigma_threshold=0.975,
|
||||
alg_anchor=False, alg_anchor_strength=2.0, alg_anchor_sigma_threshold=0.975,
|
||||
identity_transfer_enabled=False, identity_projector="None", source_id=2.0,
|
||||
@@ -8280,6 +8334,7 @@ class FunPackLTXAVSceneChainSampler:
|
||||
h3_block_repeat_times=h3_block_repeat_times,
|
||||
h3_block_repeat_video_only=h3_block_repeat_video_only,
|
||||
h3_block_repeat_span_loop=h3_block_repeat_span_loop,
|
||||
h3_block_repeat_last_steps=h3_block_repeat_last_steps,
|
||||
refinement_key=refinement_key_input,
|
||||
)
|
||||
# cut_opening_frames: let the real, untouched i2v anchor condition the scene
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""H3 block repeat / span loop: the last-N-steps window that confines the repeat to the
|
||||
tail of the schedule instead of firing on every step (see engine_settings.js hint for why)."""
|
||||
import sys
|
||||
|
||||
import torch
|
||||
|
||||
sys.path.insert(0, ".")
|
||||
from samplers import FunPackLTXAVSceneChainSampler as S # noqa: E402
|
||||
|
||||
|
||||
def _args(step, total=6):
|
||||
# sigmas holds N+1 values for N steps; sample_sigmas is the full descending schedule.
|
||||
sched = torch.linspace(1.0, 0.0, total + 1)
|
||||
return {"transformer_options": {"sample_sigmas": sched, "sigmas": sched[step:step + 1]}}
|
||||
|
||||
|
||||
def test_zero_means_every_step():
|
||||
for step in range(6):
|
||||
assert S._in_last_steps(_args(step), 0) is True
|
||||
|
||||
|
||||
def test_last_two_of_six():
|
||||
hits = [S._in_last_steps(_args(step), 2) for step in range(6)]
|
||||
assert hits == [False, False, False, False, True, True]
|
||||
|
||||
|
||||
def test_window_wider_than_schedule_is_every_step():
|
||||
assert all(S._in_last_steps(_args(step), 50) for step in range(6))
|
||||
|
||||
|
||||
def test_missing_schedule_fails_open():
|
||||
assert S._in_last_steps({"transformer_options": {}}, 2) is True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_zero_means_every_step()
|
||||
test_last_two_of_six()
|
||||
test_window_wider_than_schedule_is_every_step()
|
||||
test_missing_schedule_fails_open()
|
||||
print("ok")
|
||||
Reference in New Issue
Block a user