olivv-cs-ComfyUI-FunPack/modules/sampling/momentum/__init__.py

"""Momentum Guidance: smooth the fine-motion steps with a running average of past directions.

Each step's direction (where the model says the picture should move) is averaged into a running
memory. Once the noise has fallen below a level, the current direction is blended toward that memory,
which calms jitter in the refinement steps. It is the late-step partner of ALG, which acts on the early
ones (arXiv:2602.20360). It can damp motion; that is the price of averaging.

The edit is carried into the next step's input (core/input_steer.py), never onto the last answer, so the
last step is never edited; it is sized so the NEXT model input moves as v4's blended direction would. The sampler's own latent does
not take the edit, so later steps feel about three quarters of v4's effect (v4 edits the sampler's step).
Only the picture is touched, never the sound.
"""

import torch
from comfy.patcher_extension import WrappersMP

from ..._core import dit_hooks, input_steer, log, streams

ID = "momentum"
TITLE = "Momentum guidance"
MOUNT = "generation.sampling"
STAGE = "guidance"
CATEGORY = "guidance"
STATUS = "experimental"
REQUIRES = ["temporal_latent"]

SETTINGS = {
    "enabled": {
        "type": "bool", "default": False,
        "label": "Smooth the fine motion",
        "hint": "Calms jitter in the last steps. May also soften motion.",
    },
    "strength": {
        "type": "float", "default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05,
        "label": "How much", "ui": "slider",
        "hint": "0 = nothing, 1 = follow the running average completely.",
        "when": {"enabled": True},
    },
    "decay": {
        "type": "float", "default": 0.5, "min": 0.0, "max": 0.95, "step": 0.05,
        "label": "How long it remembers", "ui": "slider",
        "hint": "High values hold on to the noisy first steps and make garbage on short schedules.",
        "when": {"enabled": True},
    },
    "below_sigma": {
        "type": "float", "default": 0.975, "min": 0.0, "max": 1.0, "step": 0.025,
        "label": "Starts at noise level", "ui": "slider",
        "hint": "Acts once the noise has dropped below this.",
        "when": {"enabled": True},
    },
}


def blend(x, out, sigma, ema, decay, strength):
    """(new answer, new memory) for one step on a video tensor. The direction is
    d = (x - answer) / sigma; the answer is rebuilt from d blended toward the memory."""
    d = ((x - out) / max(sigma, 1e-6)).detach()
    ema = d if ema is None else decay * ema + (1.0 - decay) * d
    mixed = d + strength * (ema - d)
    return x - sigma * mixed, ema


def carry_factor(sigma, sigma_next, carried=True):
    """What to multiply an x0 edit (-sigma * dd) by so the next input moves by v4's (sigma - sigma_next) * dd.
    Applied to the answer it already does: 1. Carried into the next input it arrives scaled by (1 - sigma_next)."""
    return (sigma - sigma_next) / max(sigma * (1.0 - sigma_next), 1e-6) if carried else 1.0


def install(patcher, values, key):
    if not values.get("enabled"):
        return None
    strength = max(0.0, min(1.0, float(values.get("strength", 0.5))))
    decay = max(0.0, min(0.999, float(values.get("decay", 0.5))))
    below = float(values.get("below_sigma", 0.975))
    if strength <= 0.0:
        return "strength 0: nothing to do"
    state = {"ema": {}, "last": {}, "index": {}, "multi": False}          # by cond/uncond call: they are different directions
    steer = input_steer.Steer("Momentum")
    steer.attach(patcher, key)

    def forget(executor, *args, **kwargs):
        state["ema"].clear()
        state["last"].clear()
        state["index"].clear()
        state["multi"] = False
        try:
            return executor(*args, **kwargs)
        finally:
            state["ema"].clear()                 # a latent-sized tensor must not outlive the run

    patcher.add_wrapper_with_key(WrappersMP.OUTER_SAMPLE, f"{key}.forget", forget)

    def apply_model(executor, x, t, *args, **kwargs):
        named = streams.model_args(args, kwargs)
        to = named.get("transformer_options")
        if (to or {}).get("context_window") is not None:
            log.once(f"{ID}:windows", log.ALERT, "FunPack Momentum",
                     "off this run: it cannot follow clips cut into context windows (the windows move "
                     "every step, so there is no steady direction to average)")
            return executor(x, t, *args, **kwargs)
        step = steer.begin(x, t, to)
        out = executor(step.x, t, *args, **kwargs)
        packed = x.dim() == 3
        where = dit_hooks.current_step(to)
        if not packed and where is not None and not dit_hooks.probing(to):
            now, was = where[0], state["index"].get(tuple(int(v) for v in ((to or {}).get("cond_or_uncond") or ())))
            state["index"][tuple(int(v) for v in ((to or {}).get("cond_or_uncond") or ()))] = now
            if now == was:                               # the same step twice: a second-order corrector call
                state["multi"] = True
                log.once(f"{ID}:multi", log.ALERT, "FunPack Momentum",
                         "off for the rest of this run: the model is called more than once per step "
                         "(a second-order sampler); use euler-style sampling")
            if state["multi"]:
                return out
        if not step.steering or dit_hooks.probing(to) or (not packed and where is not None and where[0] >= where[1] - 1):
            return out                                   # a probe, an off-schedule call, the last step: not a step to edit
        sigma = float(t.max())
        which = tuple(int(v) for v in ((to or {}).get("cond_or_uncond") or ()))
        index = where[0] if where else None
        before = state["last"].get(which)
        if before is not None and ((index is not None and before[0] is not None and index < before[0])
                                   or sigma > before[1] + 1e-4):
            state["ema"].pop(which, None)                # the schedule restarted: a new pass
        state["last"][which] = (index, sigma)
        seen_out, seen_in = streams.video_of(out, named), streams.video_of(step.x, named)
        if seen_out is None or seen_in is None:
            log.once(f"{ID}:layout", log.ALERT, "FunPack Momentum",
                     "off this run: could not find the picture in this model's latent")
            return out
        video, rebuild = seen_out
        ema = state["ema"].get(which)
        if ema is not None and ema.shape != video.shape:
            ema = None
        # The memory always accumulates, so the window opens on real history, not a cold start.
        new, state["ema"][which] = blend(seen_in[0].float(), video.float(), sigma, ema, decay,
                                         strength if sigma < below else 0.0)
        if sigma >= below:
            return out
        sigma_next = float(to["sample_sigmas"][where[0] + 1]) if where else 0.0
        edit = video.float() + carry_factor(sigma, sigma_next, packed) * (new - video.float())
        return step.keep(out, rebuild(edit.to(video.dtype)))

    patcher.add_wrapper_with_key(WrappersMP.APPLY_MODEL, key, apply_model)
    return f"strength {strength:g}, memory {decay:g}, from noise level {below:g}"


PROVIDES = {"modifier": install}