olivv-cs-ComfyUI-FunPack/modules/sampling/stas/__init__.py

"""STAS: steer the massive activations of a video DiT (arXiv 2603.17825).

A few channels of a video transformer's hidden state carry huge values ("massive
activations"), strongest on the first frame and on the edges of each latent frame. They
act as the model's own rescalers. STAS sets them, at one block and over the first steps
only, to alpha x their current peak:

    g(i, d) = alpha * max_j |H[j, d]| * sign(H[i, d])     for i in S, d in M

S = every token of the first latent frame + the first and last p% of every frame's
tokens; M = channels whose peak is over 50x the mean magnitude (at most 8). Paper
settings: one block ~30% deep, first 20 of 50 steps, p = 8%, alpha 1.2-2.5 (tested on
Wan and CogVideoX, not H3). No extra model call. Alpha is learned from ratings
(core/rated_dial.py) or typed.

Whether H3 has massive channels at the chosen block is unknown: the run says which it
found, or that it steered nothing. v4's user saw motion get "much more fluid and
detailed" with this stack on; STAS is the prime suspect, not isolated.

Keep this block different from late-branch guidance's branch block: at the same block the
weak copy lacks this edit and guidance would amplify it.

Switches ComfyUI's model compiler off for the run (an in-place edit of one block's output).
"""

import math

import torch
from comfy.patcher_extension import WrappersMP

from ..._core import dit_hooks, log, rated_dial, registry, streams

ID = "stas"
TITLE = "Massive-activation steering"
MOUNT = "generation.sampling"
STAGE = "guidance"
CATEGORY = "guidance"
STATUS = "experimental"
REQUIRES = ["temporal_latent", "dit_block_hooks"]
USES = ["taste_store"]
USES_WHEN = {"mode": "learned"}      # "My value" never asks the store

SETTINGS = {
    "enabled": {
        "type": "bool", "default": False,
        "label": "Massive-activation steering",
        "hint": "Boosts the model's own rescaling channels on the first frame and frame edges, early on.",
    },
    "mode": {
        "type": "enum", "default": "learned", "options": [
            {"value": "learned", "label": "Learn from ratings"},
            {"value": "manual", "label": "My value"},
        ],
        "label": "How strong", "ui": "segmented",
        "hint": "Learn from ratings: each rating nudges how much the boost is toward what you liked. My value: you set it below.",
        "when": {"enabled": True},
    },
    "alpha": {
        "type": "float", "default": 2.0, "min": 0.5, "max": 3.0, "step": 0.1,
        "label": "Alpha", "ui": "slider",
        "hint": "Higher boosts more. The paper used 1.2-2.5.",
        "when": {"enabled": True, "mode": "manual"},
    },
    "block": {
        "type": "int", "default": 15, "min": 0, "max": 49,
        "label": "Block",
        "hint": "About 30% deep, as in the paper (15 of H3's 50).",
        "when": {"enabled": True},
    },
}

KIND = "stas"
DIAL = rated_dial.Dial(start=2.0, lo=0.5, hi=3.0, explore=0.2)
EARLY_FRACTION = 0.4        # first 20 of 50 steps
EDGE = 0.08                 # p: head and tail share of each frame's tokens
MA_RATIO = 50.0
MAX_DIMS = 8


def _say(message, level=log.ALERT):
    log.once(f"{ID}:{message}", level, "FunUI STAS", message)


def first_steps(total):
    """How many of `total` steps are steered."""
    return max(1, math.ceil(EARLY_FRACTION * int(total)))


def target_rows(frames, per_frame, edge=EDGE):
    """Row indices (within the video rows) of S: all of frame 0, then the head and tail
    `edge` share of every frame. Rows are frame-major, as H3 packs them."""
    k = max(1, round(edge * per_frame))
    within = torch.cat([torch.arange(k), torch.arange(per_frame - k, per_frame)]).unique()
    rows = (torch.arange(frames)[:, None] * per_frame + within[None, :]).flatten()
    return torch.cat([torch.arange(per_frame), rows]).unique()


def ma_dims(video):
    """-> (channel indices, their peaks, their peak/mean ratios). `video` is [n, D]."""
    peak = torch.maximum(video.amax(dim=0).float().abs(), video.amin(dim=0).float().abs())
    mean = video[::max(1, video.shape[0] // 1024)].float().abs().mean().clamp(min=1e-12)
    ratio = peak / mean
    dims = torch.nonzero(ratio > MA_RATIO).flatten()
    if dims.numel() > MAX_DIMS:
        dims = dims[ratio[dims].topk(MAX_DIMS).indices]
    return dims, peak[dims], ratio[dims]


def steer(video, rows, dims, peaks, alpha):
    """Set video[rows, dims] to alpha * peak * sign, in place."""
    if not dims.numel() or not rows.numel():
        return video
    r, d = rows.to(video.device)[:, None], dims.to(video.device)[None, :]
    target = (alpha * peaks.to(video.device))[None, :] * torch.sign(video[r, d].float())
    video[r, d] = target.to(video.dtype)
    return video


def install(patcher, values, key):
    if not values.get("enabled"):
        return None
    n = dit_hooks.block_count(patcher)
    block = int(values.get("block", 15))
    if not 0 <= block < n:
        _say(f"off: block {block} is not a block of this model (0-{n - 1})")
        return None
    manual = values.get("mode") == "manual"
    taste = None if manual else registry.current().ask("taste_store", patcher)
    if not manual and taste is None:
        _say("off: learning needs a Taste key -- set one, or pick 'My value'")
        return None
    patch = tuple(getattr(getattr(getattr(patcher, "model", None), "diffusion_model", None),
                          "patch_size", (1, 2, 2)))
    live = {"alpha": float(values.get("alpha", 2.0)), "per_frame": None, "spans": {},
            "steered": 0, "empty": 0, "dims": None, "why": None}

    def fresh():
        live.update(spans={}, steered=0, empty=0, dims=None, why=None)
        if manual:
            return log.once(f"{ID}:state", log.INFO, "FunUI STAS",
                            f"manual alpha {live['alpha']:.2f} at block {block}")
        live["alpha"], centre, held = DIAL.pick(rated_dial.history(taste.rows(KIND, blind_to="image"), {"b": block}))
        log.once(f"{ID}:state", log.INFO, "FunUI STAS",
                 f"key {taste.key!r}: learned alpha {centre:.2f} from {held} rating(s), trying "
                 f"{live['alpha']:.2f} at block {block}")

    captured = taste.collect(patcher, key, KIND, fresh=fresh) if not manual else {}

    def finish(executor, *args, **kwargs):
        if manual:
            fresh()
        out = executor(*args, **kwargs)
        if live["steered"]:
            log.once(f"{ID}:result", log.INFO, "FunUI STAS",
                     f"Active | alpha {live['alpha']:.2f} at block {block}: steered "
                     f"{live['steered']} call(s), channels {live['dims']}"
                     + (f"; {live['empty']} call(s) had no massive channel" if live["empty"] else ""))
            if not manual:
                captured["v"] = torch.tensor(live["alpha"])
                captured["b"] = torch.tensor(block)       # an alpha means something per block
        else:
            _say("Inactive | " + (live["why"] or f"no channel at block {block} is over "
                                  f"{MA_RATIO:.0f}x the mean, nothing steered -- try another block"))
        return out

    patcher.add_wrapper_with_key(WrappersMP.OUTER_SAMPLE, key, finish)

    def apply_model(executor, x, t, *args, **kwargs):
        shapes = streams.model_args(args, kwargs).get("latent_shapes") or ()
        video = [tuple(s) for s in shapes if len(tuple(s)) == 5]
        if video:
            _b, _c, _t, h, w = max(video, key=lambda s: math.prod(s))
            live["per_frame"] = -(-int(h) // patch[1]) * -(-int(w) // patch[2])
        return executor(x, t, *args, **kwargs)

    patcher.add_wrapper_with_key(WrappersMP.APPLY_MODEL, key, apply_model)

    def hook(args, extra):
        out = extra["original_block"](args)["img"]
        to = args.get("transformer_options")
        where = dit_hooks.current_step(to)
        if where is None:
            live["why"] = "no step schedule to read, nothing steered"
            return {"img": out}
        if where[0] >= first_steps(where[1]):
            return {"img": out}
        if not live["per_frame"]:
            live["why"] = "the picture's frame size was not seen (no latent shapes), nothing steered"
            return {"img": out}
        seq = int(out.shape[0])
        span = live["spans"].get(seq, "unset")
        if span == "unset":
            rows = dit_hooks.row_span(dit_hooks.target_rows(args.get("mod_segments"), seq, out.device, "video"))
            span = None
            if rows is not None and (rows[1] - rows[0]) % live["per_frame"] == 0:
                count = rows[1] - rows[0]
                span = (rows[0], count, target_rows(count // live["per_frame"], live["per_frame"]))
            live["spans"][seq] = span
        if span is None:
            live["why"] = "picture rows not found as whole frames in this sequence, nothing steered"
            return {"img": out}
        a, count, rows = span
        picture = out[a:a + count]                         # a view: steered in place
        dims, peaks, ratios = ma_dims(picture)
        if not dims.numel():
            live["empty"] += 1
            return {"img": out}
        steer(picture, rows, dims, peaks, live["alpha"])
        live["steered"] += 0 if dit_hooks.probing(to) else 1
        if live["dims"] is None:
            live["dims"] = ", ".join(f"{int(d)} ({float(r):.0f}x)" for d, r in zip(dims, ratios))
        return {"img": out}

    dit_hooks.add_block_hook(patcher, key, block, hook)
    dit_hooks.without_compiler(patcher, key)
    return (("manual" if manual else "learned") + f" alpha at block {block}, first "
            f"{EARLY_FRACTION:.0%} of steps, frame 0 + frame edges, picture rows")


PROVIDES = {"modifier": install}