olivv-cs-ComfyUI-FunPack/modules/conditioning/prompt_markup/nodes.py

"""The two markup nodes. See the module docstring for the shape."""

import torch
import torch.nn.functional as F
from comfy_api.latest import io

from ..._core import dit_hooks, log, patching, registry as registry_mod
from . import markup as mk

MARKUP = io.Custom("FUNPACK_MARKUP")
KEY = "funpack_markup.prompt"
PHRASES = "funpack_markup_phrases"   # the transformer_options key the phrase probe reads


def _ask(capability, *args):
    """The first model module with an answer, or None."""
    for _spec, provider in registry_mod.current().providers(capability):
        got = provider(*args)
        if got is not None:
            return got
    return None


def _say(message):
    log.once(f"prompt_markup:{message}", log.ALERT, "FunPack Prompt Markup", message)


class FunPackPromptMarkup(io.ComfyNode):
    @classmethod
    def define_schema(cls) -> io.Schema:
        return io.Schema(
            node_id="FunPackPromptMarkup",
            display_name="FunPack Prompt Markup",
            category="FunPack/Conditioning",
            description="Take (word:1.5), [phrase@2-4] and [a|b] out of a prompt, so the "
                        "encoder sees clean text and the markup can be applied properly.",
            inputs=[io.String.Input("text", multiline=True, default="")],
            outputs=[io.String.Output(display_name="clean"),
                     MARKUP.Output(display_name="markup"),
                     io.String.Output(display_name="status")],
        )

    @classmethod
    def execute(cls, text: str) -> io.NodeOutput:
        parsed = mk.parse(text or "")
        n = {k: len(parsed[k]) for k in ("weighted", "timed", "blended")}
        status = (", ".join(f"{v} {k}" for k, v in n.items() if v) or "no markup")
        return io.NodeOutput(parsed["clean"], parsed, status)


def _blend(cond, rows, clip, tokenizer, clean, offsets, blended):
    """(1-s)*A + s*B over the prompt rows, each span measured against the
    ORIGINAL A so several blends add instead of compounding."""
    start, end = rows
    a_rows = cond[:, start:end, :]
    delta, done = torch.zeros_like(a_rows), 0
    for c0, c1, alt, strength in blended:
        if not mk.token_spans(offsets, [(c0, c1)]):
            continue
        alt_text = clean[:c0] + alt + clean[c1:]
        encoded = clip.encode_from_tokens_scheduled(clip.tokenize(alt_text))
        b_cond, b_meta = encoded[0][0], encoded[0][1]
        n_b = len(tokenizer(alt_text, add_special_tokens=False,
                            return_offsets_mapping=True)["offset_mapping"])
        b_rows = _ask("prompt_rows", b_meta, int(b_cond.shape[1]), n_b)
        if b_rows is None:
            continue
        b = b_cond[:, b_rows[0]:b_rows[1], :].to(a_rows)
        if b.shape[1] != a_rows.shape[1]:
            # Linear along the token axis: most rows already correspond (only
            # the swapped phrase differs); a smoothing, not a word alignment.
            b = F.interpolate(b.transpose(1, 2), size=a_rows.shape[1], mode="linear",
                              align_corners=True).transpose(1, 2)
        delta += float(strength) * (b - a_rows)
        done += 1
    if not done:
        return cond, 0
    out = cond.clone()
    out[:, start:end, :] += delta
    return out, done


class FunPackApplyPromptMarkup(io.ComfyNode):
    @classmethod
    def define_schema(cls) -> io.Schema:
        return io.Schema(
            node_id="FunPackApplyPromptMarkup",
            display_name="FunPack Apply Prompt Markup",
            category="FunPack/Conditioning",
            description="Apply the markup FunPack Prompt Markup took out: weights and "
                        "timed phrases on the model, blends on the conditioning.",
            inputs=[io.Model.Input("model"), io.Clip.Input("clip"),
                    io.Conditioning.Input("positive"), io.Latent.Input("latent"),
                    MARKUP.Input("markup")],
            outputs=[io.Model.Output(display_name="model"),
                     io.Conditioning.Output(display_name="positive"),
                     io.String.Output(display_name="status")],
        )

    @classmethod
    def execute(cls, model, clip, positive, latent, markup) -> io.NodeOutput:
        patched = patching.clone(model)
        patching.strip(patched, "funpack_markup.")
        weighted, timed, blended = (markup or {}).get("weighted"), (markup or {}).get("timed"), \
            (markup or {}).get("blended")
        if not (weighted or timed or blended):
            return io.NodeOutput(patched, positive, "no markup")

        def refuse(why):
            _say(f"markup NOT applied: {why}. The prompt still went in, without it.")
            return io.NodeOutput(patched, positive, f"not applied: {why}")

        tokenizer = _ask("text_tokenizer", clip)
        if tokenizer is None:
            return refuse("this text encoder is not one a model module can place words in")
        clean = markup["clean"]
        offsets = list(tokenizer(clean, add_special_tokens=False,
                                 return_offsets_mapping=True)["offset_mapping"])
        cond, meta = positive[0][0], positive[0][1]
        cond_len = int(cond.shape[1])
        rows = _ask("prompt_rows", meta, cond_len, len(offsets))
        if rows is None:
            return refuse("the encoded prompt does not match this text -- was the clean "
                          "text from Prompt Markup what got encoded?")
        base = rows[0]
        said = []

        if blended:
            new_cond, n = _blend(cond, rows, clip, tokenizer, clean, offsets, blended)
            if n:
                positive = [[new_cond, meta], *positive[1:]]
                said.append(f"{n} blend(s)")

        w_spans = mk.token_spans(offsets, weighted or [])
        t_spans = []
        for tok0, tok1, w, t0, t1 in mk.token_spans(offsets, timed or []):
            window = _ask("video_time_rows", latent["samples"], t0, t1)
            if window is None:
                _say(f"a timed phrase's window {t0:g}-{t1:g}s misses this clip; ignored")
                continue
            t_spans.append((tok0, tok1, w, window))
        if w_spans or t_spans:
            cls._install(patched, w_spans, t_spans, base, cond_len)
            # Where the phrases sit, for anything that wants to look at them (the phrase probe).
            dit_hooks.publish(patched, PHRASES, {
                "base": base, "cond_len": cond_len,
                "spans": [(a, b) for a, b, *_ in w_spans] + [(a, b) for a, b, *_ in t_spans]})
            if w_spans:
                said.append(f"{len(w_spans)} weight(s)")
            if t_spans:
                said.append(f"{len(t_spans)} timed phrase(s)")
        return io.NodeOutput(patched, positive, ", ".join(said) or "nothing placeable")

    @staticmethod
    def _install(patcher, w_spans, t_spans, base, cond_len):
        cache = {}

        def build(seq_len, device, dtype):
            timed, video_start = [], seq_len
            for tok0, tok1, w, (n_video, r0, r1) in t_spans:
                video_start = seq_len - n_video
                if video_start < cond_len:
                    _say("this sequence is too short to hold the video the timed "
                         "phrases were placed on; they are off for this call")
                    return mk.plan(w_spans, [], base, seq_len, seq_len, device, dtype)
                timed.append((tok0, tok1, w, video_start + r0, video_start + r1))
            return mk.plan(w_spans, timed, base, seq_len, video_start, device, dtype)

        def override(func, q, k, v, heads, *args, mask=None, **kwargs):
            def run(m, qq=q):
                return func(qq, k, v, heads, *args, mask=m, **kwargs)

            # Only the packed self-attention: the prompt plus the streams after it.
            # A sequence exactly as long as the prompt is the text-only token
            # refiner, which must never see a video window's mask.
            if k.ndim != 4 or q.shape[2] != k.shape[2] or k.shape[2] <= cond_len:
                return run(mask)
            try:
                key = (int(k.shape[2]), k.device, k.dtype)
                if key not in cache:
                    cache[key] = build(*key)
                plan = cache[key]
                if plan is None:
                    return run(mask)
                if not isinstance(plan, list):
                    return run(plan if mask is None else mask + plan)
                outs = [run(b if mask is None else mask + b, q[:, :, a:z].contiguous())
                        for a, z, b in plan]
                return torch.cat(outs, dim=2 if outs[0].ndim == 4 else 1)
            except Exception as exc:             # noqa: BLE001 -- markup never costs the step
                _say(f"markup failed inside attention and was dropped ({exc})")
                return run(mask)

        dit_hooks.add_attention_override(patcher, KEY, override)