olivv-cs-ComfyUI-FunPack/modules/sampling/shot_memory/__init__.py

"""Shot memory: the starting noise remembers the shots you liked.

With the prompt fixed, what a like or dislike reacts to is mostly the SHOT -- where
things sit, how the frame is split. The model decides that from the coarse part of
the starting noise, so a rating lands on the thing that caused it instead of on an
averaged direction in a space thousands of dimensions wide.

Each run's coarse noise (one value per 4x4-latent cell per channel, averaged over
the whole clip) waits on the taste key for a rating. The next run then either
reuses a liked shot -- its coarse noise blended into the fresh noise, the fine
noise always fresh so the details are new -- or starts fresh. Both choices learn
from ratings (Thompson draws): whether reusing beats fresh for this key, which
liked shot, and how strongly (one number, nudged toward what was liked). A liked
child is a shot in its own right, so good shots breed.

The noise stays unit Gaussian: swapping one block mean for another of the same
variance changes nothing the model can tell apart from ordinary noise. Zero model
calls. Sound noise is never touched. A latent that is not empty (a second pass, an
anchor) is left alone, and the run says so.

Validated on a GPU. v4 lessons kept: reusing a shot's own seed stacks its layout
1.41x (the blend is capped per channel, only ever turned down), and a one-cell grid
has no spread (left alone).
"""

import math
import random

import torch
import torch.nn.functional as F
from comfy.patcher_extension import WrappersMP

from ..._core import dit_hooks, log, registry, streams

ID = "shot_memory"
TITLE = "Shot memory"
MOUNT = "generation.sampling"
STAGE = "latent"
CATEGORY = "sampling"
STATUS = "experimental"
REQUIRES = ["temporal_latent"]
USES = ["taste_store"]

SETTINGS = {
    "enabled": {
        "type": "bool", "default": False,
        "label": "Shot memory",
        "hint": "Some runs start from the layout of a shot you liked, with new details; ratings tune how often.",
    },
    "mode": {
        "type": "enum", "default": "learned", "options": [
            {"value": "learned", "label": "Learn from ratings"},
            {"value": "manual", "label": "My value"},
        ],
        "label": "How much", "ui": "segmented",
        "hint": "Learn from ratings: each rating nudges how much is reused toward what you liked. My value: you set it below.",
        "when": {"enabled": True},
    },
    "amount": {
        "type": "float", "default": 0.7, "min": 0.0, "max": 0.95, "step": 0.05,
        "label": "Reuse amount", "ui": "slider",
        "hint": "How much of the liked layout carries over. 0 = none.",
        "when": {"enabled": True, "mode": "manual"},
    },
}

KIND = "shot_memory"
# Latent pixels per coarse cell: 64 image px on H3's 16x VAE.
# ponytail: fixed; learn it per key if one cell size stops fitting every resolution.
CELL = 4
MAX_ROWS = 64
START_AMOUNT = 0.7
EXPLORE = 0.15
LR = 0.5
AMOUNT_MIN, AMOUNT_MAX = 0.2, 0.95


def _say(message, level=log.ALERT):
    log.once(f"{ID}:{message}", level, "FunUI Shot memory", message)


def _sign(x):
    return (x > 0) - (x < 0)


def coarse_of(video):
    """[1, C, T, H, W] noise -> its coarse part [C, h, w], unit variance, or None."""
    if video.dim() != 5 or video.shape[0] != 1:
        return None
    _b, c, t, H, W = video.shape
    h, w = H // CELL, W // CELL
    if h < 2 or w < 2:          # a layout needs cells to compare; one cell has no spread
        return None
    blocks = video[0, :, :, :h * CELL, :w * CELL].float().reshape(c, t, h, CELL, w, CELL)
    return blocks.mean(dim=(1, 3, 5)) * math.sqrt(t * CELL * CELL)


def with_coarse(video, coarse):
    """`video` with its coarse part replaced by `coarse` [C, h, w]; the fine part kept."""
    _b, c, t, _H, _W = video.shape
    h, w = coarse.shape[-2:]
    core = video[0, :, :, :h * CELL, :w * CELL].float().reshape(c, t, h, CELL, w, CELL)
    delta = coarse.to(core) / math.sqrt(t * CELL * CELL) - core.mean(dim=(1, 3, 5))
    core = core + delta[:, None, :, None, :, None]
    out = video.clone()
    out[0, :, :, :h * CELL, :w * CELL] = core.reshape(c, t, h * CELL, w * CELL).to(video.dtype)
    return out


def fit(coarse, h, w, device=None):
    """A stored coarse map on this run's grid and device (stored maps live on the CPU; the
    noise is on the GPU). Another resolution is resampled and re-standardised: the layout
    survives, exact Gaussianity does not."""
    coarse = coarse.float().to(device) if device is not None else coarse.float()
    if tuple(coarse.shape[-2:]) == (h, w):
        return coarse
    x = F.interpolate(coarse[None], size=(h, w), mode="bilinear", align_corners=False)[0]
    return (x - x.mean()) / x.std().clamp(min=1e-6)


def pooled(cond):
    if not torch.is_tensor(cond):
        return None
    p = cond.detach().float()
    while p.dim() > 1:
        p = p.mean(dim=0)
    return p.cpu()


def shots(rows):
    """The key's rated shots, oldest first, as plain dicts."""
    out = []
    for r in rows:
        p = r["rows"]
        if "coarse" in p and "id" in p:
            out.append({"id": int(p["id"]), "parent": int(p["parent"]), "amount": float(p["amount"]),
                        "reward": float(r["reward"]), "coarse": p["coarse"], "cond": p.get("cond")})
    return out


def learned_amount(rows):
    """How strongly to reuse, replayed from the rated reuses in order: a liked one pulls
    the value toward the amount it used, a disliked one pushes it away."""
    m = START_AMOUNT
    for r in rows:
        if r["parent"] < 0:
            continue
        m += LR * _sign(r["reward"]) * (r["amount"] - m)
        m = min(max(m, AMOUNT_MIN), AMOUNT_MAX)
    return m


def choose_parent(rows, cond, channels, rng=random):
    """-> (liked shot to reuse | None, why). Two Thompson draws: reuse-vs-fresh from how
    reuses and fresh shots have been rated, then which liked shot, weighted by how alike
    its prompt was."""
    by_id = {r["id"] for r in rows}
    reuse_ab, fresh_ab, kids = [1.0, 1.0], [1.0, 1.0], {}
    for r in rows:
        s = _sign(r["reward"])
        if not s:
            continue
        (fresh_ab if r["parent"] < 0 else reuse_ab)[0 if s > 0 else 1] += 1.0
        if r["parent"] in by_id:
            kids.setdefault(r["parent"], [1.0, 1.0])[0 if s > 0 else 1] += 1.0
    liked = [r for r in rows if r["reward"] > 0 and int(r["coarse"].shape[0]) == int(channels)]
    if not liked:
        return None, "fresh: no liked shot yet"
    reuse, fresh_ = rng.betavariate(*reuse_ab), rng.betavariate(*fresh_ab)
    if fresh_ >= reuse:
        return None, f"fresh (reuse {reuse:.2f} < fresh {fresh_:.2f})"
    best, best_theta = None, -1.0
    for r in liked:
        a, b = kids.get(r["id"], [1.0, 1.0])
        theta = rng.betavariate(a + 1.0, b)              # +1: its own like
        stored = r["cond"]
        if cond is not None and torch.is_tensor(stored) and stored.numel() == cond.numel():
            # ponytail: raw pooled-prompt cosine; sits high for most prompts under one key.
            theta *= float(F.cosine_similarity(cond, stored.float(), dim=0).clamp(0.0, 1.0))
        if theta > best_theta:
            best, best_theta = r, theta
    return best, f"reusing a liked shot of {len(liked)} (reuse {reuse:.2f} > fresh {fresh_:.2f})"


def blend(own, parent_coarse, amount):
    """The layout to start from: the liked shot's, `amount` of the way."""
    coarse = amount * fit(parent_coarse, *own.shape[-2:], device=own.device) + math.sqrt(1.0 - amount ** 2) * own
    # Unit variance only holds when the two layouts are unrelated. Rerunning the liked
    # shot's own seed makes them the same pattern and the blend stacks it (1.41x at 0.7):
    # back to a normal-strength layout, per channel. Only ever down: a few-cell grid's
    # measured spread can read low by chance, and dividing by it would strengthen it.
    return coarse / coarse.std(dim=(1, 2), keepdim=True).clamp(min=1.0)


def _prompt_of(guider):
    try:
        return pooled(guider.conds["positive"][0]["cross_attn"])
    except (AttributeError, KeyError, IndexError, TypeError):
        return None


def install(patcher, values, key):
    if not values.get("enabled"):
        return None
    taste = registry.current().ask("taste_store", patcher)
    if taste is None:
        _say("off: set a Taste key first -- it is what your ratings teach")
        return None
    manual = values.get("mode") == "manual"
    manual_amount = min(max(float(values.get("amount", START_AMOUNT)), 0.0), AMOUNT_MAX)
    live = {"rows": [], "plan": None}

    def fresh():
        live["rows"], live["plan"] = shots(taste.rows(KIND, blind_to="image")), None
        liked = sum(1 for r in live["rows"] if r["reward"] > 0)
        log.once(f"{ID}:state", log.INFO, "FunUI Shot memory",
                 f"key {taste.key!r}: {liked} liked shot(s) of {len(live['rows'])} rated")

    captured = taste.collect(patcher, key, KIND, keep=MAX_ROWS, fresh=fresh)

    def plan(cond, channels):
        if live["plan"] is None:
            parent, why = choose_parent(live["rows"], cond, channels)
            if parent is not None and manual and manual_amount <= 0.0:
                parent, why = None, "fresh (reuse amount is 0)"       # not a reuse: do not record one
            amount = 0.0
            if parent is not None:
                amount = manual_amount if manual else min(max(
                    learned_amount(live["rows"]) + random.gauss(0.0, EXPLORE), AMOUNT_MIN), AMOUNT_MAX)
                why += f", amount {amount:.2f}"
            log.once(f"{ID}:plan", log.INFO, "FunUI Shot memory", why)
            live["plan"] = (parent, amount)
        return live["plan"]

    def sampler_sample(executor, model_wrap, sigmas, extra_args, callback, noise,
                       latent_image=None, denoise_mask=None, disable_pbar=False):
        def run(n):
            return executor(model_wrap, sigmas, extra_args, callback, n,
                            latent_image, denoise_mask, disable_pbar)

        if dit_hooks.stage_of(model_wrap) == 2:
            return run(noise)                        # the layout was set in the first stage
        named = {"latent_shapes": getattr(getattr(model_wrap, "inner_model", None), "latent_shapes", None)}
        split = streams.video_of(noise, named)
        if split is None:
            _say("off this run: could not find the picture in this model's noise")
            return run(noise)
        video, rebuild = split
        base = streams.video_of(latent_image, named) if latent_image is not None else None
        if base is None or bool(torch.count_nonzero(base[0])):
            _say("Inactive | the starting latent is not empty (a second pass, an anchor or a "
                 "start video), so the layout is already set and shot memory leaves this run's noise alone")
            return run(noise)
        own = coarse_of(video)
        if own is None:
            _say(f"Inactive | the picture is under {2 * CELL}x{2 * CELL} latent px (two layout "
                 "cells a side) or batched, so shot memory leaves it alone")
            return run(noise)
        cond = _prompt_of(model_wrap)
        parent, amount = plan(cond, int(own.shape[0]))
        coarse = own
        if parent is not None:
            coarse = blend(own, parent["coarse"], amount)
            video = with_coarse(video, coarse)
        captured.update({
            "id": torch.tensor(random.getrandbits(62)),
            "parent": torch.tensor(parent["id"] if parent else -1),
            "amount": torch.tensor(float(amount)), "coarse": coarse.half().cpu()})
        if cond is not None:
            captured["cond"] = cond.half()
        return run(rebuild(video))

    patcher.add_wrapper_with_key(WrappersMP.SAMPLER_SAMPLE, key, sampler_sample)
    return ("learned" if not manual else f"manual {manual_amount:g}") + ": starts from a liked layout when one is drawn"


PROVIDES = {"modifier": install}