nodes/olivv-cs-ComfyUI-FunPack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
8ae739ad9cffcffd80db76ec1f6fea49c5a4f49c
olivv-cs-ComfyUI-FunPack/conditioning.py
T
digital-garbageandClaude Opus 5 8ae739ad9c perf(studio): embed the category descriptions once per encoder, not once per run 
CATEGORY_DESCRIPTIONS is a class attribute of eight fixed strings, so the
vectors depend only on the encoder. They were re-embedded every run — eight
full passes of Qwen3-VL-32B on H3 — to arrive at the same eight vectors.

Held for the process, keyed by the encoder, behind a WEAK reference: pinning a
text encoder in VRAM to save eight passes would be the wrong trade. An identity
check sits on top of the id() key because ids are reused once a model is freed,
and a stale hit would classify phrases against another encoder's geometry.

Not a cache of generated content — nothing user-typed goes into it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-25 13:34:08 +03:00

12987 lines
645 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import base64
import gc
import glob
import json
import math
import os
import random
import re
import time
import weakref
from datetime import datetime, timezone
from hashlib import md5
from typing import Optional
import numpy as np
import torch
try:
from . import funpack_log as _log
except ImportError: # flat import when ComfyUI loads the pack as a top-level module
import funpack_log as _log
import torch.nn.functional as F
from PIL import Image, ImageDraw, ImageFont
from safetensors.torch import load_file
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
import folder_paths
V2_SEED_REUSE_CHANCE = 0.20
V2_SEED_MEMORY_MAX_CONCEPTS = 128
V2_SEED_MEMORY_MAX_PER_CONCEPT = 24
# Path-outcome memory (path-planner Phase 1). A "path" = a per-scene generation config
# (concepts + family + guidance dims) EXCLUDING the seed; the seed is the random draw
# WITHIN an arm. Every rating — not just successful ones — is recorded as a labeled
# outcome on the arm it came from, so a later planner can route away from arms that keep
# producing disliked results. Counts decay so an arm's recent behavior dominates.
V2_PATH_OUTCOME_MAX_ARMS = 256
V2_PATH_OUTCOME_MAX_SEEDS_PER_ARM = 16
V2_PATH_OUTCOME_DECAY = 0.9
LORA_REFINER_TYPE_PROFILES = {
"general": {"step": 0.025, "max_offset": 0.20, "min_offset": -0.35, "bad_max_offset": 0.45, "bad_min_offset": -1.35, "culprit_bias": 0.28},
"action": {"step": 0.046, "max_offset": 0.35, "min_offset": -0.45, "bad_max_offset": 0.75, "bad_min_offset": -2.10, "culprit_bias": 0.16},
"style": {"step": 0.032, "max_offset": 0.28, "min_offset": -0.38, "bad_max_offset": 0.58, "bad_min_offset": -1.55, "culprit_bias": 0.20},
"quality": {"step": 0.022, "max_offset": 0.18, "min_offset": -0.30, "bad_max_offset": 0.38, "bad_min_offset": -1.20, "culprit_bias": 0.18},
"character": {"step": 0.024, "max_offset": 0.20, "min_offset": -0.32, "bad_max_offset": 0.42, "bad_min_offset": -1.30, "culprit_bias": 0.10},
}
RATING_LABELS = [
"-Just forget it-",
"Perfect",
"Missing details",
"Missing concept",
"Missing quality",
"Missing details + concept",
"Missing details + quality",
"Missing concept + quality",
"Awful",
]
RATING_PROFILES = {
"-Just forget it-": {
"key": "forget",
"level": 0,
"legacy_score": 0,
"legacy_range": "ignored",
"reward": 0.0,
"quality_signal": 0.0,
"concept_signal": 0.0,
"detail_signal": 0.0,
"missing_axes": [],
"prompt_emphasis": 0.0,
"skip_learning": True,
},
"Initial discovery": {
"key": "discover",
"level": 5,
"legacy_score": 6,
"legacy_range": "discovery",
"reward": 0.0,
"quality_signal": 0.0,
"concept_signal": 0.0,
"detail_signal": 0.0,
"missing_axes": [],
"prompt_emphasis": 0.0,
},
"Perfect": {
"key": "like",
"level": 8,
"legacy_score": 10,
"legacy_range": "keep",
"reward": 1.0,
"quality_signal": 1.0,
"concept_signal": 1.0,
"detail_signal": 1.0,
"missing_axes": [],
"prompt_emphasis": 0.45,
},
"Missing details": {
"key": "missing_details",
"level": 7,
"legacy_score": 8,
"legacy_range": "boost details",
"reward": 0.45,
"quality_signal": 0.85,
"concept_signal": 0.70,
"detail_signal": -1.0,
"missing_axes": ["details"],
"prompt_emphasis": 0.65,
},
"Missing concept": {
"key": "missing_concept",
"level": 6,
"legacy_score": 6,
"legacy_range": "boost concept",
"reward": 0.10,
"quality_signal": 0.85,
"concept_signal": -1.0,
"detail_signal": 0.20,
"missing_axes": ["concept"],
"prompt_emphasis": 0.95,
},
"Missing quality": {
"key": "missing_quality",
"level": 5,
"legacy_score": 4,
"legacy_range": "boost quality",
"reward": -0.25,
"quality_signal": -1.0,
"concept_signal": 0.65,
"detail_signal": 0.65,
"missing_axes": ["quality"],
"prompt_emphasis": 0.55,
},
"Missing details + concept": {
"key": "missing_details_concept",
"level": 4,
"legacy_score": 5,
"legacy_range": "boost details+concept",
"reward": -0.05,
"quality_signal": 0.75,
"concept_signal": -1.0,
"detail_signal": -1.0,
"missing_axes": ["details", "concept"],
"prompt_emphasis": 1.05,
},
"Missing details + quality": {
"key": "missing_details_quality",
"level": 3,
"legacy_score": 3,
"legacy_range": "boost details+quality",
"reward": -0.35,
"quality_signal": -1.0,
"concept_signal": 0.50,
"detail_signal": -1.0,
"missing_axes": ["details", "quality"],
"prompt_emphasis": 0.82,
},
"Missing concept + quality": {
"key": "missing_concept_quality",
"level": 2,
"legacy_score": 2,
"legacy_range": "boost concept+quality",
"reward": -0.55,
"quality_signal": -1.0,
"concept_signal": -1.0,
"detail_signal": 0.15,
"missing_axes": ["concept", "quality"],
"prompt_emphasis": 1.02,
},
"Awful": {
"key": "awful",
"level": 1,
"legacy_score": 2,
"legacy_range": "boost all",
"reward": -0.85,
"quality_signal": -1.0,
"concept_signal": -1.0,
"detail_signal": -1.0,
"missing_axes": ["details", "concept", "quality"],
"prompt_emphasis": 1.20,
"rollback_on_failure": True,
},
}
RATING_ALIASES = {
"I like it": "Perfect",
"I don't like it": "Awful",
"Missing details + concepts": "Missing details + concept",
"Missing concept + details": "Missing details + concept",
"Missing quality + details": "Missing details + quality",
"Missing quality + concept": "Missing concept + quality",
"Missing everything": "Awful",
}
CATEGORY_FEEDBACK_MAP = {
1: "general",
2: "concept",
3: "style",
4: "quality",
5: "character",
6: "details",
}
def _clamp(value, low, high):
return max(low, min(high, value))
def normalize_refiner_rating(value):
if isinstance(value, str):
cleaned = value.strip()
cleaned = RATING_ALIASES.get(cleaned, cleaned)
if cleaned in RATING_PROFILES:
return dict(RATING_PROFILES[cleaned], label=cleaned)
try:
value = int(float(cleaned))
except (TypeError, ValueError):
return dict(RATING_PROFILES["Missing concept"], label="Missing concept")
try:
legacy_score = int(value)
except (TypeError, ValueError):
legacy_score = RATING_PROFILES["Missing concept"]["legacy_score"]
legacy_score = int(_clamp(legacy_score, 1, 10))
if legacy_score >= 9:
label = "Perfect"
elif legacy_score >= 7:
label = "Missing details"
elif legacy_score >= 5:
label = "Missing concept"
elif legacy_score >= 3:
label = "Missing quality"
else:
label = "Awful"
profile = dict(RATING_PROFILES[label], label=label)
profile["legacy_score"] = legacy_score
return profile
PROTECTED_PHRASE_RE = re.compile(r'"([^"]+)"|“([^”]+)”|\\([^\\]+)\\')
def tensor_to_serializable(t: torch.Tensor) -> dict:
if not isinstance(t, torch.Tensor):
raise TypeError(f"Expected torch.Tensor, got {type(t)}")
arr = t.detach().cpu().numpy()
return {
"data": base64.b64encode(arr.tobytes()).decode("utf-8"),
"shape": list(arr.shape),
"dtype": str(arr.dtype)
}
def serializable_to_tensor(d: dict) -> torch.Tensor:
arr = np.frombuffer(base64.b64decode(d["data"]), dtype=d["dtype"]).reshape(d["shape"]).copy()
tensor = torch.from_numpy(arr).to(dtype=torch.float32)
if torch.cuda.is_available():
tensor = tensor.cuda()
return tensor
def _safe_float(value, fallback=0.0):
try:
return float(value)
except (TypeError, ValueError):
return fallback
def _safe_int(value, fallback=0):
try:
return int(value)
except (TypeError, ValueError):
return fallback
def _to_image_tensor(image):
arr = np.asarray(image).astype(np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
def render_refinement_loss_graph(refinement_key, scheduler_mode, mode, total_iterations, latest_learning_loss, points, width=960, height=540):
image = Image.new("RGBA", (width, height), (0, 0, 0, 255))
draw = ImageDraw.Draw(image)
font = ImageFont.load_default()
bg = (24, 28, 34)
panel = (30, 36, 44)
grid = (64, 76, 92)
axis = (122, 137, 156)
text = (235, 239, 244)
subtext = (168, 180, 194)
line = (74, 201, 255)
fill = (74, 201, 255, 72)
point_color = (255, 181, 71)
image.paste(bg, (0, 0, width, height))
draw.rounded_rectangle((16, 16, width - 16, height - 16), radius=18, fill=panel, outline=(49, 58, 70), width=1)
draw.text((32, 28), "FunPack Refinement Loss", fill=text, font=font)
draw.text((32, 50), f"Embedding: {refinement_key}", fill=text, font=font)
draw.text((32, 68), f"Scheduler: {scheduler_mode.upper()} Mode: {mode.upper()} Iterations: {total_iterations} Latest loss: {latest_learning_loss:.4f}", fill=subtext, font=font)
if not points:
draw.text((32, 112), "No loss history is available yet.", fill=(255, 160, 122), font=font)
return _to_image_tensor(image.convert("RGB"))
left = 70
top = 92
right = width - 28
bottom = height - 58
graph_width = max(1, right - left)
graph_height = max(1, bottom - top)
draw.line((left, top, left, bottom), fill=axis, width=1)
draw.line((left, bottom, right, bottom), fill=axis, width=1)
y_values = [_safe_float(point.get("learning_loss"), 0.0) for point in points]
x_values = [_safe_int(point.get("total_iteration"), index + 1) for index, point in enumerate(points)]
y_min = min(y_values)
y_max = max(y_values)
if abs(y_max - y_min) < 1e-9:
pad = 0.25 if y_max == 0 else abs(y_max) * 0.15
y_min -= pad
y_max += pad
else:
pad = max(0.02, (y_max - y_min) * 0.12)
y_min -= pad
y_max += pad
x_min = min(x_values)
x_max = max(x_values)
if x_min == x_max:
x_max = x_min + 1
for i in range(5):
y = top + (graph_height * i / 4.0)
draw.line((left, y, right, y), fill=grid, width=1)
y_label = y_max - ((y_max - y_min) * i / 4.0)
draw.text((18, y - 6), f"{y_label:.3f}", fill=subtext, font=font)
for i in range(5):
x = left + (graph_width * i / 4.0)
draw.line((x, top, x, bottom), fill=grid, width=1)
x_label = round(x_min + ((x_max - x_min) * i / 4.0))
draw.text((x - 10, bottom + 10), str(x_label), fill=subtext, font=font)
coords = []
for x_value, y_value in zip(x_values, y_values):
norm_x = (x_value - x_min) / max(1e-9, (x_max - x_min))
norm_y = (y_value - y_min) / max(1e-9, (y_max - y_min))
px = left + norm_x * graph_width
py = bottom - norm_y * graph_height
coords.append((px, py))
if len(coords) == 1:
px, py = coords[0]
draw.ellipse((px - 4, py - 4, px + 4, py + 4), fill=point_color)
else:
polygon = [(coords[0][0], bottom)] + coords + [(coords[-1][0], bottom)]
draw.polygon(polygon, fill=fill)
draw.line(coords, fill=line, width=3)
px, py = coords[-1]
draw.ellipse((px - 5, py - 5, px + 5, py + 5), fill=point_color, outline=(255, 243, 214))
return _to_image_tensor(image.convert("RGB"))
def update_refinement_sampler_context(refinement_key, sampler_context):
"""Write sampler settings to a separate lightweight file (no conditioning data, safe to call during sampling)."""
if not refinement_key or not isinstance(sampler_context, dict):
return
path = refinement_state_path(refinement_key, "sampler_ctx", prefix="refine_v2")
os.makedirs(os.path.dirname(path), exist_ok=True)
try:
with open(path, "w", encoding="utf-8") as f:
json.dump(sampler_context, f)
except Exception as e:
print(f"[FunPack] update_refinement_sampler_context: failed to save context: {e}")
# Fixed, prompt-agnostic key for the Absolute steering store. Unlike every other refinement
# key (which hashes a specific prompt), this one collects liked/disliked directions across
# ALL prompts into a single "global taste" file. Relative mode = per-prompt; Absolute = this.
FUNPACK_ABSOLUTE_KEY = "__funpack_absolute_global__"
# --- Audio-safe conditioning steering -------------------------------------------------------
# LTXAV conditions video and audio from two separate text contexts that the model carves out of
# ONE conditioning tensor by splitting its channel dim: comfy/ldm/lightricks/av_model.py
# `_prepare_context` does `torch.split(context, [v_context_dim, a_context_dim], -1)` — leading
# channels drive video (self.attn2), trailing channels drive audio (self.audio_attn2). So any
# conditioning steering that shifts the WHOLE tensor also corrupts the audio's text cross-
# attention (observed live as degraded audio). Confining edits to the video channel-slice gives
# "original conditioning for audio, modified for video" with no model patching.
# LTXv2 fully-processed context = caption_channels*2 = 7680 → video = 3840
# LTXv2.3 fully-processed context = cross_attn_dim + audio_cross_attn_dim = 4096+2048 = 6144 → video = 4096
# Anything else (single-stream LTXV, unrecognised dim) returns the full width, so callers no-op.
#
# The table is only a FALLBACK. From LTX 2.5 on, comfy reads the two projection widths off the
# text encoder's state dict (comfy/text_encoders/lt.py `sd_detect` -> video_projection_dim /
# audio_projection_dim) instead of assuming them, so a checkpoint is free to ship a pair this
# table has never seen. Guessing wrong here is silent and costly: an unrecognised width makes
# ltxav_video_channels return the FULL width, every steering edit then moves the audio text
# context too, and the only symptom is degraded audio. So whenever a live model is on hand we
# read its real widths and register them — see register_ltxav_split_from_model below.
_FUNPACK_AV_SPLIT_TABLE = {7680: 3840, 6144: 4096}
_FUNPACK_AV_SPLIT_LEARNED = {}
_FUNPACK_AV_LOGGED = set()
def register_ltxav_split_from_model(model):
"""Read the real video/audio text-context widths off a live LTX model and register the
resulting concat split, so ltxav_video_channels stops depending on the static table.
`cross_attention_dim` conditions video (self.attn2), `audio_cross_attention_dim` conditions
audio (self.audio_attn2), and the conditioning tensor is the two concatenated in that order
— the same split `_prepare_context` performs. Silent no-op for single-stream / non-LTX
models, which have no audio width to find."""
try:
dm = model.model.diffusion_model
except Exception:
return None
try:
v = int(getattr(dm, "cross_attention_dim", 0) or 0)
a = int(getattr(dm, "audio_cross_attention_dim", 0) or 0)
except Exception:
return None
if v <= 0 or a <= 0:
return None # single-stream (video-only) LTXV: nothing to protect
total = v + a
if _FUNPACK_AV_SPLIT_LEARNED.get(total) == v:
return v
_FUNPACK_AV_SPLIT_LEARNED[total] = v
_FUNPACK_AV_LOGGED.discard(total) # re-log, now that the split is measured rather than guessed
if _FUNPACK_AV_SPLIT_TABLE.get(total, total) != v:
print(f"[FunPack AV] model reports video={v} + audio={a} = {total}; this does NOT match "
f"the built-in table — using the model's own widths (the table would have "
f"{'split at ' + str(_FUNPACK_AV_SPLIT_TABLE[total]) if total in _FUNPACK_AV_SPLIT_TABLE else 'left audio unprotected'}).")
return v
def ltxav_video_channels(dim):
"""Leading channel count that conditions VIDEO for an LTXAV concat-context of width `dim`.
Returns `dim` (no audio half) for single-stream / unrecognised layouts."""
dim = int(dim)
v = _FUNPACK_AV_SPLIT_LEARNED.get(dim) or _FUNPACK_AV_SPLIT_TABLE.get(dim, dim)
if dim not in _FUNPACK_AV_LOGGED:
_FUNPACK_AV_LOGGED.add(dim)
if v < dim:
src = "measured off the model" if dim in _FUNPACK_AV_SPLIT_LEARNED else "known layout"
print(f"[FunPack AV] conditioning dim={dim} ({src}): steering confined to video [:{v}], audio [{v}:] protected")
else:
print(f"[FunPack AV] conditioning dim={dim}: single-stream (no audio half); steering applies to all channels")
return v
def protect_audio_channels(steered, original):
"""Restore the audio channel-slice of a steered conditioning tensor from `original`, so the
steering only moves the video stream. No-op when shapes mismatch or there is no audio half."""
try:
if not isinstance(steered, torch.Tensor) or not isinstance(original, torch.Tensor):
return steered
if tuple(steered.shape) != tuple(original.shape):
return steered
v = ltxav_video_channels(steered.shape[-1])
if v >= steered.shape[-1]:
return steered
out = steered.clone()
out[..., v:] = original[..., v:].to(device=out.device, dtype=out.dtype)
return out
except Exception as _e:
_log.failed("FunPackStudio", "audio channel protection", _e,
"the conditioning edit reached the AUDIO channels too")
return steered
def scale_conditioning_spread(cond, factor):
"""Scale the conditioning's per-token spread (its 'sigma') on VIDEO channels only by `factor`,
mean-preserving per token; audio channels are left untouched (LTXAV channel split). factor>1
amplifies the prompt's expression (toward overbake), <1 dampens it. Used by Interactive
Guessing to build a strength ladder, and by the learned safe-spread cap."""
try:
if not isinstance(cond, torch.Tensor) or abs(float(factor) - 1.0) < 1e-6:
return cond
v = ltxav_video_channels(cond.shape[-1])
out = cond.clone()
vid = out[..., :v]
m = vid.mean(dim=-1, keepdim=True)
out[..., :v] = m + (vid - m) * float(factor)
return out
except Exception as _e:
_log.failed("FunPackStudio", "conditioning spread scaling", _e,
"the conditioning keeps its original spread")
return cond
def conditioning_spread(cond):
"""Robust per-token video-channel spread (mean std over tokens) of a conditioning tensor —
the scalar Interactive Guessing's ladder and cap reason about. None if not measurable."""
try:
if not isinstance(cond, torch.Tensor):
return None
v = ltxav_video_channels(cond.shape[-1])
vid = cond[..., :v].float()
m = vid.mean(dim=-1, keepdim=True)
return float(((vid - m) ** 2).mean().clamp_min(1e-12).sqrt().item())
except Exception:
return None
def _readable_refinement_key(refinement_key):
"""A human-readable, filesystem-safe rendering of a refinement key, or None
when the key can't be represented without ambiguity (then callers fall back to
the legacy hash so two distinct keys never collide on one file). Keys in real
use are short names ("default", "123"); the hash was only ever defensive."""
key = str(refinement_key or "").strip()
if not key or len(key) > 120 or key in {".", ".."}:
return None
# Conservative whitelist: anything outside it (slashes, colons, newlines,
# unicode) would change meaning or break a path, so hash those instead.
if not re.fullmatch(r"[A-Za-z0-9 _.\-]+", key):
return None
return key
def refinement_state_path(refinement_key, mode, prefix="refine", extension="json"):
"""Path to a per-key state file. The clip state — the actual "refinement key"
users load, share and drop into the folder — is stored readably as
``<key>.json``; sidecars (value function, sampler context, latents) get a
``<key>.<mode>.<ext>`` suffix so they group with it. Unsafe/over-long keys (which
can't be a filename) keep a ``<prefix>_<md5>.ext`` hash."""
base_dir = os.path.dirname(os.path.abspath(__file__))
refinements_dir = os.path.join(base_dir, "refinements")
mode_l = (mode or "ltx2").lower()
safe = _readable_refinement_key(refinement_key)
if safe is None:
digest = md5(f"{mode_l}::{refinement_key}".encode("utf-8")).hexdigest()
return os.path.join(refinements_dir, f"{prefix}_{digest}.{extension}")
if mode_l == "clip":
return os.path.join(refinements_dir, f"{safe}.{extension}")
return os.path.join(refinements_dir, f"{safe}.{mode_l}.{extension}")
def clone_latent(latent):
"""Shallow-clone a LATENT dict, deep-copying its tensor values.
A NestedTensor (LTXAV / H3 packed audio+video) is NOT a torch.Tensor, so it is carried over
BY REFERENCE and the "clone" aliases the caller's samples. That is safe here, and an audit
should not re-open it: `latent_samples` returns None for a nested value, so
`latent_is_plain_video_tensor` is False and `_refine_latent` rejects the latent outright
(`_raise_wrong_latent`) before reaching this function. Every other caller returns the clone
unmodified — the two writes in `_refine_latent` REPLACE the "samples" key rather than
mutating it in place, so nothing can reach through the alias.
The sampler's own `_clone_latent` unbinds and clones each stream, because it works on
latents that ARE nested.
"""
if not isinstance(latent, dict):
return None
cloned = {}
for key, value in latent.items():
cloned[key] = value.detach().clone() if isinstance(value, torch.Tensor) else value
return cloned
def latent_samples(latent):
if not isinstance(latent, dict):
return None
samples = latent.get("samples")
return samples if isinstance(samples, torch.Tensor) else None
def latent_sample_type_name(latent):
if not isinstance(latent, dict):
return "missing"
samples = latent.get("samples")
if samples is None:
return "missing samples"
shape = tuple(samples.shape) if isinstance(samples, torch.Tensor) else "unknown"
return f"{type(samples).__module__}.{type(samples).__name__}, shape={shape}"
def latent_is_plain_video_tensor(latent):
samples = latent_samples(latent)
if samples is None:
return False
if samples.dim() == 5:
return True
if samples.dim() == 4 and latent.get("type") != "audio":
return True
return False
def cpu_tensor_bundle(latent):
if not isinstance(latent, dict):
return {}
bundle = {}
for key, value in latent.items():
if isinstance(value, torch.Tensor):
bundle[key] = value.detach().cpu().clone()
return bundle
def latent_from_tensor_bundle(bundle):
if not isinstance(bundle, dict):
return None
latent = {}
for key, value in bundle.items():
if key == "_meta":
continue
latent[key] = value.detach().clone() if isinstance(value, torch.Tensor) else value
return latent if latent_samples(latent) is not None else None
_GENERIC_SCENE_LABEL_PATTERN = r"\bscene\s+(?:[-+]?\d+|minus\s+[a-z][\w-]*)\b"
def split_prompt_by_transitions(prompt, placement="start"):
"""Module-level extraction of FunPackStudio._v2_split_prompt_by_transitions.
Returns [(text, visual_effect), ...] with >= 1 entry, same semantics as the method.
visual_effect is the effect that fires BEFORE this segment (None for segment 0).
"""
try:
from .templates import load_custom_transition_triggers
except ImportError:
from templates import load_custom_transition_triggers
text = str(prompt or "").strip()
if not text:
return [(text, None)]
custom_map = load_custom_transition_triggers()
custom_triggers = list(custom_map.keys())
if custom_triggers:
def _custom_trigger_pat(t):
end = r"\b" if re.search(r"\w$", t) else r"(?=\s|$)"
return r"\b" + re.escape(t) + end
custom_parts = "|".join(
_custom_trigger_pat(t) for t in sorted(custom_triggers, key=len, reverse=True)
)
split_pattern = re.compile(
r"(?:" + custom_parts + r"|" + _GENERIC_SCENE_LABEL_PATTERN + r")",
re.IGNORECASE,
)
else:
split_pattern = re.compile(_GENERIC_SCENE_LABEL_PATTERN, re.IGNORECASE)
capturing_pattern = re.compile(r"(" + split_pattern.pattern + r")", re.IGNORECASE)
parts = capturing_pattern.split(text)
segments = []
effects = []
first_segment = parts[0].strip().strip(",;.:").strip()
if not first_segment:
segments.append("")
effects.append(None)
current = first_segment
current_effect = None
i = 1
while i < len(parts):
trans_word = parts[i].strip()
following = parts[i + 1] if i + 1 < len(parts) else ""
following = following.strip().lstrip(",;.:").strip()
entry = custom_map.get(trans_word.lower()) or {}
trans_placement = entry.get("placement") or placement if isinstance(entry, dict) else placement
trans_visual_effect = entry.get("visual_effect", "none") if isinstance(entry, dict) else "none"
if trans_visual_effect == "none":
trans_visual_effect = None
if trans_placement == "end":
segments.append((current + " " + trans_word).strip().strip(",;.:").strip() if current else trans_word)
effects.append(current_effect)
current = following
current_effect = trans_visual_effect
elif trans_placement == "silent":
if current:
segments.append(current)
effects.append(current_effect)
current = following
current_effect = trans_visual_effect
else:
if current:
segments.append(current)
effects.append(current_effect)
current = re.sub(r"\b(a|an|the),\s*", r"\1 ", (trans_word + " " + following).strip().strip(",;.:").strip(), flags=re.IGNORECASE).strip()
current_effect = trans_visual_effect
i += 2
if current:
segments.append(current)
effects.append(current_effect)
def _real_content(seg):
s = split_pattern.sub("", seg).strip().strip(",;.: ")
return re.sub(r"^(a|an|the)\s*$", "", s, flags=re.IGNORECASE).strip()
i = 0
while i < len(segments):
if segments[i] == "":
i += 1
continue
if not _real_content(segments[i]) and i + 1 < len(segments):
connector = " " if re.search(r"\b(a|an|the)\s*$", segments[i], re.IGNORECASE) else ", "
segments[i + 1] = (segments[i] + connector + segments[i + 1]).strip()
segments.pop(i)
effects.pop(i)
else:
i += 1
if len(segments) > 1 and not _real_content(segments[-1]):
segments.pop()
effects.pop()
if len(segments) > 1:
return list(zip(segments, effects))
return [(text, None)]
def _expand_with_map(text):
"""Expand shortcuts EXACTLY like apply_prompt_shortcuts, but record a position map so
boundaries found in the expanded text can be projected back onto the original.
Returns (expanded, pieces) where each piece is
{exp_start, exp_end, orig_start, orig_end, is_shortcut}. Literal pieces map 1:1;
shortcut pieces map a (possibly different-length) replacement back to the original
trigger span.
"""
try:
from .templates import (load_shortcut_db, shortcut_list, shortcut_key,
_shortcut_replacements, _shortcut_trigger_pattern,
normalize_refinement_key, load_revolver,
revolver_next_replacement)
except ImportError:
from templates import (load_shortcut_db, shortcut_list, shortcut_key,
_shortcut_replacements, _shortcut_trigger_pattern,
normalize_refinement_key, load_revolver,
revolver_next_replacement)
import random
from hashlib import md5
whole = [{"exp_start": 0, "exp_end": len(text), "orig_start": 0,
"orig_end": len(text), "is_shortcut": False}]
try:
data = load_shortcut_db()
except Exception as _e:
_log.failed("FunPackShortcuts", "shortcut library load", _e,
"the prompt is used VERBATIM — no shortcut triggers were expanded")
return text, whole
candidates = []
for db_key, shortcut in (data.get("shortcuts", {}) or {}).items():
if not isinstance(shortcut, dict) or not bool(shortcut.get("enabled", True)):
continue
reps = _shortcut_replacements(shortcut.get("replacements", shortcut.get("replacement", [])))
if not reps:
continue
sc_key = normalize_refinement_key(shortcut.get("refinement_key", ""))
sc_name = str(shortcut.get("name", "") or "")
for trig in shortcut_list(shortcut.get("triggers", [])):
pat = _shortcut_trigger_pattern(trig)
if pat:
candidates.append((trig, pat, reps, sc_key, sc_name, str(db_key)))
if not candidates:
return text, whole
candidates.sort(key=lambda it: len(shortcut_key(it[0])), reverse=True)
combined = "|".join(f"(?P<t{i}>{pat})" for i, (_, pat, _, _, _, _) in enumerate(candidates))
if not combined:
return text, whole
rng = random.Random(int(md5(text.encode("utf-8")).hexdigest()[:12], 16))
# Shortcut revolver: peek the same stored cycle generation will draw from (never saved
# here) so split boundaries are found in the text generation will actually see.
try:
_revolver = load_revolver()
except Exception:
_revolver = {"enabled": False, "random": False, "state": {}}
_rev_on = bool(_revolver.get("enabled"))
_rev_state = _revolver.get("state") if isinstance(_revolver.get("state"), dict) else {}
_rev_random = bool(_revolver.get("random"))
try:
pattern = re.compile(combined, re.IGNORECASE | re.UNICODE)
except re.error as _e:
_log.failed("FunPackShortcuts", "shortcut trigger pattern", _e,
"the prompt is used VERBATIM — no shortcut triggers were expanded")
return text, whole
pieces, parts, exp_pos, last = [], [], 0, 0
for m in pattern.finditer(text):
if m.start() > last:
lit = text[last:m.start()]
pieces.append({"exp_start": exp_pos, "exp_end": exp_pos + len(lit),
"orig_start": last, "orig_end": m.start(), "is_shortcut": False})
parts.append(lit); exp_pos += len(lit)
rep = None
pc_key, pc_name = "", ""
for i, (_, _, reps, sc_key, sc_name, db_key) in enumerate(candidates):
if m.group(f"t{i}") is not None:
if _rev_on and len(reps) > 1:
rep = revolver_next_replacement(_rev_state, db_key, reps, _rev_random)
else:
rep = rng.choice(reps)
pc_key, pc_name = sc_key, sc_name; break
if rep is None:
rep = m.group(0)
pieces.append({"exp_start": exp_pos, "exp_end": exp_pos + len(rep),
"orig_start": m.start(), "orig_end": m.end(), "is_shortcut": True,
"refinement_key": pc_key, "name": pc_name})
parts.append(rep); exp_pos += len(rep)
last = m.end()
if last < len(text):
lit = text[last:]
pieces.append({"exp_start": exp_pos, "exp_end": exp_pos + len(lit),
"orig_start": last, "orig_end": len(text), "is_shortcut": False})
parts.append(lit)
return "".join(parts), (pieces or whole)
def _project_offset(pieces, exp_off, side):
"""Map an expanded-text offset back to the original. `side` is "before"/"after":
when the offset lands inside an expanded shortcut, the WHOLE original trigger token
moves to one side (before -> token leads the next chunk; after -> token trails)."""
for pc in pieces:
inside = pc["exp_start"] <= exp_off < pc["exp_end"]
at_end = exp_off == pc["exp_end"]
if inside or (at_end and pc is pieces[-1]):
if pc["is_shortcut"]:
return pc["orig_start"] if side == "before" else pc["orig_end"]
return pc["orig_start"] + (exp_off - pc["exp_start"])
return pieces[-1]["orig_end"] if pieces else exp_off
def split_timeline_verbatim(prompt):
"""Lossless split of the master prompt into anchor + scenes for the editor.
Correct order (all internal): expand shortcuts -> detect transitions + placements ->
split as Studio does (text before the first transition is the anchor) -> then COMPACT
the boundaries back onto the ORIGINAL text so each field shows verbatim chunks. No word
is expanded or dropped in the result: anchor + scene texts reproduce the prompt.
"""
text = str(prompt or "")
stripped = text.strip()
if not stripped:
return {"anchor": "", "scenes": [], "transitions": []}
def _single():
return {"anchor": "", "scenes": [{"index": 0, "text": stripped}], "transitions": []}
try:
from .templates import load_custom_transition_triggers
except ImportError:
from templates import load_custom_transition_triggers
custom_map = load_custom_transition_triggers()
expanded, pieces = _expand_with_map(text)
triggers = list(custom_map.keys())
if triggers:
def _trig_pat(t):
end = r"\b" if re.search(r"\w$", t) else r"(?=\s|$)"
return r"\b" + re.escape(t) + end
parts = "|".join(_trig_pat(t) for t in sorted(triggers, key=len, reverse=True))
split_pattern = re.compile(r"(?:" + parts + r"|" + _GENERIC_SCENE_LABEL_PATTERN + r")", re.IGNORECASE)
else:
split_pattern = re.compile(_GENERIC_SCENE_LABEL_PATTERN, re.IGNORECASE)
def _effect(group):
info = custom_map.get(re.sub(r"\s+", " ", group.strip().lower())) or {}
eff = info.get("visual_effect") or None
return (None if eff == "none" else eff), info.get("placement")
cuts = [] # (orig_offset, effect)
# (1) Scan the EXPANDED text: catches shortcut-driven splits, i.e. a shortcut whose
# trigger word is NOT a transition but whose EXPANSION resolves to one.
for m in split_pattern.finditer(expanded):
effect, placement = _effect(m.group(0))
# 'start' -> cut before the trigger (leads new scene); otherwise after it.
if placement == "start":
orig = _project_offset(pieces, m.start(), "before")
else:
orig = _project_offset(pieces, m.end(), "after")
cuts.append((orig, effect))
# (2) Scan the ORIGINAL text too: a transition trigger that is ALSO a shortcut (e.g.
# 'qcut'/'cut') gets expanded away in (1) before detection, so its split would be lost
# or misplaced. Matching the verbatim text catches it at its true offset (no projection
# needed). Non-shortcut triggers yield the same offset in both scans and dedupe to one.
for m in split_pattern.finditer(text):
effect, placement = _effect(m.group(0))
orig = m.start() if placement == "start" else m.end()
cuts.append((orig, effect))
# dedupe boundaries that map to the same original offset (e.g. several triggers inside
# one shortcut expansion, or the same trigger seen by both scans) and drop any at the start.
seen, uniq = set(), []
for off, eff in sorted(cuts, key=lambda c: c[0]):
if off <= 0 or off in seen:
continue
seen.add(off); uniq.append((off, eff))
if not uniq:
return _single()
positions = [0] + [c[0] for c in uniq] + [len(text)]
chunks = [text[positions[i]:positions[i + 1]].strip() for i in range(len(positions) - 1)]
scene_texts = chunks[1:]
effects = [c[1] for c in uniq] # effects[i] precedes scene_texts[i]
scenes = [{"index": i, "text": t} for i, t in enumerate(scene_texts)]
transitions = [
{"after_scene": i - 1, "visual_effect": effects[i]}
for i in range(1, len(scene_texts)) if effects[i]
]
return {"anchor": chunks[0], "scenes": scenes, "transitions": transitions}
def refinement_keys_in_text(text):
"""The set of non-default refinement keys whose shortcuts fire in `text`.
Pure read of one prompt chunk: expand its shortcuts and collect the refinement key bound
to each shortcut that fired. No splitting, no scene logic — the caller hands us one scene's
text and we say which keys belong to it."""
if not str(text or "").strip():
return set()
try:
_, pieces = _expand_with_map(str(text))
except Exception:
return set()
return {str(pc.get("refinement_key") or "") for pc in pieces
if pc.get("is_shortcut") and str(pc.get("refinement_key") or "")}
def split_scenes(prompt, placement="start"):
"""THE canonical scene split — one source of truth for editing, preview, key attribution,
session reset and generation.
Walks the ordered shortcut-expansion pieces (`_expand_with_map`) ONCE — no offset round-trip,
no re-scanning of expansion content. A scene boundary is created ONLY by a real transition
trigger: a shortcut whose trigger word is in the transition DB (e.g. `qcut`/`cut`), or a literal
transition word the user typed. The hard-coded generic `scene <N>` pattern is intentionally NOT
used — incidental "scene 2" text inside an expansion must never force a cut. Each non-transition
keyed shortcut contributes its refinement key to the CURRENT scene; the leading run (before the
first transition) is the anchor, whose keys fold into every scene.
Returns:
{
"anchor": <raw anchor text>, "anchor_expanded": <expanded anchor>,
"scenes": [ {"index": i,
"raw": <verbatim scene text>, # editing
"expanded": <expanded scene text, no anchor>, # generation encode
"keys": <anchor keys ∪ this scene's keys>,# attribution / reset
"effect": <transition effect before this scene | None>}, ... ],
}
A prompt with no transition triggers yields a single scene (the whole prompt). Because every
consumer reads this ONE split, scene counts and boundaries can never disagree."""
text = str(prompt or "")
try:
from .templates import load_custom_transition_triggers
except ImportError:
from templates import load_custom_transition_triggers
try:
custom_map = load_custom_transition_triggers() or {}
except Exception:
custom_map = {}
expanded_full, pieces = _expand_with_map(text)
segs = [{"raw": "", "exp": "", "keys": set(), "effect": None}]
def _norm(s):
return re.sub(r"\s+", " ", str(s or "").strip().lower())
def _open(effect):
eff = None if effect in (None, "none", "") else effect
segs.append({"raw": "", "exp": "", "keys": set(), "effect": eff})
def _transition(raw_t, exp_t, info):
place = (info.get("placement") or placement or "start")
effect = info.get("visual_effect")
if place == "end": # trigger tails the previous scene, then cut
segs[-1]["raw"] += raw_t; segs[-1]["exp"] += exp_t
_open(effect)
elif place == "silent": # trigger dropped entirely
_open(effect)
else: # "start": trigger leads the new scene
_open(effect)
segs[-1]["raw"] += raw_t; segs[-1]["exp"] += exp_t
# Literal-text trigger matcher: the user's custom triggers PLUS the generic `scene <N>` label.
# Scoped to LITERAL text only (this matcher never runs over shortcut expansions), so a "scene 2"
# the user types — or, crucially, the one the Movie Editor INJECTS into the generation prompt to
# force a split for a scene with no leading trigger (see build_combined_prompt for_generation) —
# still cuts, while a "scene 2" buried inside a shortcut's expansion does NOT.
def _tp(t):
end = r"\b" if re.search(r"\w$", t) else r"(?=\s|$)"
return r"\b" + re.escape(t) + end
_pats = [_tp(t) for t in sorted((t for t in custom_map.keys() if t), key=len, reverse=True)]
_pats.append(_GENERIC_SCENE_LABEL_PATTERN)
try:
lit_pat = re.compile("|".join(_pats), re.IGNORECASE)
except re.error:
lit_pat = None
for pc in pieces:
raw = text[pc.get("orig_start", 0):pc.get("orig_end", 0)]
exp = expanded_full[pc.get("exp_start", 0):pc.get("exp_end", 0)]
if pc.get("is_shortcut"):
# A shortcut is a transition when its TRIGGER word is in the DB (e.g. a literal
# "qcut" entry) OR when its EXPANSION resolves to a transition phrase (the common
# case: a "cut" shortcut expanding to the "Scene cut." split phrase). Match either,
# so an expansion-keyed split marker still cuts here exactly as it does in the
# verbatim editor split. Generic "scene <N>" is intentionally NOT matched on
# expansions — only user-defined custom triggers — so incidental labels never cut.
info = custom_map.get(_norm(raw))
if info is None:
info = custom_map.get(_norm(exp))
if info is not None: # transition-trigger shortcut
_transition(raw, exp, info)
else: # content shortcut: text + its key
segs[-1]["raw"] += raw; segs[-1]["exp"] += exp
key = str(pc.get("refinement_key") or "").strip()
if key:
segs[-1]["keys"].add(key)
elif lit_pat is not None: # literal text may hold typed triggers
last = 0
for m in lit_pat.finditer(raw):
segs[-1]["raw"] += raw[last:m.start()]; segs[-1]["exp"] += raw[last:m.start()]
info = custom_map.get(_norm(m.group(0)))
if info is None:
# Generic `scene <N>` label: a pure split DELIMITER (the Movie Editor injects
# one to force a boundary for a scene with no leading trigger; a user may also
# type it). It is NEVER content, so cut on it but DROP it ("silent") instead of
# letting it lead the new scene's text. Without this the literal "scene 1",
# "scene 2", … get encoded into each scene's conditioning (prompt pollution).
info = {"placement": "silent"}
_transition(m.group(0), m.group(0), info)
last = m.end()
segs[-1]["raw"] += raw[last:]; segs[-1]["exp"] += raw[last:]
else:
segs[-1]["raw"] += raw; segs[-1]["exp"] += exp
anchor = segs[0]
anchor_keys = set(anchor["keys"])
body = segs[1:]
if not body: # no transitions -> one scene
return {"anchor": "", "anchor_expanded": "",
"scenes": [{"index": 0, "raw": anchor["raw"].strip(),
"expanded": anchor["exp"].strip(), "keys": anchor_keys, "effect": None}]}
scenes = [{"index": i, "raw": s["raw"].strip(), "expanded": s["exp"].strip(),
"keys": anchor_keys | s["keys"], "effect": s["effect"]} for i, s in enumerate(body)]
return {"anchor": anchor["raw"].strip(), "anchor_expanded": anchor["exp"].strip(), "scenes": scenes}
def split_scenes_from_segments(anchor, segments, placement="start", postfix=""):
"""Canonical scene structure from EXPLICIT per-scene boundaries supplied by the caller (the
Movie Editor's own scene list) instead of re-deriving boundaries from in-text delimiters.
The editor already KNOWS where each scene starts (they are separate Scene objects), so it
hands us that list directly — no `scene N` marker ever has to be injected into the prompt.
`anchor` is the shared prefix text; `segments` is one raw text per editor scene (each may
carry its own leading transition trigger, which supplies that scene's effect). Each segment
becomes EXACTLY one scene; its shortcut expansion + refinement keys + effect come from
split_scenes (reused per item, where there is no boundary ambiguity), and the anchor keys
fold into every scene — identical semantics to split_scenes(), same output shape."""
def _flatten(split_out):
# Collapse a split_scenes() result to one scene: any leading anchor text + every
# sub-scene joined; first non-None effect wins; keys unioned.
raw_parts, exp_parts, keys, effect = [], [], set(), None
ar = (split_out.get("anchor") or "").strip()
ae = (split_out.get("anchor_expanded") or "").strip()
if ar:
raw_parts.append(ar)
if ae:
exp_parts.append(ae)
for s in (split_out.get("scenes") or []):
r = (s.get("raw") or "").strip()
e = (s.get("expanded") or "").strip()
if r:
raw_parts.append(r)
if e:
exp_parts.append(e)
keys |= set(s.get("keys") or set())
if effect is None and s.get("effect"):
effect = s.get("effect")
return " ".join(raw_parts).strip(), " ".join(exp_parts).strip(), keys, effect
a_raw, a_exp, anchor_keys, _ = _flatten(split_scenes(str(anchor or ""), placement=placement))
# Postfix mirrors the anchor: shortcut-expanded once, its keys fold into every scene, and
# its expanded text is appended to each scene downstream (see the structural fold in V2).
p_raw, p_exp, postfix_keys, _ = _flatten(split_scenes(str(postfix or ""), placement=placement))
shared_keys = anchor_keys | postfix_keys
scenes = []
for i, seg in enumerate(segments or []):
r, e, k, eff = _flatten(split_scenes(str(seg or ""), placement=placement))
scenes.append({"index": i, "raw": r, "expanded": e, "keys": shared_keys | k, "effect": eff})
return {"anchor": a_raw, "anchor_expanded": a_exp,
"postfix": p_raw, "postfix_expanded": p_exp, "scenes": scenes}
def refinement_key_pool_for(prompt):
"""Every non-default refinement key the prompt activates (anchor + all scenes) — the exact
set a session reset wipes. Derived from the one canonical split."""
pool = set()
for scene in split_scenes(prompt).get("scenes", []) or []:
pool |= scene.get("keys") or set()
return pool
def resolve_scene_refinement_keys(raw_positive):
"""Per-scene non-default refinement keys, read straight off the canonical scene split.
Returns a list of sets aligned 1:1 with `split_scenes(raw_positive)["scenes"]`; an empty set
means that scene steers/trains with the project DEFAULT key. There is no scene-count parameter
and no divergence fallback: every consumer reads the ONE canonical split, so attribution can't
disagree with the preview or with generation. Anchor keys are already folded into each scene by
`split_scenes`."""
try:
return [scene.get("keys") or set()
for scene in split_scenes(str(raw_positive or "")).get("scenes", []) or []]
except Exception as error:
print(f"[FunPackStudio] Scene refinement-key attribution failed: {error}")
return []
def parse_timeline_segments(prompt):
"""Return {"anchor": str, "scenes": [...], "transitions": [...]} for timeline preview.
Mirrors _v2_transition_scene_texts: segments[0] is the character anchor prepended
to every scene, segments[1:] are the actual generated scenes.
If no transitions found, the single segment is treated as scene 1 with no anchor.
"""
segments = split_prompt_by_transitions(prompt, placement="silent")
if len(segments) == 1:
return {
"anchor": "",
"scenes": [{"index": 0, "text": segments[0][0]}],
"transitions": [],
}
anchor = segments[0][0].strip()
scenes = [{"index": i, "text": text.strip()} for i, (text, _) in enumerate(segments[1:])]
transitions = [
{"after_scene": i - 1, "visual_effect": effect}
for i, (_, effect) in enumerate(segments[1:])
if i > 0
]
return {"anchor": anchor, "scenes": scenes, "transitions": transitions}
class _ClipTokenizerAdapter:
"""Presents a live Comfy tokenizer through the slice of the HuggingFace API the refinement
code uses: call it for `input_ids`, decode ids back to text.
Comfy's inner tokenizers take text and nothing else, so truncation and the leading start
token are handled here. Dropping the start token matters: concept spans are found by
matching a phrase's ids inside the whole prompt's ids, and a BOS on the phrase never
matches."""
def __init__(self, sd_tokenizer):
self._t = sd_tokenizer.tokenizer
self._start = getattr(sd_tokenizer, "start_token", None)
self.name_or_path = str(getattr(sd_tokenizer, "embedding_key", "") or "CLIP")
def __call__(self, text, add_special_tokens=True, truncation=False, max_length=None, **_):
ids = list((self._t(text) or {}).get("input_ids") or [])
if not add_special_tokens and self._start is not None:
while ids and ids[0] == self._start:
ids.pop(0)
if truncation and max_length:
ids = ids[: int(max_length)]
return {"input_ids": ids}
def encode(self, text, **kwargs):
return self(text, **kwargs)["input_ids"]
def decode(self, token_ids, skip_special_tokens=True):
try:
return self._t.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
except TypeError:
return self._t.decode(list(token_ids))
class FunPackVideoRefiner:
LATENT_OUTPUT_INDEX = 6
NO_LATENT_REFERENCE_ERROR = "No available latent to operate. Please connect reference latent to input of Video Refiner."
WRONG_LATENT_ERROR = (
"Video Refiner latent input must be a plain video LATENT with tensor samples. "
"Audio latents and LTX audio/video combined NestedTensor latents are not supported here. "
"Connect only the video latent to Video Refiner, then feed the refined video latent into "
"LTXVConcatAVLatent as video_latent."
)
SAVED_LATENT_ONLY_STATUS = (
"Running refinement on saved latent. Changing reference latent shape and size will cause no effect to generation."
)
_tokenizers = {}
_tokenizer_sources = {
"ltx2": [
("DreamFast/gemma-3-12b-it-heretic-v2", {
"trust_remote_code": True,
"use_fast": True,
}),
],
# MiniMax H3 encodes with Qwen3-VL, not Gemma. Falling through to the ltx2 entry
# downloaded a 12B Gemma tokenizer for a Qwen model: every token span it produced was
# measured with the wrong vocabulary, and it fetched ~50 MB over the network to do it.
"minimax_h3": [
("Qwen/Qwen3-VL-32B-Instruct", {"use_fast": True}),
("Qwen/Qwen3-32B", {"use_fast": True}),
],
"wan": [
("Wan-AI/Wan2.2-T2V-A14B", {
"subfolder": "google/umt5-xxl",
"use_fast": True,
}),
("Wan-AI/Wan2.2-I2V-A14B", {
"subfolder": "google/umt5-xxl",
"use_fast": True,
}),
("Wan-AI/Wan2.2-Animate-14B", {
"subfolder": "google/umt5-xxl",
"use_fast": True,
}),
("Wan-AI/Wan2.1-T2V-1.3B", {
"subfolder": "google/umt5-xxl",
"use_fast": True,
}),
("Runware/Wan2.2-TI2V-5B", {
"subfolder": "google/umt5-xxl",
"use_fast": True,
}),
("ddwf/Wan2.2-Animate-14B", {
"subfolder": "google/umt5-xxl",
"use_fast": True,
}),
],
}
@classmethod
def _get_tokenizer_sources(cls, mode="ltx2"):
mode = (mode or "ltx2").lower()
sources = list(cls._tokenizer_sources.get(mode, cls._tokenizer_sources["ltx2"]))
if mode == "wan":
local_sources = []
models_dir = getattr(folder_paths, "models_dir", None)
if models_dir:
local_sources.extend(
(path, {})
for path in glob.glob(os.path.join(models_dir, "Wan", "*", "google", "umt5-xxl"))
if os.path.isdir(path)
)
text_encoder_tokenizer = os.path.join(models_dir, "text_encoders", "google", "umt5-xxl")
if os.path.isdir(text_encoder_tokenizer):
local_sources.append((text_encoder_tokenizer, {}))
if local_sources:
sources = local_sources + sources
return sources
@staticmethod
def _clip_text_tokenizer(clip):
"""The tokenizer that actually produced the conditioning, or None.
`clip.tokenizer` is a Comfy SD1Tokenizer holding one named sub-tokenizer — `gemma3_12b`
on LTX-2.3, `gemma4` on LTX-2.5, `umt5_xxl` on WAN. The name is on `.clip`."""
top = getattr(clip, "tokenizer", None)
name = getattr(top, "clip", None) if top is not None else None
sub = getattr(top, name, None) if isinstance(name, str) else None
return sub if getattr(sub, "tokenizer", None) is not None else None
@staticmethod
def _encoder_family_changed(stored, current):
"""Whether learned state from `stored` may still be applied under `current`.
Either side being empty means unknown — an older session file, or a run with no CLIP
connected — and unknown must never trigger a reset, or every conditioning-only run
would wipe the key it was meant to use."""
return bool(current and stored and stored != current)
@classmethod
def _encoder_family(cls, clip):
"""Short name of the text encoder behind `clip` ("gemma3_12b", "gemma4", ...), or ""."""
sub = cls._clip_text_tokenizer(clip)
return str(getattr(sub, "embedding_key", "") or "") if sub is not None else ""
@classmethod
def _get_tokenizer(cls, mode="ltx2", clip=None):
"""Prefer the CLIP's own tokenizer. A downloaded one has to guess the encoder family,
and guessing wrong is silent: the spans still resolve, they just point at the wrong
positions. LTX-2.3 encodes with Gemma3 and LTX-2.5 with Gemma4."""
if clip is not None:
sub = cls._clip_text_tokenizer(clip)
if sub is not None:
try:
return _ClipTokenizerAdapter(sub)
except Exception as e:
print(f"[FunPackVideoRefiner] CLIP tokenizer unusable, falling back to download: {e}")
mode = (mode or "ltx2").lower()
cached = cls._tokenizers.get(mode)
if cached is not None:
return cls._tokenizers[mode]
sources = cls._get_tokenizer_sources(mode)
# Local cache FIRST, every source, before any network call. `from_pretrained` reaches
# HuggingFace with no token, no timeout and no progress, on ComfyUI's execution
# thread — mid-run that is an unbounded stall with nothing in the log to explain it,
# and on a rate-limited unauthenticated request it can be a long one. A tokenizer
# that is already on disk should never pay for that.
for local_only in (True, False):
for model_id, kwargs in sources:
if not local_only:
print(f"[FunPackVideoRefiner] tokenizer for '{mode}' is not in the local "
f"HuggingFace cache — DOWNLOADING '{model_id}'. This blocks the run "
f"until it finishes; set HF_TOKEN to avoid unauthenticated rate limits.")
try:
cls._tokenizers[mode] = AutoTokenizer.from_pretrained(
model_id, **dict(kwargs, local_files_only=local_only))
return cls._tokenizers[mode]
except Exception as e:
if not local_only:
print(f"[FunPackVideoRefiner] Tokenizer load failed for mode '{mode}' "
f"from '{model_id}': {e}")
return None
def _normalize_prompt_for_mode(self, prompt: str, mode: str) -> str:
prompt = (prompt or "").strip()
if mode == "wan":
return re.sub(r"\s+", " ", prompt)
return prompt
def _prompt_looks_like_refusal(self, prompt: str) -> bool:
text = re.sub(r"\s+", " ", str(prompt or "").strip().lower())
if not text:
return False
text = text.replace("’", "'").replace("`", "'")
prefix = text[:520]
refusal_patterns = (
r"^(?:i'?m|i am)?\s*sorry\b.{0,120}\b(?:can(?:not|'?t)|unable|not able|won'?t)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate|do that|with this request)\b",
r"^(?:i|we)\s+(?:can(?:not|'?t)|won'?t|am unable|are unable|am not able|are not able)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate|do that|with this request)\b",
r"^as an ai\b.{0,160}\b(?:can(?:not|'?t)|unable|not able|won'?t)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate)\b",
r"^(?:i|we)\s+(?:must|have to|need to)\s+(?:decline|refuse)\b",
)
return any(re.search(pattern, prefix) for pattern in refusal_patterns)
def _v2_run_looks_like_refusal(self, run):
if not isinstance(run, dict):
return False
return (
self._prompt_looks_like_refusal(run.get("prompt", "")) or
self._prompt_looks_like_refusal(run.get("encoded_prompt", ""))
)
def _get_conditioning_seq_len(self, conditioning: torch.Tensor) -> int:
if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1:
return 0
return conditioning.shape[1] if conditioning.dim() == 3 else conditioning.shape[0]
def _get_conditioning_token_mask(self, conditioning: torch.Tensor):
if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1:
return None
if conditioning.dim() == 3:
token_energy = conditioning.detach().abs().sum(dim=-1)
mask = token_energy.gt(1e-12).any(dim=0)
else:
token_energy = conditioning.detach().abs().sum(dim=-1)
mask = token_energy.gt(1e-12)
if not bool(mask.any()):
seq_len = self._get_conditioning_seq_len(conditioning)
return torch.ones(seq_len, dtype=torch.bool, device=conditioning.device)
return mask
def _get_effective_seq_len(self, token_mask, fallback_seq_len: int) -> int:
if token_mask is None:
return fallback_seq_len
active_positions = torch.nonzero(token_mask, as_tuple=False).flatten()
if active_positions.numel() == 0:
return fallback_seq_len
return min(fallback_seq_len, int(active_positions[-1].item()) + 1)
def _mask_to_embedding_dims(self, token_mask, reference: torch.Tensor):
if token_mask is None or not isinstance(reference, torch.Tensor) or reference.dim() <= 1:
return None
token_mask = token_mask.to(device=reference.device, dtype=reference.dtype)
if reference.dim() == 3:
return token_mask.view(1, -1, 1)
return token_mask.view(-1, 1)
def _masked_sequence_mean(self, conditioning: torch.Tensor, token_mask):
if not isinstance(conditioning, torch.Tensor):
return conditioning
if conditioning.dim() <= 1 or token_mask is None:
return conditioning
mask = self._mask_to_embedding_dims(token_mask, conditioning)
if conditioning.dim() == 3:
denom = mask.sum(dim=1).clamp_min(1.0)
return (conditioning * mask).sum(dim=1) / denom
denom = mask.sum(dim=0).clamp_min(1.0)
return (conditioning * mask).sum(dim=0) / denom
def _tokenize_ids(self, tokenizer, text: str, add_special_tokens: bool, max_length: Optional[int] = None):
if not tokenizer or not text:
return []
kwargs = {"add_special_tokens": add_special_tokens}
if max_length is not None and max_length > 0:
kwargs["truncation"] = True
kwargs["max_length"] = max_length
try:
tokenized = tokenizer(text, **kwargs)
input_ids = tokenized.get("input_ids", tokenized)
if isinstance(input_ids, list) and input_ids and isinstance(input_ids[0], list):
input_ids = input_ids[0]
return input_ids if isinstance(input_ids, list) else list(input_ids)
except Exception:
try:
return tokenizer.encode(text, **kwargs)
except Exception:
return []
def _iter_prompt_segments(self, prompt: str):
if not prompt:
return
last_end = 0
for match in PROTECTED_PHRASE_RE.finditer(prompt):
start, end = match.span()
if start > last_end:
yield ("text", prompt[last_end:start])
protected_text = (match.group(1) or match.group(2) or match.group(3) or "").strip()
if protected_text:
yield ("protected", protected_text)
last_end = end
if last_end < len(prompt):
yield ("text", prompt[last_end:])
def _mask_quoted_text(self, prompt: str):
if not prompt:
return ""
return PROTECTED_PHRASE_RE.sub(lambda match: " " * len(match.group(0)), prompt)
def _fallback_word_groups_from_prompt(self, prompt: str, seq_len: int, token_mask=None, existing_groups=None):
if not prompt or seq_len <= 0:
return []
effective_seq_len = self._get_effective_seq_len(token_mask, seq_len)
if effective_seq_len <= 0:
return []
active_positions = list(range(effective_seq_len))
if token_mask is not None:
active_positions = [
index
for index, enabled in enumerate(token_mask[:effective_seq_len].detach().cpu().tolist())
if enabled
]
if not active_positions:
return []
existing = {
str(group[2]).strip().lower()
for group in (existing_groups or [])
if isinstance(group, (list, tuple)) and len(group) >= 3
}
candidates = []
seen = set(existing)
for segment_type, segment_text in self._iter_prompt_segments(prompt):
if segment_type == "protected":
pieces = [segment_text.strip()]
else:
pieces = re.findall(r"[\w'’.-]+", segment_text, flags=re.UNICODE)
for piece in pieces:
clean = piece.strip(" \t\r\n,;:!?()[]{}\"'")
lower = clean.lower()
if lower in seen or not self._is_valuable_token(clean):
continue
seen.add(lower)
candidates.append(clean)
if not candidates:
return []
fallback = []
step_count = len(candidates)
max_slots = len(active_positions)
for index, word in enumerate(candidates[:max_slots]):
if step_count == 1:
pos_index = len(active_positions) // 2
else:
pos_index = round(index * (len(active_positions) - 1) / max(1, min(step_count, max_slots) - 1))
start = int(active_positions[max(0, min(len(active_positions) - 1, pos_index))])
fallback.append((start, min(effective_seq_len, start + 1), word, []))
return fallback
def _build_word_groups(self, prompt: str, tokenizer, seq_len: int, token_mask=None):
if not prompt or seq_len <= 0:
return []
if not tokenizer:
return self._fallback_word_groups_from_prompt(prompt, seq_len, token_mask)
effective_seq_len = self._get_effective_seq_len(token_mask, seq_len)
token_mask_list = None
if token_mask is not None:
token_mask_list = token_mask[:effective_seq_len].detach().cpu().tolist()
full_token_ids = self._tokenize_ids(
tokenizer,
prompt,
add_special_tokens=True,
max_length=effective_seq_len
)[:effective_seq_len]
word_groups = []
grouped_seen = set()
for segment_type, segment_text in self._iter_prompt_segments(prompt):
if segment_type != "protected":
continue
clean_phrase = segment_text.strip()
lower = clean_phrase.lower()
if lower in grouped_seen or len(clean_phrase) < 3 or not any(c.isalpha() for c in clean_phrase):
continue
grouped_seen.add(lower)
phrase_token_list = self._tokenize_ids(
tokenizer,
clean_phrase,
add_special_tokens=False
)
if not phrase_token_list:
continue
for start in range(max(0, len(full_token_ids) - len(phrase_token_list) + 1)):
end = start + len(phrase_token_list)
if token_mask_list is not None and not all(token_mask_list[start:min(effective_seq_len, end)]):
continue
if full_token_ids[start:end] == phrase_token_list and start < effective_seq_len:
word_groups.append((start, min(effective_seq_len, end), clean_phrase, phrase_token_list))
break
raw_words = [w.strip() for w in self._mask_quoted_text(prompt).split() if w.strip()]
for word in raw_words:
clean_word = word
lower = clean_word.lower()
if lower in grouped_seen or len(clean_word) < 3 or not self._is_valuable_token(clean_word):
continue
grouped_seen.add(lower)
word_token_list = self._tokenize_ids(
tokenizer,
clean_word,
add_special_tokens=False
)
if not word_token_list:
continue
found = False
for start in range(max(0, len(full_token_ids) - len(word_token_list) + 1)):
end = start + len(word_token_list)
if token_mask_list is not None and not all(token_mask_list[start:min(effective_seq_len, end)]):
continue
if full_token_ids[start:end] == word_token_list and start < effective_seq_len:
word_groups.append((start, min(effective_seq_len, end), clean_word, word_token_list))
found = True
break
if not found:
continue
if len(word_groups) < 2:
word_groups.extend(
self._fallback_word_groups_from_prompt(
prompt,
seq_len,
token_mask,
existing_groups=word_groups,
)
)
return sorted(
[group for group in word_groups if group[1] > group[0]],
key=lambda group: (group[0], group[1], group[2].lower()),
)
def _missing_axes_for_rating_key(self, rating_key):
for profile in RATING_PROFILES.values():
if profile.get("key") == rating_key:
return set(profile.get("missing_axes", []))
return set()
def _axis_counter_name(self, axis):
if axis == "concept":
return "missing_concept_count"
if axis == "details":
return "missing_detail_count"
if axis == "quality":
return "missing_quality_count"
return None
def _axis_memory_boost(self, missing_axes):
axes = set(missing_axes or [])
boost = 0.0
if "concept" in axes:
boost += 0.42
if "details" in axes:
boost += 0.30
if "quality" in axes:
boost += 0.24
if len(axes) >= 2:
boost += 0.16
return boost
def _apply_missing_memory_pressure(self, item, missing_axes, iter_num):
axes = set(missing_axes or [])
if not isinstance(item, dict) or not axes:
return item
item["wanted_count"] = max(0, int(item.get("wanted_count", 0)) + 1)
item["missing_count"] = max(0, int(item.get("missing_count", 0)) + 1)
for axis in axes:
counter = self._axis_counter_name(axis)
if counter:
item[counter] = max(0, int(item.get(counter, 0)) + 1)
item["last_missing_iter"] = int(iter_num)
return item
def _phrase_position_score(self, item):
likes = float(item.get("liked_count", 0))
neutral = float(item.get("neutral_count", 0))
awful = float(item.get("awful_count", 0))
wanted = float(item.get("wanted_count", 0))
missing = float(item.get("missing_count", 0))
return float(item.get("score", 0.0)) + likes * 0.45 + neutral * 0.06 + wanted * 0.26 + missing * 0.18 - awful * 0.85
def _rating_memory_deltas(self, rating_key):
if rating_key == "like":
return 0.70, 1, 0, 0
if rating_key == "missing_details":
return 0.56, 0, 1, 0
if rating_key == "missing_concept":
return 0.72, 0, 1, 0
if rating_key == "missing_quality":
return 0.48, 0, 1, 0
if rating_key == "missing_details_concept":
return 0.84, 0, 1, 0
if rating_key == "missing_details_quality":
return 0.68, 0, 1, 0
if rating_key == "missing_concept_quality":
return 0.80, 0, 1, 0
if rating_key == "awful":
return -1.10, 0, 0, 1
if rating_key == "discover":
return 0.0, 0, 1, 0
return 0.12, 0, 1, 0
def _history_modified_conditioning(self, history_entry, reference, device):
if not isinstance(history_entry, dict):
return None
mod_data = history_entry.get("modified_embeds")
if mod_data is None:
return None
try:
candidate = serializable_to_tensor(mod_data).to(device)
except Exception:
return None
if not isinstance(reference, torch.Tensor) or list(candidate.shape) != list(reference.shape):
return None
return candidate
def _history_modified_conditioning_any_shape(self, history_entry, device, dtype=None):
if not isinstance(history_entry, dict):
return None
mod_data = history_entry.get("modified_embeds")
if mod_data is None:
return None
try:
candidate = serializable_to_tensor(mod_data).to(device)
if dtype is not None:
candidate = candidate.to(dtype=dtype)
except Exception:
return None
if not isinstance(candidate, torch.Tensor) or candidate.dim() <= 1:
return None
return candidate
def _conditioning_canvas_compatible(self, candidate, template):
if (
not isinstance(candidate, torch.Tensor) or
not isinstance(template, torch.Tensor) or
candidate.dim() <= 1 or
template.dim() <= 1 or
candidate.dim() != template.dim() or
int(candidate.shape[-1]) != int(template.shape[-1])
):
return False
if candidate.dim() == 3 and int(candidate.shape[0]) != int(template.shape[0]):
return False
return self._get_conditioning_seq_len(candidate) > 0
def _resize_conditioning_sequence_like(self, source, target):
if not isinstance(source, torch.Tensor) or not isinstance(target, torch.Tensor):
return None
if source.dim() != target.dim() or source.dim() <= 1:
return None
if int(source.shape[-1]) != int(target.shape[-1]):
return None
if source.dim() == 3 and int(source.shape[0]) != int(target.shape[0]):
return None
if list(source.shape) == list(target.shape):
return source.to(device=target.device, dtype=target.dtype)
target_seq = self._get_conditioning_seq_len(target)
if target_seq <= 0:
return None
try:
working = source.to(device=target.device, dtype=target.dtype)
if working.dim() == 3:
resized = F.interpolate(
working.transpose(1, 2),
size=target_seq,
mode="linear",
align_corners=False,
).transpose(1, 2)
else:
resized = F.interpolate(
working.transpose(0, 1).unsqueeze(0),
size=target_seq,
mode="linear",
align_corners=False,
).squeeze(0).transpose(0, 1)
except Exception:
return None
if list(resized.shape) != list(target.shape):
return None
return resized
def _serialized_tensor_shape(self, payload):
if not isinstance(payload, dict):
return None
shape = payload.get("shape")
if not isinstance(shape, list) or not shape:
return None
try:
return tuple(int(dim) for dim in shape)
except (TypeError, ValueError):
return None
def _serialized_conditioning_compatible(self, payload, template):
shape = self._serialized_tensor_shape(payload)
if (
shape is None or
not isinstance(template, torch.Tensor) or
template.dim() <= 1 or
len(shape) != int(template.dim()) or
int(shape[-1]) != int(template.shape[-1])
):
return False
if len(shape) == 3 and int(shape[0]) != int(template.shape[0]):
return False
return (shape[1] if len(shape) == 3 else shape[0]) > 0
def _serialized_conditioning_seq_len(self, payload):
shape = self._serialized_tensor_shape(payload)
if shape is None or len(shape) <= 1:
return 0
return int(shape[1] if len(shape) == 3 else shape[0])
def _legacy_auto_inject_token_allowed(self, token):
token = str(token or "").strip().lower()
if not token:
return False
category = self._infer_concept_category([token])
if category in {"appearance", "subject", "environment", "character"}:
return False
return True
def _get_scheduler_factors(self, mode, rating, reward, similarity, iter_num, total_iters,
word_importance, word_groups, global_adaptive, device):
if mode == "original":
base_lr = 0.18 / (1 + 0.08 * (iter_num ** 0.5))
confidence = max(0.2, min(1.0, (rating - 3.0) / 5.0))
if similarity > 0.93:
confidence *= 0.4
return base_lr, confidence, 1.0, {}
confidence = max(0.15, min(1.0, (rating - 2.5) / 6.0))
if abs(reward) < 0.15 or similarity > 0.94:
confidence *= 0.35
prodigy_d = global_adaptive.setdefault("prodigy_d", {})
d_coef = 1.0 if mode == "accurate" else 1.8
adaptive_lr = global_adaptive.get("prodigy_lr_base", 1.0)
word_lr_mult = {}
for _, _, full_word, _ in word_groups:
wkey = full_word.lower()
if wkey not in word_importance:
continue
g = abs(reward) + 1e-8
if wkey not in prodigy_d:
prodigy_d[wkey] = g
else:
prodigy_d[wkey] = 0.9 * prodigy_d[wkey] + 0.1 * g
word_lr_mult[wkey] = adaptive_lr / (prodigy_d[wkey] ** 0.5 + 1e-8) * d_coef
current_step = global_adaptive.setdefault("current_step", 0)
current_step = min(current_step + 1, 500)
global_adaptive["current_step"] = current_step
warmup = global_adaptive.get("warmup_steps", 8)
progress = min(1.0, current_step / max(50, global_adaptive.get("total_steps_estimate", 150)))
if current_step < warmup:
lr_scale = current_step / max(1, warmup)
else:
lr_scale = 0.5 * (1.0 + math.cos(math.pi * (progress - warmup / (warmup + 50))))
if mode == "accurate":
lr_scale *= 0.75
confidence *= 0.9
exploration_mult = max(0.3, 1.0 - 0.6 * progress)
else:
lr_scale = min(2.2, lr_scale * 1.6)
confidence = min(1.0, confidence * 1.3)
exploration_mult = max(0.6, 1.3 - 0.7 * progress)
return lr_scale, confidence, exploration_mult, word_lr_mult
# =========================================================================
# TOKEN / WORD UTILITIES
# =========================================================================
def _get_top_tokens(self, token_dict, tokenizer, top_k=10):
if not token_dict or not tokenizer:
return "N/A"
sorted_tokens = sorted(token_dict.items(), key=lambda x: x[1], reverse=True)[:top_k]
top_list = []
for key, score in sorted_tokens:
try:
text = key if isinstance(key, str) else tokenizer.decode([int(key)], skip_special_tokens=True).strip()
if text:
top_list.append(f"{text}({score:.2f})")
except Exception:
top_list.append(f"{key}({score:.2f})")
return ", ".join(top_list) if top_list else "None"
def _is_valuable_token(self, token_text):
if not token_text:
return False
t = token_text.strip()
if '<' in t or '>' in t or len(t) < 3:
return False
t_lower = t.lower()
stopwords = {
"the", "a", "an", "and", "or", "but", "with", "for", "of", "in", "on", "at",
"to", "from", "by", "is", "are", "was", "were", "be", "been", "being", "have",
"has", "had", "do", "does", "did", "will", "would", "her", "his", "him", "she",
"he", "it", "they", "them", "this", "that", "these", "those", "i", "you", "my",
"your", "our", "their", "me", "us"
}
if t_lower in stopwords:
return False
if t in {",", ".", "!", "?", ":", ";", "-", "*", "(", ")", "[", "]", "{", "}", "'", "\"", "..."} or t.isdigit():
return False
if not any(c.isalpha() for c in t):
return False
return True
def _infer_concept_category(self, phrase_words: list):
words = set(phrase_words or [])
if not words:
return "general"
category_terms = {
"quality": {"masterpiece", "best", "quality", "detailed", "highres", "high-res", "ultra", "perfect"},
"style": {"anime", "cinematic", "photorealistic", "painterly", "illustration", "stylized", "realistic", "film", "noir"},
"camera": {"closeup", "close-up", "wide", "shot", "angle", "zoom", "pan", "tracking", "dolly", "camera", "focus", "bokeh", "framing"},
"action": {"running", "walking", "flying", "jumping", "smiling", "turning", "dancing", "moving", "motion", "looking", "holding", "standing", "sitting"},
"environment": {"forest", "city", "street", "room", "beach", "mountain", "temple", "sunset", "night", "rain", "snow", "sky", "background"},
"appearance": {"hair", "eyes", "dress", "jacket", "armor", "face", "skin", "beard", "smile", "pose", "outfit"},
"subject": {"girl", "boy", "woman", "man", "person", "character", "robot", "dragon", "cat", "dog", "bird", "child"},
}
best_category, best_score = "general", 0
for category, terms in category_terms.items():
score = len(words & terms)
if score > best_score:
best_category, best_score = category, score
return best_category
def _default_concept_cluster(self, phrase_words: list):
category = self._infer_concept_category(phrase_words)
return {
"label": " ".join((phrase_words or [])[:6]),
"anchor_words": list(phrase_words or []),
"category": category,
"category_source": "auto",
"category_confidence": 0.35 if category == "general" else 0.65,
"word_importance": {},
"presence_target": 1.0,
"priority_weight": 1.0,
"overrep_sensitivity": 1.0,
"stability_weight": 1.0,
"semantic_fidelity": 1.0,
"user_affinity": 1.0,
"question_history": [],
"last_question_type": None,
"last_question_iter": 0,
"usage_count": 0,
"last_seen_iter": 0,
}
def _ensure_concept_cluster_defaults(self, cluster: dict):
if not isinstance(cluster, dict):
return self._default_concept_cluster([])
anchor_words = list(cluster.get("anchor_words", []))
defaults = self._default_concept_cluster(anchor_words)
defaults.update(cluster)
defaults.setdefault("category_source", "auto")
defaults.setdefault("category_confidence", 0.35 if defaults.get("category") == "general" else 0.65)
if defaults.get("category_source") != "user" and (
not defaults.get("category") or defaults.get("category") == "general"
):
defaults["category"] = self._infer_concept_category(defaults.get("anchor_words", []))
defaults["category_confidence"] = 0.35 if defaults["category"] == "general" else max(0.65, defaults.get("category_confidence", 0.0))
defaults["question_history"] = list(defaults.get("question_history", []))[-24:]
return defaults
def _clip_profile_value(self, value, low=0.5, high=1.8):
return max(low, min(high, float(value)))
def _feedback_question_specs(self):
return {
"presence": {
"prompt": "How well is '{label}' represented in the output?",
"legend": "1=absent 2=weak 3=slightly weak 4=slightly strong 5=perfect 6=overrepresented",
},
"priority": {
"prompt": "How important should '{label}' be relative to the other concepts?",
"legend": "1=much less important 2=less important 3=slightly less 4=slightly more 5=important 6=top priority",
},
"balance": {
"prompt": "How balanced is '{label}' compared with nearby concepts?",
"legend": "1=far too weak 2=too weak 3=slightly weak 4=slightly strong 5=balanced 6=overpowering",
},
"fidelity": {
"prompt": "How accurately does '{label}' match what you meant?",
"legend": "1=wrong 2=mostly wrong 3=slightly off 4=close 5=correct 6=too literal",
},
"stability": {
"prompt": "How stable should '{label}' be across future outputs?",
"legend": "1=very unstable 2=unstable 3=slightly unstable 4=slightly stable 5=stable 6=very rigid",
},
"preference": {
"prompt": "What is your preference for having '{label}' in future outputs?",
"legend": "1=strongly less 2=less 3=slightly less 4=slightly more 5=more 6=much more",
},
"category": {
"prompt": "What kind of concept is '{label}'?",
"legend": "1=general 2=concept 3=style 4=quality 5=character 6=details",
},
}
def _get_concept_mean_importance(self, cluster: dict):
local_imp = cluster.get("word_importance", {})
if not local_imp:
return 1.0
return sum(local_imp.values()) / max(1, len(local_imp))
def _get_question_base_weight(self, question_type: str, category: str):
weights = {
"presence": {"subject": 1.15, "appearance": 1.12, "action": 1.10, "environment": 1.08, "style": 0.95, "camera": 0.92, "quality": 0.90, "general": 1.0},
"priority": {"style": 1.15, "camera": 1.12, "quality": 1.08, "environment": 1.02, "subject": 0.98, "appearance": 0.98, "action": 1.0, "general": 1.0},
"balance": {"style": 1.18, "camera": 1.10, "quality": 1.08, "environment": 1.02, "subject": 0.98, "appearance": 1.0, "action": 1.0, "general": 1.0},
"fidelity": {"subject": 1.15, "action": 1.12, "environment": 1.08, "appearance": 1.05, "style": 0.95, "camera": 0.92, "quality": 0.90, "general": 1.0},
"stability": {"action": 1.10, "camera": 1.10, "style": 1.06, "subject": 1.04, "appearance": 1.0, "environment": 1.0, "quality": 0.95, "general": 1.0},
"preference": {"style": 1.14, "environment": 1.10, "appearance": 1.05, "subject": 1.03, "camera": 1.02, "action": 1.0, "quality": 0.95, "general": 1.0},
}
return weights.get(question_type, {}).get(category, 1.0)
def _score_question_type(self, question_type: str, cluster: dict, category: str,
mean_imp: float, neighbor_mean: float, rating_shift: float,
similarity: float, iter_num: int):
uncertainty = max(0.0, 1.0 - min(1.0, abs(mean_imp - cluster.get("presence_target", 1.0))))
dominance = max(0.0, mean_imp - 1.15)
neighbor_conflict = abs(mean_imp - neighbor_mean)
freshness = 1.0
if cluster.get("last_question_type") == question_type:
freshness -= 0.35
if iter_num - int(cluster.get("last_question_iter", 0)) < 3:
freshness -= 0.20
freshness = max(0.25, freshness)
if question_type == "presence":
score = 0.50 * uncertainty + 0.22 * abs(cluster.get("presence_target", 1.0) - mean_imp) + 0.16 * rating_shift + 0.12 * max(0.0, 1.0 - similarity)
elif question_type == "priority":
score = 0.40 * rating_shift + 0.28 * abs(cluster.get("priority_weight", 1.0) - 1.0) + 0.18 * dominance + 0.14 * uncertainty
elif question_type == "balance":
score = 0.44 * dominance + 0.28 * neighbor_conflict + 0.16 * abs(cluster.get("overrep_sensitivity", 1.0) - 1.0) + 0.12 * rating_shift
elif question_type == "fidelity":
score = 0.38 * (2.0 - cluster.get("semantic_fidelity", 1.0)) + 0.24 * rating_shift + 0.20 * uncertainty + 0.18 * max(0.0, 1.0 - similarity)
elif question_type == "stability":
score = 0.35 * rating_shift + 0.30 * abs(cluster.get("stability_weight", 1.0) - 1.0) + 0.20 * max(0.0, 1.0 - similarity) + 0.15 * uncertainty
elif question_type == "category":
category_uncertainty = max(0.0, 1.0 - float(cluster.get("category_confidence", 0.35)))
needs_category_discovery = (
cluster.get("category_source") != "user" and
(category == "general" or category_uncertainty >= 0.50)
)
if not needs_category_discovery:
return 0.0
score = 0.65 + 0.25 * category_uncertainty + 0.10 * rating_shift
else: # preference
score = 0.34 * abs(cluster.get("user_affinity", 1.0) - 1.0) + 0.28 * rating_shift + 0.20 * uncertainty + 0.18 * abs(cluster.get("priority_weight", 1.0) - 1.0)
return score * self._get_question_base_weight(question_type, category) * freshness
def _select_feedback_question(self, ordered_concept_ids: list, concept_clusters: dict,
concept_groups: dict, current_concept_labels: dict,
last_rating: int, rating: int,
similarity: float, iter_num: int):
if not ordered_concept_ids:
return None
question_specs = self._feedback_question_specs()
rating_shift = min(1.0, abs(rating - last_rating) / 4.0)
candidates = []
category_candidates = []
for cid in ordered_concept_ids:
cluster = concept_clusters.get(cid)
if not cluster:
continue
cluster = self._ensure_concept_cluster_defaults(cluster)
concept_clusters[cid] = cluster
category = cluster.get("category", "general")
mean_imp = self._get_concept_mean_importance(cluster)
neighbor_ids = self._get_concept_neighbors(cid, ordered_concept_ids, radius=2)
neighbor_means = [
self._get_concept_mean_importance(concept_clusters[nid])
for nid in neighbor_ids if nid in concept_clusters
]
neighbor_mean = sum(neighbor_means) / len(neighbor_means) if neighbor_means else 1.0
chosen_group_id = None
for gid, group in concept_groups.items():
if cid in group.get("concept_ids", []):
chosen_group_id = gid
break
for question_type in question_specs.keys():
score = self._score_question_type(
question_type, cluster, category, mean_imp,
neighbor_mean, rating_shift, similarity, iter_num
)
candidate = {
"concept_id": cid,
"concept_label": current_concept_labels.get(cid, cluster.get("label", "")),
"question_type": question_type,
"neighbor_ids": neighbor_ids,
"group_id": chosen_group_id,
"category": category,
"score": score,
}
if question_type == "category":
category_candidates.append(candidate)
else:
candidates.append(candidate)
if category_candidates:
category_candidate = max(category_candidates, key=lambda x: x["score"])
if category_candidate["score"] >= 0.60:
return category_candidate
if not candidates:
return None
return max(candidates, key=lambda x: x["score"])
def _format_feedback_question(self, concept_label: str, question_type: str):
spec = self._feedback_question_specs().get(question_type, self._feedback_question_specs()["presence"])
return (
f"Concept: '{concept_label}'\n"
f"{spec['prompt'].format(label=concept_label)}\n"
f"{spec['legend']}"
)
# =========================================================================
# MULTI-LEVEL CONCEPT SYSTEM
# =========================================================================
def _parse_concepts(self, prompt: str):
"""
Level 2 → Level 3 boundary: split the prompt on commas/semicolons into
concept phrases. Each phrase is returned as a list of significant lowercase
words, preserving prompt order.
Example:
"masterpiece, anime girl, long hair, dark forest"
-> [["masterpiece"], ["anime","girl"], ["long","hair"], ["dark","forest"]]
"""
if not prompt:
return []
result = []
for segment_type, segment_text in self._iter_prompt_segments(prompt):
if segment_type == "protected":
protected_phrase = segment_text.strip().lower()
if len(protected_phrase) >= 3 and any(c.isalpha() for c in protected_phrase):
result.append([protected_phrase])
continue
phrases = [p.strip() for p in re.split(r'[,;]', segment_text) if p.strip()]
for phrase in phrases:
words = [w.strip().lower() for w in phrase.split() if self._is_valuable_token(w.strip())]
if words:
result.append(words)
return result
def _concept_overlap_words(self, prompt: str):
words = []
seen = set()
for phrase_words in self._parse_concepts(prompt):
for word in phrase_words:
clean = str(word).strip().lower()
if not clean or clean in seen:
continue
seen.add(clean)
words.append(clean)
return words
def _ordered_prompt_words(self, prompt: str):
words = []
for phrase_words in self._parse_concepts(prompt):
for word in phrase_words:
clean = str(word).strip().lower()
if clean and self._is_valuable_token(clean):
words.append(clean)
return words
def _ordered_prompt_phrases(self, prompt: str):
phrases = []
if not prompt:
return phrases
phrase_texts = []
for segment_type, segment_text in self._iter_prompt_segments(prompt):
if segment_type == "protected":
phrase_texts.append(segment_text)
else:
phrase_texts.extend(p.strip() for p in re.split(r'[,;]', segment_text) if p.strip())
for phrase_text in phrase_texts:
text = re.sub(r"\s+", " ", str(phrase_text or "").strip().lower()).strip(" ,;:.")
if not text or len(text) < 3:
continue
phrase_tokens = [
token.strip().lower()
for token in re.findall(r"[\w'’.-]+", text, flags=re.UNICODE)
if self._is_valuable_token(token.strip())
]
if not phrase_tokens:
continue
phrases.append({
"text": text,
"tokens": phrase_tokens,
})
return phrases
def _history_prompt_texts(self, prompt_key: str, history_entry: dict):
texts = []
for key in ("canonical_prompt", "last_positive_prompt"):
value = history_entry.get(key)
if isinstance(value, str) and value.strip():
texts.append(value)
if isinstance(prompt_key, str) and prompt_key.strip():
texts.append(prompt_key)
for item in reversed(history_entry.get("history", [])[-4:]):
value = item.get("prompt") if isinstance(item, dict) else None
if isinstance(value, str) and value.strip():
texts.append(value)
unique = []
seen = set()
for text in texts:
normalized = text.strip()
if normalized and normalized not in seen:
seen.add(normalized)
unique.append(normalized)
return unique
def _prompt_history_concept_words(self, prompt_key: str, history_entry: dict):
stored = history_entry.get("prompt_concept_words")
if isinstance(stored, list) and stored:
return [str(word).lower() for word in stored if str(word).strip()]
words = []
seen = set()
for text in self._history_prompt_texts(prompt_key, history_entry):
for word in self._concept_overlap_words(text):
if word not in seen:
seen.add(word)
words.append(word)
return words
def _conditioning_similarity(self, reference: torch.Tensor, current: torch.Tensor, token_mask=None):
if not isinstance(reference, torch.Tensor) or not isinstance(current, torch.Tensor):
return 0.0
if list(reference.shape) != list(current.shape):
return 0.0
current = current.to(reference.device) if current.device != reference.device else current
try:
if reference.dim() >= 2 and current.dim() >= 2:
ref_mean = self._masked_sequence_mean(reference, token_mask)
cur_mean = self._masked_sequence_mean(current, token_mask)
return float(F.cosine_similarity(ref_mean, cur_mean, dim=-1).mean().item())
return float(F.cosine_similarity(
reference.flatten().unsqueeze(0),
current.flatten().unsqueeze(0),
dim=-1
).mean().item())
except Exception:
return 0.0
def _conditioning_history_similarity(self, history_entry: dict, raw_positive: torch.Tensor):
source_data = history_entry.get("source_conditioning_embeds") or history_entry.get("reference_embeds")
try:
source_reference = serializable_to_tensor(source_data)
except Exception:
return 0.0
return self._conditioning_similarity(source_reference, raw_positive)
def _find_prompt_variant_history(self, exact_prompt_key: str, current_words: list,
raw_positive: torch.Tensor, prompt_histories: dict):
if not exact_prompt_key or not current_words or not prompt_histories:
return None, None
current_set = set(current_words)
if len(current_set) < 2:
return None, None
best_key = None
best_info = None
best_score = 0.0
for candidate_key, history_entry in prompt_histories.items():
if candidate_key == exact_prompt_key or not isinstance(history_entry, dict):
continue
candidate_words = self._prompt_history_concept_words(candidate_key, history_entry)
candidate_set = set(candidate_words)
if len(candidate_set) < 2:
continue
shared = current_set & candidate_set
shared_count = len(shared)
union = current_set | candidate_set
overlap = shared_count / max(1, len(union))
coverage = shared_count / max(1, min(len(current_set), len(candidate_set)))
semantic_score = max(overlap, coverage * 0.72)
conditioning_similarity = self._conditioning_history_similarity(history_entry, raw_positive)
if conditioning_similarity < 0.68:
continue
if shared_count < 3 and not (coverage >= 0.85 and conditioning_similarity >= 0.82):
continue
if not (
(overlap >= 0.46 and conditioning_similarity >= 0.78) or
(coverage >= 0.68 and conditioning_similarity >= 0.72) or
(overlap >= 0.60 and conditioning_similarity >= 0.68)
):
continue
score = semantic_score * 0.58 + conditioning_similarity * 0.42
if score > best_score:
best_score = score
best_key = candidate_key
best_info = {
"score": round(float(score), 4),
"overlap": round(float(overlap), 4),
"coverage": round(float(coverage), 4),
"conditioning_similarity": round(float(conditioning_similarity), 4),
"shared_words": sorted(shared)[:12],
"matched_prompt_key": candidate_key,
}
return best_key, best_info
def _remember_prompt_variant(self, active: dict, exact_prompt_key: str, positive_prompt: str,
current_words: list, match_info: dict = None):
active.setdefault("canonical_prompt", positive_prompt or exact_prompt_key)
existing_words = set(
str(word).lower()
for word in active.get("prompt_concept_words", [])
if str(word).strip()
)
for word in current_words:
existing_words.add(str(word).lower())
active["prompt_concept_words"] = sorted(existing_words)[:256]
variants = list(active.get("prompt_variants", []))[-11:]
variant = {
"prompt_key": exact_prompt_key,
"prompt": (positive_prompt or exact_prompt_key)[:240],
}
if match_info:
variant.update({
"overlap": match_info.get("overlap"),
"coverage": match_info.get("coverage"),
"conditioning_similarity": match_info.get("conditioning_similarity"),
})
variants.append(variant)
active["prompt_variants"] = variants
def _history_entry_prompt_words(self, entry: dict):
if not isinstance(entry, dict):
return []
stored = entry.get("prompt_words")
if isinstance(stored, list) and stored:
return [str(word).lower() for word in stored if str(word).strip()]
prompt_text = entry.get("prompt_full") or entry.get("prompt") or ""
return self._concept_overlap_words(prompt_text)
def _retarget_prompt_history_to_source(self, active: dict, old_source: torch.Tensor,
new_source: torch.Tensor):
if (
not isinstance(active, dict) or
not isinstance(old_source, torch.Tensor) or
not isinstance(new_source, torch.Tensor) or
list(old_source.shape) != list(new_source.shape)
):
return 0
shift = new_source.to(old_source.device) - old_source
migrated = 0
def _shift_serialized(field_name: str):
nonlocal migrated
payload = active.get(field_name)
if payload is None:
return
try:
tensor = serializable_to_tensor(payload).to(old_source.device)
if list(tensor.shape) != list(old_source.shape):
return
active[field_name] = tensor_to_serializable(tensor + shift)
migrated += 1
except Exception:
return
_shift_serialized("reference_embeds")
_shift_serialized("liked_reference_embeds")
for item in active.get("history", []):
if not isinstance(item, dict) or item.get("modified_embeds") is None:
continue
try:
tensor = serializable_to_tensor(item.get("modified_embeds")).to(old_source.device)
if list(tensor.shape) != list(old_source.shape):
continue
item["modified_embeds"] = tensor_to_serializable(tensor + shift)
item["retargeted_to_prompt_variant"] = True
migrated += 1
except Exception:
continue
return migrated
def _nudge_delta_words(self, words: list, delta: float, concept_clusters: dict,
word_importance: dict, iter_num: int, role: str):
touched = []
for raw_word in words:
word = str(raw_word).strip().lower()
if not self._is_valuable_token(word):
continue
cid, is_new = self._match_concept([word], concept_clusters, threshold=0.18)
if cid is None:
continue
if is_new or cid not in concept_clusters:
concept_clusters[cid] = self._default_concept_cluster([word])
else:
concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
if word not in concept_clusters[cid].get("anchor_words", []):
concept_clusters[cid]["anchor_words"].append(word)
cluster = concept_clusters[cid]
local_imp = cluster.setdefault("word_importance", {})
local_imp[word] = max(0.35, min(2.8, float(local_imp.get(word, 1.0)) + delta))
word_importance[word] = max(0.35, min(2.8, float(word_importance.get(word, 1.0)) + delta * 0.45))
cluster["presence_target"] = self._clip_profile_value(
float(cluster.get("presence_target", 1.0)) + delta * 0.28
)
cluster["priority_weight"] = self._clip_profile_value(
float(cluster.get("priority_weight", 1.0)) + delta * 0.20
)
if delta > 0:
cluster["semantic_fidelity"] = self._clip_profile_value(
float(cluster.get("semantic_fidelity", 1.0)) + delta * 0.12,
low=0.6,
high=1.8,
)
else:
cluster["overrep_sensitivity"] = self._clip_profile_value(
float(cluster.get("overrep_sensitivity", 1.0)) + abs(delta) * 0.12
)
cluster["last_seen_iter"] = iter_num
cluster["usage_count"] = int(cluster.get("usage_count", 0)) + 1
cluster["last_prompt_delta_role"] = role
touched.append(word)
return touched
def _missing_axis_matches_category(self, axis: str, category: str):
category = (category or "general").lower()
if axis == "concept":
return category in {"subject", "action", "appearance", "character", "concept", "general"}
if axis == "details":
return category in {"camera", "action", "environment", "appearance", "style", "general"}
if axis == "quality":
return category in {"quality", "style", "general"}
return False
def _apply_missing_axis_prompt_pressure(self, word_groups, word_to_concept, rating_profile,
concept_clusters, word_importance, iter_num):
missing_axes = set(rating_profile.get("missing_axes", []))
if not missing_axes or not word_groups:
return ""
axis_boosts = {"concept": 0.34, "details": 0.26, "quality": 0.22}
touched = []
for _, _, full_word, _ in word_groups:
word = str(full_word).strip().lower()
if not self._is_valuable_token(word):
continue
cid = word_to_concept.get(word)
category = "general"
cluster = None
if cid and cid in concept_clusters:
concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
cluster = concept_clusters[cid]
category = cluster.get("category", "general")
matched_axes = [axis for axis in missing_axes if self._missing_axis_matches_category(axis, category)]
pressure = sum(axis_boosts.get(axis, 0.0) for axis in matched_axes)
if not pressure:
pressure = sum(axis_boosts.get(axis, 0.0) for axis in missing_axes) * 0.35
if not pressure:
continue
word_importance[word] = max(0.35, min(2.8, float(word_importance.get(word, 1.0)) + pressure * 0.32))
if cluster is not None:
local_imp = cluster.setdefault("word_importance", {})
local_imp[word] = max(0.35, min(2.8, float(local_imp.get(word, 1.0)) + pressure * 0.55))
cluster["missing_count"] = int(cluster.get("missing_count", 0)) + 1
for axis in missing_axes:
counter = self._axis_counter_name(axis)
if counter:
cluster[counter] = int(cluster.get(counter, 0)) + 1
if "concept" in missing_axes or "details" in missing_axes:
cluster["presence_target"] = self._clip_profile_value(
float(cluster.get("presence_target", 1.0)) + pressure * 0.18
)
cluster["priority_weight"] = self._clip_profile_value(
float(cluster.get("priority_weight", 1.0)) + pressure * 0.14
)
if "quality" in missing_axes:
cluster["semantic_fidelity"] = self._clip_profile_value(
float(cluster.get("semantic_fidelity", 1.0)) + pressure * 0.16,
low=0.6,
high=1.8,
)
cluster["stability_weight"] = self._clip_profile_value(
float(cluster.get("stability_weight", 1.0)) + pressure * 0.12
)
cluster["last_missing_iter"] = iter_num
touched.append(word)
if not touched:
return ""
axes = "+".join(sorted(missing_axes))
return f"Missing-axis pressure: {axes} reinforced {', '.join(touched[:10])}{'...' if len(touched) > 10 else ''}."
def _apply_prompt_delta_attribution(self, history: list, current_words: list,
rating_profile: dict, concept_clusters: dict,
word_importance: dict, iter_num: int):
if len(history) < 2:
return ""
previous_entry = history[-2]
rated_entry = history[-1]
previous_words = self._history_entry_prompt_words(previous_entry)
rated_words = self._history_entry_prompt_words(rated_entry) or current_words
if not previous_words or not rated_words:
return ""
previous_set = set(previous_words)
rated_set = set(rated_words)
added = sorted(rated_set - previous_set)
removed = sorted(previous_set - rated_set)
if not added and not removed:
return ""
previous_profile = normalize_refiner_rating(rated_entry.get("rating_label", rated_entry.get("rating", 0)))
current_key = rating_profile.get("key", "")
previous_missing_axes = set(previous_profile.get("missing_axes", []))
current_missing_axes = set(rating_profile.get("missing_axes", []))
word_importance = word_importance if isinstance(word_importance, dict) else {}
boosted = []
softened = []
reason = ""
if current_key == "like" and previous_missing_axes:
boost = 0.22 + (0.10 if "concept" in previous_missing_axes else 0.0)
boosted = self._nudge_delta_words(
added[:24],
boost,
concept_clusters,
word_importance,
iter_num,
"added_helped",
)
softened = self._nudge_delta_words(
removed[:24],
-0.08,
concept_clusters,
word_importance,
iter_num,
"removed_after_missing",
)
reason = f"{previous_profile.get('label', 'missing')} -> {rating_profile.get('label', 'Perfect')}"
elif current_missing_axes and previous_profile.get("key") == "like":
boost = 0.14 + (0.08 if "concept" in current_missing_axes else 0.0)
boosted = self._nudge_delta_words(
removed[:24],
boost,
concept_clusters,
word_importance,
iter_num,
"removed_needed",
)
softened = self._nudge_delta_words(
added[:24],
-0.10,
concept_clusters,
word_importance,
iter_num,
"added_hurt",
)
reason = f"{previous_profile.get('label', 'Perfect')} -> {rating_profile.get('label', 'missing')}"
if not boosted and not softened:
return ""
parts = [f"Prompt delta: {reason}."]
if boosted:
parts.append("boosted " + ", ".join(boosted[:8]))
if softened:
parts.append("softened " + ", ".join(softened[:8]))
return " ".join(parts)
def _build_prompt_fallback_concept(self, prompt: str, concept_clusters: dict):
if not prompt:
return None
fallback_words = []
for segment_type, segment_text in self._iter_prompt_segments(prompt):
if segment_type == "protected":
protected_phrase = segment_text.strip().lower()
if len(protected_phrase) >= 3 and any(c.isalpha() for c in protected_phrase):
fallback_words.append(protected_phrase)
continue
fallback_words.extend(
w.strip().lower()
for w in re.split(r'[\s,;]+', segment_text)
if self._is_valuable_token(w.strip())
)
if len(fallback_words) >= 8:
break
fallback_words = fallback_words[:8]
if not fallback_words:
return None
fallback_id = "prompt_" + md5("|".join(fallback_words).encode()).hexdigest()[:10]
if fallback_id not in concept_clusters:
concept_clusters[fallback_id] = self._default_concept_cluster(fallback_words)
concept_clusters[fallback_id]["label"] = " ".join(fallback_words[:5])
else:
concept_clusters[fallback_id] = self._ensure_concept_cluster_defaults(concept_clusters[fallback_id])
return fallback_id
def _force_feedback_fallback(self, ordered_concept_ids: list, concept_clusters: dict,
concept_groups: dict, current_concept_labels: dict,
rating_shift: float, similarity: float):
if not ordered_concept_ids:
return None
ranked = []
for cid in ordered_concept_ids:
cluster = concept_clusters.get(cid)
if not cluster:
continue
cluster = self._ensure_concept_cluster_defaults(cluster)
concept_clusters[cid] = cluster
mean_imp = self._get_concept_mean_importance(cluster)
dominance = abs(mean_imp - cluster.get("presence_target", 1.0))
ranked.append((dominance, cid, cluster))
if not ranked:
return None
_, chosen_cid, cluster = max(ranked, key=lambda x: x[0])
if rating_shift >= 2.0:
question_type = "fidelity"
elif similarity < 0.84:
question_type = "stability"
elif cluster.get("category") in {"style", "camera", "quality"}:
question_type = "balance"
else:
question_type = "presence"
chosen_group_id = None
for gid, group in concept_groups.items():
if chosen_cid in group.get("concept_ids", []):
chosen_group_id = gid
break
return {
"concept_id": chosen_cid,
"concept_label": current_concept_labels.get(chosen_cid, cluster.get("label", "")),
"question_type": question_type,
"neighbor_ids": self._get_concept_neighbors(chosen_cid, ordered_concept_ids, radius=2),
"group_id": chosen_group_id,
"category": cluster.get("category", "general"),
"score": 0.0,
}
def _match_concept(self, phrase_words: list, concept_clusters: dict, threshold: float = 0.38):
"""
Match a phrase to an existing concept cluster using Jaccard similarity on
anchor words. If no cluster clears `threshold`, a new cluster id is minted.
Returns (cluster_id, is_new_cluster).
Tune threshold: lower -> more isolated clusters; higher -> more merging.
"""
if not phrase_words:
return None, False
phrase_set = set(phrase_words)
best_id, best_score = None, 0.0
for cid, cluster in concept_clusters.items():
anchor_set = set(cluster.get("anchor_words", []))
if not anchor_set:
continue
union = len(phrase_set | anchor_set)
if union == 0:
continue
jaccard = len(phrase_set & anchor_set) / union
if jaccard > best_score:
best_score, best_id = jaccard, cid
if best_score >= threshold:
return best_id, False
new_id = md5("|".join(sorted(phrase_words)).encode()).hexdigest()[:10]
return new_id, True
def _apply_category_feedback(self, cluster: dict, feedback_rating: int, iter_num: int = 0):
category = CATEGORY_FEEDBACK_MAP.get(int(feedback_rating), "general")
cluster["category"] = category
cluster["category_source"] = "user"
cluster["category_confidence"] = 1.0
if category in {"concept", "character", "details"}:
cluster["presence_target"] = self._clip_profile_value(cluster.get("presence_target", 1.0) + 0.10)
cluster["priority_weight"] = self._clip_profile_value(cluster.get("priority_weight", 1.0) + 0.10)
cluster["semantic_fidelity"] = self._clip_profile_value(
cluster.get("semantic_fidelity", 1.0) + 0.08,
low=0.6,
high=1.8,
)
elif category == "quality":
cluster["priority_weight"] = self._clip_profile_value(cluster.get("priority_weight", 1.0) + 0.06)
cluster["overrep_sensitivity"] = self._clip_profile_value(cluster.get("overrep_sensitivity", 1.0) + 0.08)
elif category == "style":
cluster["stability_weight"] = self._clip_profile_value(cluster.get("stability_weight", 1.0) + 0.08)
cluster["user_affinity"] = self._clip_profile_value(cluster.get("user_affinity", 1.0) + 0.05)
cluster["question_history"] = list(cluster.get("question_history", []))[-23:]
cluster["question_history"].append({
"iteration": iter_num,
"type": "category",
"rating": feedback_rating,
"category": category,
})
cluster["last_question_type"] = "category"
cluster["last_question_iter"] = iter_num
def _build_word_concept_map(self, prompt: str, concept_clusters: dict):
"""
Parse the prompt into concept phrases (Level 3), match or create clusters,
and return:
- word_to_concept: dict mapping each significant word -> cluster_id
- ordered_concept_ids: list of cluster IDs in prompt order (no duplicates)
Updates concept_clusters in-place. The ordered list is the backbone for
group building and neighbour lookup at Level 4.
"""
word_to_concept = {}
ordered_concept_ids = []
current_concept_labels = {}
for phrase_words in self._parse_concepts(prompt):
cid, is_new = self._match_concept(phrase_words, concept_clusters)
if cid is None:
continue
phrase_label = " ".join(phrase_words[:6])
if is_new:
concept_clusters[cid] = self._default_concept_cluster(phrase_words)
else:
concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
# Expand anchor vocabulary with words newly seen in this phrase
existing = set(concept_clusters[cid]["anchor_words"])
for w in phrase_words:
if w not in existing:
concept_clusters[cid]["anchor_words"].append(w)
if (
concept_clusters[cid].get("category_source") != "user" and
concept_clusters[cid].get("category") == "general"
):
concept_clusters[cid]["category"] = self._infer_concept_category(
concept_clusters[cid]["anchor_words"]
)
concept_clusters[cid]["category_confidence"] = (
0.35 if concept_clusters[cid]["category"] == "general" else 0.65
)
concept_clusters[cid]["last_prompt_label"] = phrase_label
current_concept_labels[cid] = phrase_label
for w in phrase_words:
word_to_concept[w] = cid
if cid not in ordered_concept_ids:
ordered_concept_ids.append(cid)
return word_to_concept, ordered_concept_ids, current_concept_labels
def _build_concept_groups(self, ordered_concept_ids: list, concept_clusters: dict,
existing_groups: dict, current_concept_labels: dict,
window: int = 3):
"""
Level 4: group consecutive concept phrases into semantic sentence-level
units using a non-overlapping sliding window of size `window`.
Groups model the "sentences" of the prompt — natural semantic blocks
like (quality tags), (subject + appearance), (setting), (style + tech).
Existing groups are matched by their concept_ids set and updated in-place
to preserve their accumulated reward_ema and usage_count. New groups are
created when a previously unseen concept combination appears.
Returns the updated groups dict.
"""
if not ordered_concept_ids:
return existing_groups
groups = dict(existing_groups)
for i in range(0, len(ordered_concept_ids), window):
chunk = ordered_concept_ids[i:i + window]
if not chunk:
continue
chunk_set = frozenset(chunk)
# Try to match an existing group by concept set identity
existing_gid = None
for gid, g in groups.items():
if frozenset(g.get("concept_ids", [])) == chunk_set:
existing_gid = gid
break
if existing_gid:
# Preserve the canonical prompt order even if it shifted slightly
groups[existing_gid]["concept_ids"] = chunk
groups[existing_gid]["label"] = " | ".join(
current_concept_labels.get(cid, concept_clusters.get(cid, {}).get("label", cid))
for cid in chunk
)
else:
labels = [current_concept_labels.get(cid, concept_clusters[cid]["label"])
for cid in chunk if cid in concept_clusters]
gid = md5("|".join(chunk).encode()).hexdigest()[:10]
groups[gid] = {
"label": " | ".join(labels),
"concept_ids": chunk,
"reward_ema": 0.0,
"usage_count": 0,
"last_seen_iter": 0,
}
return groups
def _get_concept_neighbors(self, concept_id: str, ordered_concept_ids: list,
radius: int = 2):
"""
Return the concept IDs positionally adjacent to `concept_id` within
`radius` steps in the prompt's ordered concept list.
Adjacent concepts compete for embedding space: if "anime style" is
overrepresented, it likely crowds its neighbours "cinematic lighting"
and "detailed background". This adjacency is what makes the neighbour
signal in _apply_concept_feedback meaningful.
"""
if concept_id not in ordered_concept_ids:
return []
idx = ordered_concept_ids.index(concept_id)
neighbors = []
for offset in range(-radius, radius + 1):
if offset == 0:
continue
ni = idx + offset
if 0 <= ni < len(ordered_concept_ids):
neighbors.append(ordered_concept_ids[ni])
return neighbors
def _get_dominant_concept(self, ordered_concept_ids: list, concept_clusters: dict):
"""
Return (cluster_id, avg_importance, label) for the concept with the
highest mean word importance — i.e. what the embedding currently weighs
most heavily. This is the "main thing forming the video" signal.
"""
best_cid, best_score, best_label = None, -1.0, ""
for cid in ordered_concept_ids:
if cid not in concept_clusters:
continue
imp_vals = list(concept_clusters[cid]["word_importance"].values())
if not imp_vals:
continue
avg = sum(imp_vals) / len(imp_vals)
if avg > best_score:
best_score, best_cid, best_label = avg, cid, concept_clusters[cid]["label"]
return best_cid, best_score, best_label
def _apply_concept_feedback(self, concept_id: str, feedback_rating: int,
question_type: str,
concept_clusters: dict, neighbor_ids: list,
word_importance: dict, concept_groups: dict,
iter_num: int = 0):
"""
Multi-level feedback propagation.
Level 3 — rated concept phrase:
All word importances inside the rated cluster shift by `direct_delta`.
This is the primary, high-confidence signal.
Level 3 — neighbour concepts (adjacent phrases in the prompt):
Receive a dampened signal modelled on embedding-space competition.
- Concept being boosted (absent/weak): neighbours get a small inhibitory
nudge, giving the boosted concept more semantic room.
- Concept being reduced (overrepresented): neighbours also receive a mild
reduction — the whole semantic area is too heavy.
Magnitude: 22% of direct_delta; same sign for overrep, opposite for absent.
Level 4 — concept groups:
The reward_ema of every group containing the rated concept is updated,
enabling group-level health tracking over time.
Level 2 — global word_importance fallback:
Each change is also written at 40% strength into the flat global dict
so the scheduler and prodigy system retain a valid signal.
Feedback scale:
1 = absent -> direct_delta = +0.90 (strong boost)
2 = weak -> direct_delta = +0.50
3 = slightly weak -> direct_delta = +0.20
4 = slightly strong -> direct_delta = -0.15
5 = perfect -> direct_delta = +0.04 (small stability reward)
6 = overrepresented -> direct_delta = -0.55 (strong reduction)
"""
if concept_id not in concept_clusters:
return
cluster = self._ensure_concept_cluster_defaults(concept_clusters[concept_id])
concept_clusters[concept_id] = cluster
question_type = question_type or "presence"
if question_type == "category":
self._apply_category_feedback(cluster, feedback_rating, iter_num)
return
direct_deltas = {
"presence": {1: 0.90, 2: 0.50, 3: 0.20, 4: -0.15, 5: 0.04, 6: -0.55},
"priority": {1: -0.25, 2: -0.12, 3: -0.05, 4: 0.08, 5: 0.18, 6: 0.30},
"balance": {1: 0.72, 2: 0.40, 3: 0.16, 4: -0.10, 5: 0.00, 6: -0.62},
"fidelity": {1: 0.25, 2: 0.16, 3: 0.08, 4: 0.02, 5: 0.00, 6: -0.06},
"stability": {1: 0.10, 2: 0.06, 3: 0.03, 4: 0.00, 5: -0.02, 6: -0.06},
"preference": {1: -0.30, 2: -0.18, 3: -0.08, 4: 0.10, 5: 0.22, 6: 0.36},
}
direct_delta = direct_deltas.get(question_type, direct_deltas["presence"]).get(feedback_rating, 0.0)
centered = (feedback_rating - 3.5) / 2.5
# --- Level 3: rated concept ---
local_imp = cluster["word_importance"]
for wkey in list(local_imp.keys()):
local_imp[wkey] = max(0.3, min(2.8, local_imp[wkey] + direct_delta))
if wkey in word_importance:
word_importance[wkey] = max(0.3, min(2.8, word_importance[wkey] + direct_delta * 0.4))
if question_type == "presence":
cluster["presence_target"] = self._clip_profile_value(
cluster.get("presence_target", 1.0) + direct_delta * 0.22
)
elif question_type == "priority":
cluster["priority_weight"] = self._clip_profile_value(
cluster.get("priority_weight", 1.0) + centered * 0.18
)
elif question_type == "balance":
cluster["overrep_sensitivity"] = self._clip_profile_value(
cluster.get("overrep_sensitivity", 1.0) + max(0.0, centered) * 0.24 - max(0.0, -centered) * 0.12
)
cluster["presence_target"] = self._clip_profile_value(
cluster.get("presence_target", 1.0) + direct_delta * 0.12
)
elif question_type == "fidelity":
cluster["semantic_fidelity"] = self._clip_profile_value(
cluster.get("semantic_fidelity", 1.0) + (centered * 0.16),
low=0.6, high=1.8
)
elif question_type == "stability":
cluster["stability_weight"] = self._clip_profile_value(
cluster.get("stability_weight", 1.0) + centered * 0.18
)
elif question_type == "preference":
cluster["user_affinity"] = self._clip_profile_value(
cluster.get("user_affinity", 1.0) + centered * 0.20
)
cluster["priority_weight"] = self._clip_profile_value(
cluster.get("priority_weight", 1.0) + centered * 0.08
)
cluster["presence_target"] = self._clip_profile_value(
cluster.get("presence_target", 1.0) + centered * 0.10
)
# --- Level 3: neighbour concepts ---
# Boosted concept -> inhibit neighbours (give it semantic space).
# Reduced concept -> also soften neighbours (the whole area is too heavy).
if direct_delta > 0:
neighbor_delta = -direct_delta * 0.22
else:
neighbor_delta = direct_delta * 0.22
for nid in neighbor_ids:
if nid not in concept_clusters or nid == concept_id:
continue
concept_clusters[nid] = self._ensure_concept_cluster_defaults(concept_clusters[nid])
n_local = concept_clusters[nid]["word_importance"]
for wkey in list(n_local.keys()):
n_local[wkey] = max(0.3, min(2.8, n_local[wkey] + neighbor_delta))
if wkey in word_importance:
word_importance[wkey] = max(0.3, min(2.8,
word_importance[wkey] + neighbor_delta * 0.4))
# --- Level 4: concept group reward EMA ---
# Maps feedback 1-6 onto a [-1, +1] reward signal.
group_reward_map = {1: -1.0, 2: -0.5, 3: -0.1, 4: 0.4, 5: 1.0, 6: -0.6}
normalized = group_reward_map.get(feedback_rating, 0.0)
for gid, g in concept_groups.items():
if concept_id in g.get("concept_ids", []):
g["reward_ema"] = 0.75 * g.get("reward_ema", 0.0) + 0.25 * normalized
g["usage_count"] = g.get("usage_count", 0) + 1
cluster["question_history"] = list(cluster.get("question_history", []))[-23:]
cluster["question_history"].append({
"iteration": iter_num,
"type": question_type,
"rating": feedback_rating,
})
cluster["last_question_type"] = question_type
cluster["last_question_iter"] = iter_num
# =========================================================================
# SIGMA REFINEMENT
# =========================================================================
def _ensure_sigma_state_defaults(self, global_adaptive: dict):
global_adaptive.setdefault("sigma_profile", [0.0] * 32)
global_adaptive.setdefault("last_applied_sigma_profile", [0.0] * 32)
global_adaptive.setdefault("sigma_iterations", 0)
global_adaptive.setdefault("sigma_avg_reward_ema", 0.0)
global_adaptive.setdefault("sigma_exploration_base", 0.035)
global_adaptive.setdefault("sigma_history", [])
def _resolve_sigma_strength(self, strength_mode: str):
strength_map = {
"off": 0.0,
"subtle": 0.18,
"medium": 0.35,
"strong": 0.60,
"max": 1.0,
}
return strength_map.get((strength_mode or "subtle").lower(), 0.18)
def _sigma_resample_profile(self, profile, target_len):
if target_len <= 0:
return np.zeros((0,), dtype=np.float32)
arr = np.asarray(profile, dtype=np.float32)
if arr.size == 0:
arr = np.zeros((32,), dtype=np.float32)
if arr.size == target_len:
return arr.copy()
if arr.size == 1:
return np.full((target_len,), float(arr[0]), dtype=np.float32)
src = np.linspace(0.0, 1.0, arr.size)
dst = np.linspace(0.0, 1.0, target_len)
return np.interp(dst, src, arr).astype(np.float32)
def _sigma_smooth_noise(self, noise):
if noise.size <= 2:
return noise
kernel = np.array([0.25, 0.5, 0.25], dtype=np.float32)
smoothed = np.convolve(noise, kernel, mode="same")
smoothed = np.convolve(smoothed, kernel, mode="same")
return smoothed.astype(np.float32)
def _sigma_enforce_monotonic(self, tuned: torch.Tensor, original: torch.Tensor):
out = tuned.clone()
count = int(out.shape[0])
if count <= 2:
return out
eps = max(1e-6, float((original[0] - original[-1]).abs().item()) * 1e-6)
for i in range(1, count - 1):
max_allowed = out[i - 1] - eps
if out[i] >= max_allowed:
out[i] = max_allowed
out[-1] = original[-1]
for i in range(count - 2, 0, -1):
min_allowed = out[i + 1] + eps
if out[i] <= min_allowed:
out[i] = min_allowed
out[0] = original[0]
out[-1] = original[-1]
return out
def _refine_sigma_schedule(self, sigmas, rating: int, global_adaptive: dict, strength_mode: str, seed: int,
rating_profile: Optional[dict] = None):
if not isinstance(sigmas, torch.Tensor):
return torch.FloatTensor([]), "Sigma refinement inactive."
original_sigmas = sigmas.detach().clone()
if original_sigmas.numel() <= 2:
return original_sigmas, "Sigma refinement skipped: schedule too short."
self._ensure_sigma_state_defaults(global_adaptive)
rating_profile = rating_profile or normalize_refiner_rating(rating)
rating_key = rating_profile.get("key", "")
reward = float(rating_profile.get("reward", (rating - 5.5) / 4.5))
sigma_iterations = int(global_adaptive.get("sigma_iterations", 0))
profile = np.asarray(global_adaptive.get("sigma_profile", [0.0] * 32), dtype=np.float32)
last_applied = np.asarray(global_adaptive.get("last_applied_sigma_profile", [0.0] * 32), dtype=np.float32)
sigma_lr = 0.22 / (1.0 + 0.08 * math.sqrt(max(1, sigma_iterations)))
profile = np.clip(profile + (last_applied * reward * sigma_lr), -1.0, 1.0)
profile = np.clip(profile - profile.mean(), -1.0, 1.0)
sigma_avg_reward_ema = float(global_adaptive.get("sigma_avg_reward_ema", 0.0))
sigma_avg_reward_ema = 0.85 * sigma_avg_reward_ema + 0.15 * reward
sigma_exploration_base = float(global_adaptive.get("sigma_exploration_base", 0.035))
sigma_exploration_scale = max(0.18, 1.0 - max(0.0, sigma_avg_reward_ema) * 0.7)
if seed != 0:
np.random.seed(seed % (2 ** 32))
noise = np.random.normal(
0.0,
sigma_exploration_base * sigma_exploration_scale,
size=profile.shape[0]
).astype(np.float32)
applied_profile = np.clip(profile + self._sigma_smooth_noise(noise), -1.0, 1.0)
applied_profile = np.clip(applied_profile - applied_profile.mean(), -1.0, 1.0)
middle_profile = self._sigma_resample_profile(applied_profile, max(0, original_sigmas.shape[0] - 2))
middle_profile_tensor = torch.tensor(middle_profile, dtype=original_sigmas.dtype, device=original_sigmas.device)
tuned_sigmas = original_sigmas.clone()
sigma_strength = self._resolve_sigma_strength(strength_mode)
for idx in range(1, int(original_sigmas.shape[0]) - 1):
delta = float(middle_profile_tensor[idx - 1].item())
current = original_sigmas[idx]
prev_sigma = original_sigmas[idx - 1]
next_sigma = original_sigmas[idx + 1]
if delta >= 0.0:
tuned_sigmas[idx] = current + (prev_sigma - current) * sigma_strength * delta
else:
tuned_sigmas[idx] = current + (current - next_sigma) * sigma_strength * delta
tuned_sigmas = self._sigma_enforce_monotonic(tuned_sigmas, original_sigmas)
global_adaptive["sigma_profile"] = profile.tolist()
global_adaptive["last_applied_sigma_profile"] = applied_profile.tolist()
global_adaptive["sigma_iterations"] = sigma_iterations + 1
global_adaptive["sigma_avg_reward_ema"] = sigma_avg_reward_ema
global_adaptive["sigma_exploration_base"] = max(
0.012,
min(0.05, sigma_exploration_base * (0.97 if rating_key == "like" else 1.02))
)
sigma_history = list(global_adaptive.get("sigma_history", []))[-119:]
sigma_history.append({
"iteration": sigma_iterations + 1,
"rating": int(rating),
"rating_label": rating_profile.get("label", str(rating)),
"reward": round(float(reward), 6),
"lr": round(float(sigma_lr), 6),
})
global_adaptive["sigma_history"] = sigma_history
mean_shift = float((tuned_sigmas[1:-1] - original_sigmas[1:-1]).abs().mean().item()) if tuned_sigmas.numel() > 2 else 0.0
max_shift = float((tuned_sigmas[1:-1] - original_sigmas[1:-1]).abs().max().item()) if tuned_sigmas.numel() > 2 else 0.0
sigma_status = (
f"Sigma: iter {global_adaptive['sigma_iterations']} | strength {strength_mode} ({sigma_strength:.2f}) | "
f"mean shift {mean_shift:.6f} | max shift {max_shift:.6f} | "
f"endpoints preserved ({float(original_sigmas[0].item()):.6f} -> {float(original_sigmas[-1].item()):.6f})"
)
return tuned_sigmas, sigma_status
# =========================================================================
# LATENT REFINEMENT
# =========================================================================
def _resize_tensor_like(self, tensor: torch.Tensor, reference: torch.Tensor):
if not isinstance(tensor, torch.Tensor) or not isinstance(reference, torch.Tensor):
return None
out = tensor.to(device=reference.device, dtype=reference.dtype)
if list(out.shape) == list(reference.shape):
return out
if out.dim() != reference.dim() or out.dim() not in {4, 5}:
return None
batch = reference.shape[0]
if out.shape[0] < batch:
reps = [1] * out.dim()
reps[0] = math.ceil(batch / max(1, out.shape[0]))
out = out.repeat(*reps)
out = out[:batch]
channels = reference.shape[1]
if out.shape[1] < channels:
pad_shape = list(out.shape)
pad_shape[1] = channels - out.shape[1]
out = torch.cat([out, torch.zeros(pad_shape, device=out.device, dtype=out.dtype)], dim=1)
out = out[:, :channels]
if out.dim() == 4:
return F.interpolate(out, size=reference.shape[-2:], mode="bilinear", align_corners=False)
return F.interpolate(out, size=reference.shape[-3:], mode="trilinear", align_corners=False)
def _load_saved_latent(self, refinement_key, mode):
path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt")
if not os.path.exists(path):
return None, "Latent: no saved latent for this key."
try:
data = torch.load(path, map_location="cpu", weights_only=False)
except Exception as e:
return None, f"Latent: failed to load saved latent ({e})."
if not isinstance(data, dict) or not isinstance(data.get("samples"), torch.Tensor):
return None, "Latent: saved latent file is invalid."
return data, "Latent: saved latent loaded."
def _value_links_to_output(self, value, source_id, output_index):
if isinstance(value, (list, tuple)):
if len(value) >= 2 and str(value[0]) == source_id:
try:
if int(value[1]) == output_index:
return True
except (TypeError, ValueError):
pass
return any(self._value_links_to_output(item, source_id, output_index) for item in value)
if isinstance(value, dict):
return any(self._value_links_to_output(item, source_id, output_index) for item in value.values())
return False
def _is_output_connected(self, prompt, unique_id, output_index):
if prompt is None or unique_id is None:
return False
source_id = str(unique_id)
prompt_nodes = prompt.get("output", prompt) if isinstance(prompt, dict) else {}
if not isinstance(prompt_nodes, dict):
return False
for node in prompt_nodes.values():
if not isinstance(node, dict):
continue
inputs = node.get("inputs", {})
if isinstance(inputs, dict) and self._value_links_to_output(inputs, source_id, output_index):
return True
workflow = prompt.get("workflow") if isinstance(prompt, dict) else None
links = workflow.get("links", []) if isinstance(workflow, dict) else []
for link in links:
if isinstance(link, (list, tuple)) and len(link) >= 3:
try:
if str(link[1]) == source_id and int(link[2]) == output_index:
return True
except (TypeError, ValueError):
continue
elif isinstance(link, dict):
try:
if str(link.get("origin_id")) == source_id and int(link.get("origin_slot")) == output_index:
return True
except (TypeError, ValueError):
continue
return False
def _latent_refinement_disabled(self, latent):
return clone_latent(latent), "Latent: disabled because latent refinement input/output path is not fully connected."
def _raise_wrong_latent(self, latent):
sample_type = latent_sample_type_name(latent)
latent_type = latent.get("type", "unspecified") if isinstance(latent, dict) else "missing"
raise ValueError(f"{self.WRONG_LATENT_ERROR} Received samples={sample_type}, type={latent_type}.")
def _save_latent_reference(self, latent, refinement_key, mode):
samples = latent_samples(latent)
if samples is None:
return False
path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt")
os.makedirs(os.path.dirname(path), exist_ok=True)
bundle = cpu_tensor_bundle(latent)
bundle["_meta"] = {
"refinement_key": refinement_key,
"mode": (mode or "ltx2").lower(),
"samples_shape": list(samples.shape),
}
torch.save(bundle, path)
return True
def _delete_latent_reference(self, refinement_key, mode):
path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt")
try:
os.remove(path)
return True
except FileNotFoundError:
return False
except OSError:
return False
def _refine_latent(self, latent, refinement_key, mode, rating, reward, global_adaptive,
rating_profile: Optional[dict] = None):
if isinstance(latent, dict) and not latent_is_plain_video_tensor(latent):
self._raise_wrong_latent(latent)
current = latent_samples(latent)
saved_bundle, load_status = self._load_saved_latent(refinement_key, mode)
if current is None:
if saved_bundle is None:
raise ValueError(self.NO_LATENT_REFERENCE_ERROR)
saved_latent = latent_from_tensor_bundle(saved_bundle)
saved_samples = latent_samples(saved_latent)
if saved_samples is None:
raise ValueError(self.NO_LATENT_REFERENCE_ERROR)
latent_state = global_adaptive.setdefault("latent_refinement", {})
latent_state["last_shape"] = list(saved_samples.shape)
latent_state["saved_shape"] = list(saved_samples.shape)
latent_state["status"] = self.SAVED_LATENT_ONLY_STATUS
return saved_latent, latent_state["status"]
refined_latent = clone_latent(latent)
if saved_bundle is None:
self._save_latent_reference(latent, refinement_key, mode)
global_adaptive.setdefault("latent_refinement", {})["last_shape"] = list(current.shape)
return refined_latent, f"{load_status} Current latent saved as reference."
raw_saved_samples = saved_bundle.get("samples")
if list(raw_saved_samples.shape) != list(current.shape):
self._save_latent_reference(latent, refinement_key, mode)
latent_state = global_adaptive.setdefault("latent_refinement", {})
latent_state.pop("momentum", None)
latent_state["last_shape"] = list(current.shape)
latent_state["saved_shape"] = list(raw_saved_samples.shape)
return refined_latent, "Latent: input shape changed. Reference rewritten, passthrough."
saved_samples = raw_saved_samples.to(device=current.device, dtype=current.dtype)
latent_state = global_adaptive.setdefault("latent_refinement", {})
latent_state["last_shape"] = list(current.shape)
latent_state["saved_shape"] = list(raw_saved_samples.shape)
momentum_data = latent_state.get("momentum")
if isinstance(momentum_data, dict):
try:
momentum = serializable_to_tensor(momentum_data).to(device=current.device, dtype=current.dtype)
momentum = self._resize_tensor_like(momentum, current)
except Exception:
momentum = None
else:
momentum = None
if momentum is None:
momentum = torch.zeros_like(current)
nonzero_mask = current.ne(0) & saved_samples.ne(0)
if not bool(nonzero_mask.any()):
refined_latent["samples"] = current
latent_state["status"] = "Latent: only zero-valued positions available, passthrough."
return refined_latent, latent_state["status"]
rating_profile = rating_profile or normalize_refiner_rating(rating)
rating_key = rating_profile.get("key", "")
missing_axes = set(rating_profile.get("missing_axes", []))
if rating_key == "like":
lr = 0.035
elif missing_axes:
lr = 0.018
if "details" in missing_axes:
lr += 0.006
if "concept" in missing_axes:
lr += 0.012
if "quality" in missing_axes:
lr += 0.035
lr = min(0.065, lr)
elif rating_key == "dislike":
lr = 0.055
else:
lr = 0.018
target_delta = (saved_samples - current) * nonzero_mask.to(dtype=current.dtype)
update = target_delta * (reward * lr)
momentum = 0.82 * momentum + 0.18 * update
max_step = max(0.001, min(0.08, float(current.detach().abs().mean().item()) * 0.18))
latent_delta = torch.clamp(momentum, min=-max_step, max=max_step)
refined_samples = current + latent_delta
refined_samples = torch.where(nonzero_mask, refined_samples, current)
refined_samples = torch.nan_to_num(refined_samples, nan=0.0)
range_min = torch.minimum(current, saved_samples)
range_max = torch.maximum(current, saved_samples)
refined_samples = torch.maximum(torch.minimum(refined_samples, range_max), range_min)
refined_latent["samples"] = refined_samples
latent_state["momentum"] = tensor_to_serializable(momentum.detach().cpu())
latent_state["last_rating"] = int(rating)
latent_state["last_rating_label"] = rating_profile.get("label", str(rating))
latent_state["last_reward"] = round(float(reward), 4)
latent_state["nonzero_ratio"] = round(float(nonzero_mask.float().mean().item()), 6)
latent_state["status"] = (
f"Latent: adjusted shape {tuple(current.shape)} | nonzero {latent_state['nonzero_ratio']:.1%} | "
f"max step {max_step:.5f} | bounded to input/reference range"
)
return refined_latent, latent_state["status"]
# =========================================================================
# LORA WEIGHT SUGGESTIONS
# =========================================================================
def _lora_state_id(self, lora_name: str, lora_type: str):
return md5(f"{lora_name}::{lora_type}".encode("utf-8")).hexdigest()[:16]
FunPackGemmaEmbeddingRefiner = FunPackVideoRefiner
# A vision block leaves exactly one tag spare (the tags come from the tokenizer's INPUT
# sequence, the conditioning is what came back). More than a couple means the two describe
# different sequences, not a rounding difference.
_TAG_SURPLUS_IS_ROUTINE = 2
def _tag_composition(meta, count):
"""(text positions, image positions) among the first `count` tags. Best-effort."""
try:
tags = meta.get("minimax_token_tags")
flat = tags.reshape(-1)[:count]
vision = int((flat == 0).sum().item())
return int(flat.shape[0]) - vision, vision
except Exception: # noqa: BLE001
return -1, -1
def _h3_reconcile_token_tags(conditioning, label=""):
"""Keep MiniMax H3's `minimax_token_tags` aligned with the conditioning it rides on.
The DiT walks the tag vector position-by-position, so a conditioning whose token count
no longer matches its tags is an IndexError on the FIRST forward — reported as a bare
"list index out of range" with nothing pointing at the prompt or the conditioning.
Studio edits conditioning in many places, and H3 tokenizes reference images INTO the
same span (ref_image_size "match" scales them to the output resolution), so the count
moves when either the prompt or the resolution changes — which is what made this look
intermittent.
Grow the tags when tokens were appended, trim them when there are more tags than tokens.
Both keep position i meaning position i, which is all the DiT asks of them. Trimming
matters most on image prompts: the tags are built from the tokenizer's INPUT sequence and
the conditioning is what came back, so a vision block normally leaves one tag spare —
and dropping the whole vector there would throw away the modality-0 marks that tell the
DiT those positions are an image, not text. Say so either way. Non-H3 conditioning has
no tags and passes straight through.
"""
if not isinstance(conditioning, list):
return conditioning
try:
from .minimax_h3 import extend_token_tags, tags_match, token_tags_length, trim_token_tags
except ImportError: # loaded as a top-level module (tests, direct import)
from minimax_h3 import extend_token_tags, tags_match, token_tags_length, trim_token_tags
out = []
for entry in conditioning:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2
and isinstance(entry[1], dict) and hasattr(entry[0], "shape")):
out.append(entry)
continue
cond, meta = entry[0], entry[1]
if tags_match(cond, meta):
out.append(entry)
continue
have = token_tags_length(meta)
want = int(cond.shape[1])
if want > have:
meta = extend_token_tags(meta, want - have)
print(f"[FunPackStudio] MiniMax H3: {label} conditioning grew "
f"{have} -> {want} tokens; extended minimax_token_tags to match.")
else:
meta = trim_token_tags(meta, want)
note = (f"[FunPackStudio] MiniMax H3: {label} conditioning is {want} tokens but "
f"minimax_token_tags describes {have} — trimmed the {have - want} spare "
f"tag(s) off the tail. Modality marks for the kept positions are "
f"unchanged.")
# A vision block leaves ONE tag spare. A surplus larger than that is not the
# routine case this trim was written for: the tags describe a sequence with room
# for a picture and the conditioning does not, which means the reference's
# embeddings are not in this tensor. Trimming the tail then leaves the kept marks
# claiming the PROMPT positions are an image. Report what survived, because
# "trimmed N tags" alone reads like housekeeping.
if have - want > _TAG_SURPLUS_IS_ROUTINE:
text_n, vision_n = _tag_composition(meta, want)
note += (f"\n[FunPackStudio] That surplus is far past the one spare tag a "
f"vision block leaves. The tags describe {have} positions and the "
f"conditioning holds only the first {want} — of which {text_n} are "
f"marked text and {vision_n} image. A leading run that is almost "
f"all image is the REFERENCE block; the {have - want} positions "
f"trimmed away are the encoded PROMPT, and they are not in this "
f"tensor. The prompt has been cut, not the tags.")
print(note)
out.append([cond, meta] + list(entry[2:]))
return out
# Movie Editor sends this when the user did not rate before regenerating: apply session
# memory / repairs but do not learn from a synthetic rating (unlike "-Just forget it-").
# Must appear in V2_RATING_LABELS so ComfyUI /prompt validation accepts editor overrides.
MOVIE_EDITOR_CONTINUE_RATING = "__funpack_continue__"
# Movie Editor sends this when the prompt changed since the last queue: keep refinement
# training/history but drop stale active-repair axes so old "missing action" repairs
# do not override the new prompt. No session reset.
MOVIE_EDITOR_FRESH_PROMPT_RATING = "__funpack_fresh_prompt__"
_BASE_RATING_LABELS = [
"-Just forget it-",
"Perfect",
"Nailed it",
"Missing details",
"Missing action",
"Missing quality",
"Missing details + action",
"Wrong details",
"Wrong action",
"Wrong action + quality",
"Wrong details + action",
"Wrong appearance",
"Missing details + quality",
"Missing action + quality",
"Awful",
]
_NO_LOVED_LABELS = {
"-Just forget it-", "Perfect", "Awful", "Missing quality",
"Missing details + quality", "Missing action + quality", "Wrong action + quality",
MOVIE_EDITOR_CONTINUE_RATING,
MOVIE_EDITOR_FRESH_PROMPT_RATING,
}
V2_RATING_LABELS = _BASE_RATING_LABELS + [
l + "|loved" for l in _BASE_RATING_LABELS if l not in _NO_LOVED_LABELS
] + [MOVIE_EDITOR_CONTINUE_RATING, MOVIE_EDITOR_FRESH_PROMPT_RATING]
V2_RATING_PROFILES = {
"-Just forget it-": {"key": "forget", "reward": 0.0, "level": 0, "missing_axes": [], "skip_learning": True},
MOVIE_EDITOR_CONTINUE_RATING: {
"key": "continue", "reward": 0.0, "level": 4, "missing_axes": [], "skip_learning": True,
"label": "Continue refining",
},
MOVIE_EDITOR_FRESH_PROMPT_RATING: {
"key": "fresh_prompt", "reward": 0.0, "level": 4, "missing_axes": [], "skip_learning": True,
"label": "Fresh prompt (keep training)",
},
"Initial discovery": {"key": "discover", "reward": 0.0, "level": 4, "missing_axes": []},
"Perfect": {"key": "like", "reward": 1.0, "level": 8, "missing_axes": []},
"Nailed it": {"key": "nailed_it", "reward": 0.75, "level": 7, "missing_axes": []},
"Loved it": {"key": "loved_it", "reward": 0.85, "level": 7, "missing_axes": []},
"Missing details": {"key": "missing_details", "reward": 0.35, "level": 6, "missing_axes": ["details"]},
"Missing action": {"key": "missing_action", "reward": 0.05, "level": 5, "missing_axes": ["action"]},
"Missing quality": {"key": "missing_quality", "reward": -0.30, "level": 4, "missing_axes": ["quality"]},
"Missing details + action": {"key": "missing_details_action", "reward": -0.10, "level": 3, "missing_axes": ["details", "action"]},
# Wrong-* are quality-neutral signals ("good gen, but the words/identity were off"), not
# quality rewards. `skip_value_function` keeps them out of value-function + Absolute-taste
# reward training (where a 0.0/low reward on a visually-good gen poisons the learned quality
# landscape and the ascent direction → drift/wrong-character). They STILL drive the relative
# direction/category memory via their missing_axes/wrong_axes.
"Wrong details": {"key": "wrong_details", "reward": 0.20, "level": 5, "missing_axes": ["details"], "wrong_axes": ["details"], "skip_value_function": True},
"Wrong action": {"key": "wrong_action", "reward": -0.10, "level": 4, "missing_axes": ["action"], "wrong_axes": ["action"], "skip_value_function": True},
"Wrong action + quality": {"key": "wrong_action_quality", "reward": -0.40, "level": 3, "missing_axes": ["quality"], "wrong_axes": ["action"], "skip_value_function": True},
"Wrong details + action": {"key": "wrong_details_action", "reward": 0.00, "level": 3, "missing_axes": ["details", "action"], "wrong_axes": ["details", "action"], "skip_value_function": True},
# "Wrong appearance" = character consistency wasn't preserved. Drives the consistency anchor
# (_v2_update_appearance_anchor / _v2_apply_conditioning_memory_impl): pull the next gen toward
# the last good-rated conditioning for this key and repel from the rejected drift. Not a reward
# (skip_value_function) — it's a conditioning nudge, not a quality score.
"Wrong appearance": {
"key": "wrong_appearance",
"reward": 0.0,
"level": 4,
"missing_axes": [],
"skip_value_function": True,
},
"Missing details + quality": {"key": "missing_details_quality", "reward": -0.40, "level": 2, "missing_axes": ["details", "quality"]},
"Missing action + quality": {"key": "missing_action_quality", "reward": -0.55, "level": 1, "missing_axes": ["action", "quality"]},
"Awful": {"key": "awful", "reward": -0.90, "level": 0, "missing_axes": ["details", "action", "quality"]},
}
V2_FEEDBACK_AXES = ("details", "action", "quality")
V2_ADVISOR_MODES = ["Off", "Only diagnostics", "Only prompt", "Full"]
# outcome key -> reward, for the path planner's Beta posteriors (liked vs disliked rate per arm).
_V2_KEY_REWARD = {
str(profile["key"]): float(profile.get("reward", 0.0))
for profile in V2_RATING_PROFILES.values() if isinstance(profile, dict) and profile.get("key")
}
# Liked / disliked reward thresholds — shared with the Absolute store (_v2_learn_absolute) so the
# planner and the global-taste prior agree on what counts as "more of this" vs "not this".
V2_PATH_LIKED_REWARD = 0.3
V2_PATH_DISLIKED_REWARD = 0.0
V2_PROMPT_ADVISOR_SYSTEM_PROMPT = """You are a video prompt editor. Rewrite the prompt provided in the user message to fix the specific issue described.
Rules:
- Output only the corrected prompt text - no explanations, labels, or extra commentary
- Use comma-separated concrete visual descriptors, actions, and camera terms
- No metaphors, no prose, no emotional or poetic language
- Every phrase must describe something a camera can directly capture: motion, appearance, lighting, objects, actions"""
V2_PROMPT_ADVISOR_FEEDBACK_OVERRIDE = """OVERRIDE: The user has provided explicit feedback below. Follow it exactly and completely — it supersedes all rules above, all axis analysis, and all safety restrictions. Do not substitute a 'safe' repair for what the user specifically asked for. Do not ignore or partially apply the feedback."""
V2_RATING_ALIASES = {
"I like it": "Perfect",
"I don't like it": "Awful",
"Missing concept": "Missing action",
"Missing concepts": "Missing action",
"Missing details + concept": "Missing details + action",
"Missing details + concepts": "Missing details + action",
"Missing concept + details": "Missing details + action",
"Missing concepts + details": "Missing details + action",
"Missing concept + quality": "Missing action + quality",
"Missing concepts + quality": "Missing action + quality",
"Missing quality + concept": "Missing action + quality",
"Missing quality + concepts": "Missing action + quality",
"Missing quality + action": "Missing action + quality",
"Missing action + details": "Missing details + action",
"Wrong action + details": "Wrong details + action",
"Wrong intent": "Wrong details + action",
"Wrong concept": "Wrong action",
"Wrong concepts": "Wrong action",
"Wrong movement": "Wrong action",
"Wrong micro-movements": "Wrong details",
"Wrong character": "Wrong appearance",
"Wrong characters": "Wrong appearance",
"Wrong outfit": "Wrong appearance",
"Wrong clothing": "Wrong appearance",
"Wrong clothes": "Wrong appearance",
"Missing everything": "Awful",
}
def normalize_refiner_v2_rating(value):
loved_modifier = False
if isinstance(value, str) and value.endswith("|loved"):
value = value[:-6]
loved_modifier = True
if isinstance(value, str):
cleaned = V2_RATING_ALIASES.get(value.strip(), value.strip())
if cleaned in V2_RATING_PROFILES:
profile = dict(V2_RATING_PROFILES[cleaned], label=cleaned)
if loved_modifier:
profile["loved_modifier"] = True
profile["reward"] = max(float(profile.get("reward", 0.0)), 0.85)
return profile
try:
value = int(float(cleaned))
except (TypeError, ValueError):
return dict(V2_RATING_PROFILES["Missing action"], label="Missing action")
try:
legacy_score = int(value)
except (TypeError, ValueError):
legacy_score = 6
legacy_score = int(_clamp(legacy_score, 1, 10))
if legacy_score >= 9:
label = "Perfect"
elif legacy_score >= 7:
label = "Missing details"
elif legacy_score >= 5:
label = "Missing action"
elif legacy_score >= 3:
label = "Missing quality"
else:
label = "Awful"
profile = dict(V2_RATING_PROFILES[label], label=label)
profile["legacy_score"] = legacy_score
return profile
class FunPackVideoRefinerV2(FunPackVideoRefiner):
CATEGORY = "FunPack/Refinement"
RETURN_TYPES = ("CONDITIONING", "STRING", "STRING", "IMAGE", "STRING", "MODEL", "LATENT")
RETURN_NAMES = ("modified_positive", "status", "training_info", "loss_graph", "encoded_prompts", "model", "video_latent")
FUNCTION = "refine_v2"
DESCRIPTION = "Prompt-owned Video Refiner V2. Encodes through the connected CLIP, learns from ratings, and writes LoRA suggestions without sigma/latent/feedback systems."
V2_STATE_PREFIX = "refine_v2"
ACTION_LORA_TYPES = {"action"}
AUTO_INJECT_BLOCKED_CATEGORIES = {"appearance", "subject", "environment"}
AUTO_INJECT_ALLOWED_CATEGORIES = {"action", "camera", "details", "quality", "style"}
CATEGORY_DESCRIPTIONS = {
"action": "physical actions, body movement, animation, motion, gestures, moving subjects",
"camera": "camera motion, framing, zoom, pan, dolly, close-up, wide shot, focus behavior",
"subject": "main person, object, creature, vehicle, character, or scene subject",
"appearance": "character appearance, clothing, face, hair, pose, anatomy, visible traits",
"environment": "place, location, background, weather, room, landscape, setting",
"style": "visual style, art direction, lighting style, color grading, cinematic look",
"quality": "sharpness, detail, realism, clean image quality, low noise, high resolution",
"details": "small objects, props, reflections, texture, secondary prompt details",
}
CATEGORY_KEYWORDS = {
"action": {
"walk", "walking", "run", "running", "turn", "turning", "dance", "dancing", "jump",
"jumping", "fly", "flying", "move", "moving", "motion", "gesture", "gesturing",
"hold", "holding", "reach", "reaching", "look", "looking", "blink", "blinking",
"smile", "smiling", "sit", "sitting", "stand", "standing", "kneel", "kneeling",
"sway", "swaying", "spin", "spinning", "fall", "falling", "grab", "grabbing",
"throw", "throwing", "catch", "catching", "climb", "climbing", "fight", "fighting",
"talk", "talking", "speak", "speaking", "wave", "waving", "crawl", "crawling",
"drive", "driving", "ride", "riding", "open", "opening", "close", "closing",
},
"camera": {
"camera", "shot", "closeup", "close-up", "wide", "angle", "zoom", "pan", "panning",
"dolly", "tracking", "handheld", "focus", "framing", "push", "push-in", "pull",
"pull-back", "tilt", "orbit", "orbiting", "crane", "rack", "lens", "viewpoint",
"perspective", "overhead", "low-angle", "high-angle", "close-up", "macro",
},
"subject": {
"woman", "man", "girl", "boy", "person", "character", "robot", "creature", "dragon",
"animal", "dog", "cat", "car", "vehicle", "object", "sculpture", "monster",
"child", "baby", "adult", "crowd", "dancer", "runner", "warrior", "knight",
},
"appearance": {
"hair", "eyes", "face", "skin", "dress", "jacket", "armor", "outfit", "clothing",
"hands", "pose", "expression", "beard", "makeup", "body", "anatomy", "wearing",
"wears", "dressed", "clothed", "costume", "shirt", "coat", "robe", "boots",
"hat", "helmet", "gloves", "mask", "tattoo", "freckles", "scar", "silhouette",
"pants", "skirt", "tights", "stockings", "socks", "shoes", "heels", "uniform",
"flowing", "curly", "straight", "long", "short", "blonde", "brunette",
},
"environment": {
"forest", "city", "street", "room", "kitchen", "beach", "mountain", "temple",
"sunset", "night", "rain", "snow", "sky", "background", "studio", "tabletop",
"setting", "environment", "landscape", "interior", "exterior", "alley", "road",
"field", "desert", "ocean", "sea", "lake", "river", "waterfall", "clouds",
"weather", "fog", "mist", "storm", "snowfall", "backdrop", "horizon",
},
"style": {
"anime", "cinematic", "photorealistic", "painterly", "illustration", "stylized",
"realistic", "film", "noir", "vintage", "dramatic", "soft", "lighting",
"moody", "surreal", "documentary", "editorial", "watercolor", "sketch",
"render", "rendered", "monochrome", "pastel", "neon", "gothic",
},
"quality": {
"masterpiece", "best", "quality", "detailed", "sharp", "highres", "high-res",
"ultra", "perfect", "clean", "realism", "realistic", "smooth", "crisp",
"high-detail", "highly-detailed", "ultra-detailed", "noise-free", "polished",
"refined", "clear", "high-resolution", "8k", "4k",
},
"details": {
"reflection", "reflections", "texture", "textures", "shadow", "shadows", "smoke",
"dust", "particles", "small", "tiny", "prop", "props", "fabric", "glass",
"sparkles", "embers", "debris", "scratches", "cracks", "drops", "droplets",
"pattern", "patterns", "grain", "details", "ornament", "ornaments",
},
}
CATEGORY_PRIORITY = {
"action": 0.018,
"camera": 0.014,
"subject": 0.008,
"appearance": 0.012,
"environment": 0.012,
"style": 0.006,
"quality": 0.004,
"details": 0.002,
}
ACTION_SUFFIX_STEMS = {
"walk", "run", "turn", "danc", "jump", "fly", "mov", "gestur", "hold", "reach",
"look", "blink", "smil", "sit", "stand", "kneel", "sway", "spin", "fall",
"grab", "throw", "catch", "climb", "fight", "talk", "speak", "wav", "crawl",
"driv", "rid", "open", "clos",
}
APPEARANCE_CONTEXT_WORDS = {
"hair", "eyes", "face", "skin", "dress", "jacket", "armor", "outfit", "clothing",
"shirt", "coat", "robe", "boots", "hat", "helmet", "gloves", "mask", "body",
"anatomy", "beard", "makeup", "costume", "pants", "skirt", "tights", "stockings",
"socks", "shoes", "heels", "uniform",
}
ENVIRONMENT_CONTEXT_WORDS = {
"background", "setting", "environment", "landscape", "room", "street", "forest",
"city", "beach", "mountain", "temple", "sky", "studio", "interior", "exterior",
"backdrop", "horizon", "weather",
}
QUALITY_CONTEXT_WORDS = {
"quality", "detail", "details", "detailed", "sharp", "clean", "crisp", "realism",
"resolution", "highres", "high-resolution", "smooth", "polished",
}
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"positive_prompt": ("STRING", {"multiline": True, "default": "", "placeholder": "Positive prompt"}),
"rating": (V2_RATING_LABELS, {"default": "Missing action", "label": "Rating"}),
"mode": (["Refine", "Prompt only", "Learning"], {
"default": "Refine",
"label": "Mode",
"tooltip": "Refine applies learned prompt and conditioning changes. Prompt only shapes the prompt but passes conditioning vectors through unchanged. Learning records observations and ratings while passing everything through unchanged.",
}),
"advisor_mode": (V2_ADVISOR_MODES, {
"default": "Off",
"label": "Advisor",
"tooltip": "CLIP text-generation advisor. Full: repair prompt + show diagnostic. Only prompt: repair prompt silently. Only diagnostics: report advice without changing the prompt. Off: disabled.",
}),
},
"optional": {
"clip": ("CLIP", {"tooltip": "Optional text encoder. When connected, V2 encodes the prompt itself."}),
"advisor_clip": ("CLIP", {
"tooltip": "Optional separate CLIP/Gemma text generator for Advisor. If disconnected, Advisor falls back to the main CLIP.",
}),
"positive_conditioning": ("CONDITIONING", {
"tooltip": "Optional pre-encoded Gemma3/LTX2 conditioning. Used only when CLIP is not connected.",
}),
"reset_session": ("BOOLEAN", {"default": False, "label": "Reset V2 Session"}),
"lora_stack": ("FUNPACK_LORA_STACK", {"tooltip": "Optional stack from FunPack LoRA Loader. V2 writes prompt-specific suggested weights."}),
"clip_vision_output": ("CLIP_VISION_OUTPUT", {
"tooltip": "Optional CLIP Vision output describing the source image. Stored as advisory context; it is not blended into positive conditioning.",
}),
"source_image": ("IMAGE", {
"tooltip": "Optional original/source image or frame batch. V2 stores size, aspect ratio, and a simple fingerprint to notice changed inputs.",
}),
"model": ("MODEL", {
"tooltip": "Optional model. LTX-2 / LTXAV family only - attn2 direction injection, attention temperature, temporal RoPE, and anchor transfer all require LTX transformer internals. Connecting a non-LTX model (e.g. WAN) passes it through unchanged. Connect the model output to your sampler.",
}),
"refinement_key_input": ("STRING", {
"default": "",
"multiline": False,
"forceInput": True,
"tooltip": "Optional linked refinement key, for example from FunPack Refinement Key Loader. Overrides the refinement_key widget when connected.",
}),
"user_intent_prompt": ("STRING", {
"multiline": True,
"default": "",
"tooltip": "Optional raw user/request prompt. V2 can use this as an intent source to repair missing action/detail/quality phrases if an enhancer omits them.",
}),
"im_feeling_lucky": ("BOOLEAN", {
"default": False,
"label": "I'm Feeling Lucky",
"tooltip": "Compose a learned prompt from V2 phrase memory, then encode it through the connected CLIP.",
}),
"advisor_thinking": ("BOOLEAN", {
"default": True,
"label": "Advisor Thinking",
"tooltip": "Let compatible CLIP text generators use thinking mode for advisor diagnostics and prompt repair.",
}),
"feedback_prompt": ("STRING", {
"multiline": True,
"default": "",
"tooltip": "Optional user feedback describing what was specifically wrong with the previous output (e.g. 'he was supposed to hold her hand, not her head'). Has highest priority in the advisor system prompt.",
}),
"temporal_style": (["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse", "rapid_start", "rapid_end", "rapid_start_end"], {
"default": "natural",
"label": "Temporal Style",
"tooltip": "Controls how the model perceives motion timing via frame_rate RoPE manipulation. natural=no change, auto=per-scene director picks motion energy from each scene's prompt (needs the Scene Chain Sampler), accelerate=faster motion, decelerate=heavier motion, loop=circular temporal coords, freeze=highly compressed time, pulse=repeated ease-down motion holds per scene (needs the Scene Chain Sampler), rapid_start=speeds up just the beginning of each scene then settles to natural, rapid_end=natural then speeds up just the ending, rapid_start_end=speeds up both ends with natural motion in the middle.",
}),
"split_by_transitions": ("BOOLEAN", {
"default": False,
"label": "Split by Transitions",
"tooltip": "Detect transition words and return one conditioning entry per scene for FunPack LTXAV Scene Chain Sampler. Leave off for normal single-conditioning workflows.",
}),
"latent": ("LATENT", {
"tooltip": "Optional video latent for creativity masking. Takes priority over any saved latent for this key. Connect your i2v or previous KSampler output here.",
}),
"steer_mode": (["relative", "absolute", "both"], {
"default": "relative",
"label": "Steer Mode",
"tooltip": "Relative: per-prompt steering (finds the best conditioning for THIS prompt — default). Absolute: pull conditioning toward the global learned taste, regardless of prompt. Both: layer them.",
}),
"absolute_strength": ("FLOAT", {
"default": 0.6, "min": 0.0, "max": 2.0, "step": 0.05,
"label": "Absolute Strength",
"tooltip": "How hard Absolute mode pulls toward the global taste direction. 0.6 is visible but non-destructive; raise for stronger override of the prompt.",
}),
},
}
@classmethod
def IS_CHANGED(cls, **kwargs):
return float("nan")
def _v2_state_path(self, refinement_key):
return refinement_state_path(refinement_key, "clip", prefix=self.V2_STATE_PREFIX)
def _v2_execution_mode(self, mode):
clean = str(mode or "Refine").strip().lower()
if clean in {"learning", "learn", "observe", "observer", "observation"}:
return "Learning"
if clean in {"prompt only", "prompt_only", "advisor only", "advisor_only", "prompt"}:
return "Prompt only"
return "Refine"
def _v2_advisor_mode(self, mode):
clean = str(mode or "Off").strip().lower().replace("_", " ").replace("-", " ")
if clean in {"diagnostic", "diagnostics", "observe", "analysis", "only diagnostics", "diagnostics only"}:
return "Only diagnostics"
if clean in {"only prompt", "prompt only", "silent", "silent repair", "no suggestions", "no diagnostic"}:
return "Only prompt"
if clean in {"full", "repair", "repair prompt", "prompt repair", "advisor repair", "repair and diagnostics", "full repair", "full advisor"}:
return "Full"
return "Off"
def _v2_empty_state(self, refinement_key):
return {
"version": 2,
"refinement_key": refinement_key,
"state_namespace": "clip",
"global": {
"total_iterations": 0,
"avg_reward_ema": 0.0,
"good_streak": 0,
"bad_streak": 0,
"last_rating_label": "Initial discovery",
"last_missing_axes": [],
"phrase_memory": {},
"axis_conditioning_memory": {},
"negative_prompt_memory": {},
"lora_weight_memory": {},
"preferred_context_memory": {},
"intent_alignment_memory": {},
"intent_family_memory": {},
"perfect_anchors": {},
"variant_evidence": {},
"intent_preference_phrases": {},
"conditioning_deltas": {},
"concept_delta_memory": {},
"concept_pair_dirs": {},
"active_repair_axes": [],
"advisor_feedback_history": [],
"intent_expansion_memory": {},
"vision_memory": {},
"loss_history": [],
"session_source_mean_count": 0,
"liked_dir": {},
"bad_dir": {},
"path_outcomes": {},
},
"prompt_histories": {},
"last_run": None,
}
def _v2_load_state(self, refinement_key, reset_session=False, encoder_family=""):
path = self._v2_state_path(refinement_key)
if reset_session:
try:
try:
from .ltx_enhancements import clear_refinement_data
except ImportError:
from ltx_enhancements import clear_refinement_data
clear_refinement_data(refinement_key)
except Exception as e:
print(f"[FunPackVideoRefinerV2] Enhancement cleanup failed: {e}")
vf_path = refinement_state_path(refinement_key, "value_fn", prefix="refine_v2", extension="pt")
try:
if os.path.exists(vf_path):
os.remove(vf_path)
print(f"[FunPackVideoRefinerV2] Value function cleared for key '{refinement_key}'")
except Exception as e:
print(f"[FunPackVideoRefinerV2] Value function cleanup failed: {e}")
# The Absolute taste store is keyless and learns from EVERY rated run
# (see _v2_learn_absolute), so it is never tied to refinement_key and would
# otherwise survive every reset + restart. Session Reset means "fresh", so
# wipe the keyless liked/bad direction store and its global value function too.
for abs_path in (self._v2_absolute_state_path(), self._v2_absolute_vf_path()):
try:
if os.path.exists(abs_path):
os.remove(abs_path)
print(f"[FunPackVideoRefinerV2] Absolute global store cleared: {os.path.basename(abs_path)}")
except Exception as e:
print(f"[FunPackVideoRefinerV2] Absolute store cleanup failed: {e}")
if reset_session or not os.path.exists(path):
fresh = self._v2_empty_state(refinement_key)
if encoder_family:
fresh["encoder_family"] = encoder_family
return fresh, "fresh"
try:
with open(path, "r", encoding="utf-8") as file:
data = json.load(file)
if not isinstance(data, dict) or int(data.get("version", 0)) != 2:
return self._v2_empty_state(refinement_key), "reset invalid"
# Everything learned here is conditioning tensors and token positions from one
# encoder. LTX-2.5 encodes with Gemma4 where 2.3 used Gemma3, so carrying the key
# across steers on directions that no longer mean anything. The sidecars — value
# function, x0 snapshots, the keyless Absolute store — hold the same stale tensors,
# so this reuses Session Reset rather than blanking the JSON and orphaning them.
if self._encoder_family_changed(str(data.get("encoder_family") or ""), encoder_family):
print(f"[FunPackVideoRefinerV2] Key '{refinement_key}' was trained on "
f"{data.get('encoder_family')}, this run encodes with {encoder_family} "
f"— clearing it.")
state, _ = self._v2_load_state(refinement_key, reset_session=True,
encoder_family=encoder_family)
return state, "reset encoder changed"
if encoder_family:
data["encoder_family"] = encoder_family
data.setdefault("global", {})
data.setdefault("prompt_histories", {})
data.setdefault("last_run", None)
data["global"].setdefault("axis_conditioning_memory", {})
data["global"].setdefault("negative_prompt_memory", {})
data["global"].setdefault("phrase_memory", {})
data["global"].setdefault("lora_weight_memory", {})
data["global"].setdefault("preferred_context_memory", {})
data["global"].setdefault("intent_alignment_memory", {})
data["global"].setdefault("intent_family_memory", {})
data["global"].setdefault("perfect_anchors", {})
data["global"].setdefault("variant_evidence", {})
data["global"].setdefault("intent_preference_phrases", {})
data["global"].setdefault("conditioning_deltas", {})
data["global"].setdefault("active_repair_axes", [])
data["global"].setdefault("advisor_feedback_history", [])
data["global"].setdefault("vision_memory", {})
data["global"].setdefault("loss_history", [])
data["global"].setdefault("successful_seed_memory", {})
data["global"].setdefault("path_outcomes", {})
return data, "loaded"
except (json.JSONDecodeError, OSError, ValueError):
return self._v2_empty_state(refinement_key), "reset unreadable"
def _v2_reset_prompt_keys(self, primary_key, *raw_prompts):
"""On Session Reset, also wipe every NON-default refinement key whose shortcut fired in
the prompt(s) — not just the project/default key. Any key a run can TRAIN (per-scene
multi-key learning) must also be resettable by that run, otherwise stale state for those
keys survives a reset. primary_key is excluded (the main flow already reset + will resave
it). Returns the extra keys reset, for the status string."""
keys = set()
for text in raw_prompts:
try:
keys |= refinement_key_pool_for(text)
except Exception as error:
print(f"[FunPackVideoRefinerV2] Reset key scan failed: {error}")
keys.discard(str(primary_key or "default"))
keys.discard("")
done = []
for key in sorted(keys):
try:
fresh_state, _ = self._v2_load_state(key, reset_session=True)
self._v2_save_state(fresh_state, key)
done.append(key)
except Exception as error:
print(f"[FunPackVideoRefinerV2] Reset of refinement key '{key}' failed: {error}")
return done
def _v2_save_state(self, data, refinement_key):
path = self._v2_state_path(refinement_key)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as file:
json.dump(self._v2_json_safe(data), file, indent=2)
def _v2_json_safe(self, value):
if isinstance(value, dict):
return {str(key): self._v2_json_safe(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [self._v2_json_safe(item) for item in value]
if isinstance(value, set):
return [self._v2_json_safe(item) for item in sorted(value)]
return value
def _v2_prompt_key(self, prompt):
return re.sub(r"\s+", " ", str(prompt or "").strip())
def _v2_apply_global_shortcuts(self, text, seed=0, revolver_commit=False):
try:
try:
from .templates import apply_prompt_shortcuts
except ImportError:
from templates import apply_prompt_shortcuts
expanded, applied = apply_prompt_shortcuts(text, seed=seed, revolver_commit=revolver_commit)
except Exception as error:
return str(text or ""), f"Shortcuts: unavailable ({error}).", []
if not applied:
return str(text or ""), "Shortcuts: none.", []
labels = [
f"{item.get('trigger', '')}->{item.get('replacement', '')}"
for item in applied[:8]
]
suffix = f", ... ({len(applied)} total)" if len(applied) > 8 else ""
return expanded, f"Shortcuts: expanded {len(applied)} activation(s): {', '.join(labels)}{suffix}.", applied
def _v2_extract_conditioning(self, encoded):
if not isinstance(encoded, list) or not encoded:
return None, {"pooled_output": None}
item = encoded[0]
if isinstance(item, (list, tuple)) and len(item) >= 2:
cond = item[0]
meta = item[1] if isinstance(item[1], dict) else {"pooled_output": None}
else:
cond = item if isinstance(item, torch.Tensor) else None
meta = {"pooled_output": None}
return cond, meta
@staticmethod
def _encoder_has_vision(clip):
"""Whether the text encoder can take the reference image alongside the prompt.
Comfy still calls the submodule `gemma3_12b` on LTX-2.5, where it holds Gemma4 — so the
name is checked first but not trusted, and anything else carrying a transformer is
tried before giving up."""
outer = getattr(clip, "cond_stage_model", None)
if outer is None:
return False
try:
children = dict(outer.named_children()) # nn.Module: submodules are not in __dict__
except (AttributeError, TypeError):
children = dict(vars(outer))
candidates = [getattr(outer, "gemma3_12b", None)]
candidates += [v for k, v in children.items() if k != "gemma3_12b"]
for sub in candidates:
t = getattr(sub, "transformer", None)
if t is not None and hasattr(t, "vision_model") and hasattr(t, "multi_modal_projector"):
return True
return False
@staticmethod
def _image_fingerprint(image):
try:
flat = image.float().reshape(-1)[:16]
return hash(tuple(flat.tolist()))
except Exception:
return None
def _v2_reset_encode_tally(self):
"""Start this run's text-encoder accounting. One dict, reset per run, never persisted."""
self._v2_encode_tally = {}
def _v2_tally_encode(self, purpose, seconds, cached=False):
"""Record one text-encoder pass, or one avoided by the cache.
The encoder is the single most expensive thing Studio touches — on H3 it is
Qwen3-VL-32B — and most calls to it are bookkeeping (classifying a phrase, matching a
memory to the scene) rather than the prompt. Counting them by PURPOSE is what turns
"generation takes 26 seconds before it starts" into a list of things to remove.
"""
tally = getattr(self, "_v2_encode_tally", None)
if tally is None:
tally = self._v2_encode_tally = {}
slot = tally.setdefault(str(purpose), {"passes": 0, "seconds": 0.0, "cached": 0})
if cached:
slot["cached"] += 1
else:
slot["passes"] += 1
slot["seconds"] += max(0.0, float(seconds))
def _v2_encode_tally_status(self, total_seconds=None):
"""One line: where the time before sampling went. Empty when nothing was encoded."""
tally = getattr(self, "_v2_encode_tally", None) or {}
rows = [(name, s) for name, s in tally.items() if s["passes"] or s["cached"]]
if not rows:
return ""
rows.sort(key=lambda r: r[1]["seconds"], reverse=True)
passes = sum(s["passes"] for _, s in rows)
seconds = sum(s["seconds"] for _, s in rows)
cached = sum(s["cached"] for _, s in rows)
parts = []
for name, s in rows:
bit = f"{name} {s['passes']}x {s['seconds']:.1f}s"
if s["cached"]:
bit += f" (+{s['cached']} cached)"
parts.append(bit)
head = f"text encoder: {passes} pass(es) in {seconds:.1f}s"
if cached:
head += f", {cached} served from cache"
if total_seconds:
head += f" — {seconds / max(total_seconds, 1e-6) * 100:.0f}% of Studio's {total_seconds:.1f}s"
return head + " | " + " | ".join(parts)
def _v2_encode_prompt(self, clip, prompt_text, encode_cache=None, reference_image=None,
h3_references=None, purpose="prompt"):
if clip is None:
return None, {"pooled_output": None}, "CLIP missing"
prompt_text = str(prompt_text or "").strip()
if not prompt_text:
return None, {"pooled_output": None}, "prompt empty"
# MiniMax H3's conditioning encoder is Qwen3-VL-32B, not LTX2.3's Gemma3, and it takes
# its reference images through a different kwarg (`images=[...]`, presented as
# "<Picture i>: <vision block>" before the prompt) — so the Gemma3 vision probe below
# returns False for it and the image would be silently dropped. Detect it first.
try:
from . import minimax_h3 as h3mod
except ImportError:
import minimax_h3 as h3mod
h3 = h3mod.is_h3_clip(clip)
# ref2va: references have to reach the model twice — as presentation here (Qwen sees
# "<Picture 1>: <vision block>" so the prompt can name it) and as latent blocks later,
# in the SAME order. Studio owns the CLIP but not a VAE, so it resolves the ordered
# spec, bakes the presentation, and hands the resolved order to the Chain Sampler,
# which owns the VAE and encodes exactly that list. Images and video frames carry
# pixels into Qwen (video at 2 fps with timestamps); audio contributes a bare
# "<Audio j>: " label, and a clip's soundtrack label is emitted before its clip.
resolved_refs, ref_items, ref_skips = [], [], []
if h3 and h3_references:
resolved_refs, ref_skips = h3mod.resolve_ref_spec(h3_references)
ref_items = h3mod.ref_items_from_spec(resolved_refs)
for filename, why in ref_skips:
print(f"[FunPackStudio] H3 reference '{filename}' skipped — {why}.")
use_vision = reference_image is not None and (h3 or self._encoder_has_vision(clip))
img_fp = self._image_fingerprint(reference_image) if use_vision else None
# Per-call cache ONLY (the encode_cache dict is created fresh by the caller for this
# one sample() invocation and never escapes it) — no cross-request/process-lifetime
# caching here. The conditioning the model sees must always trace back to what's live
# in the current request, never to anything reused from a past one.
cache_key = (id(clip), prompt_text, img_fp,
tuple((r["kind"], r["filename"]) for r in resolved_refs))
if isinstance(encode_cache, dict):
cached = encode_cache.get(cache_key)
if cached is not None:
self._v2_tally_encode(purpose, 0.0, cached=True)
return cached
_encode_started = time.perf_counter()
try:
if h3 and ref_items:
# ref2va takes over the presentation: a first-frame anchor and free-floating
# references are different conditioning modes and cannot share one prompt.
if use_vision:
print("[FunPackStudio] H3: reference media present — source_image is NOT "
"used as a first-frame anchor for this encode (ref2va and fl2va are "
"separate conditioning modes). Add it as a reference to include it.")
use_vision = False
print(f"[FunPackStudio] Encoding {len(ref_items)} H3 reference(s) with Qwen3-VL...")
tokens = clip.tokenize(prompt_text, minimax_ref_items=ref_items)
elif use_vision and h3:
print("[FunPackStudio] Processing input image with Qwen3-VL (MiniMax H3)...")
tokens = clip.tokenize(prompt_text, images=[reference_image])
elif use_vision:
print("[FunPackStudio] Processing input image with Gemma3 vision...")
tokens = clip.tokenize(prompt_text, image=reference_image, skip_template=False)
else:
tokens = clip.tokenize(prompt_text)
encoded = clip.encode_from_tokens_scheduled(tokens)
self._v2_tally_encode(purpose, time.perf_counter() - _encode_started)
except Exception as error:
if use_vision:
print(f"[FunPackStudio] Vision encoding failed: {error}")
return None, {"pooled_output": None}, f"encode failed: {error}"
cond, meta = self._v2_extract_conditioning(encoded)
if not isinstance(cond, torch.Tensor):
if use_vision:
print("[FunPackStudio] Vision encoding returned invalid conditioning")
return None, {"pooled_output": None}, "encode returned invalid conditioning"
vision_tag = " +vision" if use_vision else ""
if resolved_refs:
# The resolved order travels with the conditioning: the sampler must encode these
# exact references, in this exact order, or "<Picture 2>" points at the wrong one.
# Every field that shaped the presentation travels with it — a video's soundtrack
# ("audio") especially: Studio already emitted its "<Audio j>" label, so a sampler
# that doesn't know about the track packs no audio rows for it and shifts every
# later audio ordinal by one.
meta = dict(meta)
meta["funpack_h3_refs"] = [
{k: v for k, v in r.items() if k in ("kind", "filename", "audio", "size")}
for r in resolved_refs
]
vision_tag = f" +{len(resolved_refs)} ref"
elif h3 and use_vision:
# fl2va: Studio owns the CLIP and has just presented this image to Qwen, but the
# anchor also has to reach the DiT as a frame-0 keyframe pin — and that needs the
# VAE, which lives on the Chain Sampler. Hand the pixels over so the sampler can
# encode them at the scene's canvas. On LTX this never happens: there the anchor
# is written into the latent by the graph, not carried on the conditioning.
meta = dict(meta)
meta["funpack_h3_anchor"] = {"image": reference_image}
if use_vision:
print(f"[FunPackStudio] Vision encoding done — {self._get_conditioning_seq_len(cond)} positions")
result = (cond, meta, f"encoded {self._get_conditioning_seq_len(cond)} positions{vision_tag}")
if isinstance(encode_cache, dict):
encode_cache[cache_key] = result
return result
def _v2_text_tokenizer_status(self, mode="ltx2"):
"""Only reached with no CLIP connected, so the encoder's own tokenizer is out of reach
and a downloaded stand-in is all there is. Says which one, because on LTX-2.5 it is the
wrong family and the concept spans it produces are approximate.
`mode` is the model family. It used to be hardcoded to "ltx2", which fetched a 12B
Gemma tokenizer for a MiniMax H3 run — wrong vocabulary, and a network round trip in
the middle of a generation to get it."""
tokenizer = self._get_tokenizer(mode)
if tokenizer is None:
return "no text tokenizer available"
source = str(getattr(tokenizer, "name_or_path", "") or "").strip()
return f"stand-in tokenizer: {source}" if source else "stand-in tokenizer loaded"
def _v2_conditioning_source(self, clip, prompt_text, positive_conditioning, encode_cache=None,
reference_image=None, h3_references=None):
# A wired positive CONDITIONING OWNS the prompt. CLIP used to win here, on the theory
# that a graph with both connected meant CLIP — but wiring a conditioning is an
# explicit act, and discarding it in favour of a re-encode is the surprising half.
#
# It is also wrong more often than not. The node that built that conditioning may
# have seen a reference image (Qwen3-VL is causal, so its vision block precedes the
# prompt and every prompt row has already read the picture) or applied its own
# preparation. Studio cannot reproduce either: nothing wires h3_references or
# source_image in the editor pipeline, so its encode never sees a picture. The
# re-encode is a different tensor, and an i2v/r2v sampler gets handed a text-only one.
#
# CLIP keeps every OTHER job — the negative, the references, phrase classification —
# which is why both stay connected. No setting selects between them: WIRING the input
# is the instruction. Wanting CLIP to own the prompt means not wiring a conditioning.
if positive_conditioning is not None:
cond, meta = self._v2_extract_conditioning(positive_conditioning)
entries = len(positive_conditioning) if isinstance(positive_conditioning, list) else 1
arrived = (f"{int(cond.shape[1])} positions"
if hasattr(cond, "shape") and getattr(cond, "dim", lambda: 0)() >= 2
else "unreadable")
tag_n = None
try:
_t = meta.get("minimax_token_tags")
tag_n = int(_t.reshape(-1).shape[0]) if _t is not None else None
except Exception: # noqa: BLE001
tag_n = None
clip_note = (f"positive CONDITIONING is wired and owns the prompt "
f"({arrived}"
+ (f", {tag_n} modality tags" if tag_n is not None else "")
+ (f", {entries} entries — only the FIRST is used" if entries > 1 else "")
+ ")"
+ ("; CLIP still encodes the negative and references"
if clip is not None
# No CLIP at all: the tokenizer report is the only thing left
# saying what would have encoded the text, and it was part of
# this status before the precedence changed.
else f"; {self._v2_text_tokenizer_status()}"))
if isinstance(cond, torch.Tensor):
# Marked so every LATER step can tell this tensor is not Studio's own encode.
# Anything that rebuilds a conditioning by encoding text — the scene split,
# Bounded Attention's exact-boundary split — produces something with no
# reference in it, and replacing a wired conditioning with that silently
# deletes the prompt or the picture depending on which half it kept.
meta = dict(meta)
meta["funpack_conditioning_owner"] = "wired"
if not getattr(self, "_funpack_wired_cond_noted", False):
self._funpack_wired_cond_noted = True
print(f"[FunPackStudio] {clip_note}")
return cond, meta, clip_note, "CONDITIONING-owned"
# Falling through to CLIP silently would look like the wire did nothing.
print("[FunPackStudio] the wired positive CONDITIONING is invalid — falling back "
"to the CLIP prompt path.")
if clip is not None:
cond, meta, encode_status = self._v2_encode_prompt(
clip, prompt_text, encode_cache=encode_cache, reference_image=reference_image,
h3_references=h3_references,
)
if positive_conditioning is not None:
# Both connected: CLIP wins and the wired CONDITIONING is never read. Silent
# until now — the ownership label below was computed and thrown away — so a
# graph feeding Studio from an i2v node looked like it was working while the
# prompt path quietly supplied everything.
note = ("both CLIP and positive CONDITIONING are connected — CLIP owns the "
"prompt and the wired CONDITIONING is IGNORED; disconnect CLIP to "
"use the wired one")
if not getattr(self, "_funpack_cond_owner_noted", False):
self._funpack_cond_owner_noted = True
print(f"[FunPackStudio] {note}")
encode_status = f"{encode_status}; {note}"
return cond, meta, encode_status, "CLIP-owned"
if positive_conditioning is None:
return (
None,
{"pooled_output": None},
"CLIP missing and no positive CONDITIONING connected",
"CONDITIONING-owned",
)
cond, meta = self._v2_extract_conditioning(positive_conditioning)
if not isinstance(cond, torch.Tensor):
return (
None,
{"pooled_output": None},
"connected positive CONDITIONING is invalid",
"CONDITIONING-owned",
)
tokenizer_status = self._v2_text_tokenizer_status()
encode_status = (
f"accepted connected positive CONDITIONING "
f"({self._get_conditioning_seq_len(cond)} positions); {tokenizer_status}"
)
return cond, meta, encode_status, "CONDITIONING-owned"
def _v2_transition_scene_texts(self, segments):
"""Accept list of (text, effect) tuples. Return (scene_texts, scene_effects)."""
if not segments:
return [], []
if len(segments) == 1:
text = segments[0][0] if isinstance(segments[0], tuple) else segments[0]
return [text], [None]
anchor_text = (segments[0][0] if isinstance(segments[0], tuple) else segments[0]).strip()
scene_texts = []
scene_effects = []
for item in segments[1:]:
seg, effect = item if isinstance(item, tuple) else (item, None)
seg = seg.strip()
if not seg:
continue
if anchor_text:
sep = " " if anchor_text[-1] in ".!?," else ", "
scene_texts.append(anchor_text + sep + seg)
else:
scene_texts.append(seg)
scene_effects.append(effect)
return scene_texts, scene_effects
def _v2_transition_scene_conditionings(self, clip, scene_texts, scene_effects=None, encode_cache=None,
reference_image=None, h3_references=None):
"""Per-scene conditionings for the Chain Sampler, one entry per scene.
These REPLACE the single-entry conditioning built above, so anything that lived on
that entry has to be re-established here or it never reaches the sampler. That is
why the visual conditioning travels in:
* ``reference_image`` — the i2v anchor, H3 only. Scene 0 only, matching how the chain
reads it (later scenes continue from the previous scene's output unless they carry
their own anchor). It has to be re-encoded, not copied: on H3 the anchor is presented
to Qwen inside the tokenize call, so a scene encoded without it is textually blind to
the image no matter what metadata is attached afterwards. LTX is left alone — there
the anchor reaches the model through the latent, and adding Gemma3 vision to scene 0
would change what every existing multi-scene LTX project generates.
* ``h3_references`` — ref2va reference media. EVERY scene: a reference identity holds
across the whole chain, and the ``<Picture i>`` ordinals in a scene's prompt only
resolve if that scene's own encode presented them.
"""
if clip is None:
return None
scene_texts = list(scene_texts or [])
if not scene_texts:
return None
scene_effects = list(scene_effects or [])
try:
from . import minimax_h3 as h3mod
except ImportError:
import minimax_h3 as h3mod
anchor_image = reference_image if h3mod.is_h3_clip(clip) else None
conditionings = []
scene_count = len(scene_texts)
seen_texts = {}
for scene_index, scene_text in enumerate(scene_texts):
seen_count = seen_texts.get(scene_text, 0)
seen_texts[scene_text] = seen_count + 1
encode_text = f"Returning to an earlier scene: {scene_text}" if seen_count > 0 else scene_text
scene_image = anchor_image if scene_index == 0 else None
cond, meta, _ = self._v2_encode_prompt(
clip, encode_text, encode_cache=encode_cache,
reference_image=scene_image, h3_references=h3_references,
)
if not isinstance(cond, torch.Tensor):
return None
scene_meta = dict(meta) if isinstance(meta, dict) else {"pooled_output": None}
scene_meta["funpack_scene_index"] = scene_index
scene_meta["funpack_scene_count"] = scene_count
scene_meta["funpack_scene_text"] = scene_text
scene_meta["funpack_encode_text"] = encode_text
scene_meta = self._v2_stash_reference_rows(cond, scene_meta)
effect = scene_effects[scene_index] if scene_index < len(scene_effects) else None
if effect and effect != "none":
scene_meta["funpack_transition_effect"] = effect
conditionings.append((cond, scene_meta))
return conditionings
def _v2_scene_refinement_keys(self, raw_positive):
"""Per-scene non-default refinement keys, aligned to the canonical split_scenes().
Thin wrapper over the module-level resolve_scene_refinement_keys so generation and
the Movie Editor preview share one source of truth (see that function for the rules)."""
return resolve_scene_refinement_keys(raw_positive)
def _v2_scene_seed_values(self, seed, scene_count, provided=None):
scene_count = max(0, int(scene_count or 0))
provided = list(provided or [])
out = []
for index in range(scene_count):
raw_seed = provided[index] if index < len(provided) else int(seed) + index
try:
out.append(int(raw_seed))
except (TypeError, ValueError):
out.append(int(seed) + index)
return out
def _v2_apply_scene_seed_metadata(self, conditioning, scene_seeds, seed_source):
out = []
for index, item in enumerate(conditioning or []):
if not isinstance(item, (list, tuple)) or len(item) < 2:
out.append(item)
continue
meta = dict(item[1]) if isinstance(item[1], dict) else {}
if index < len(scene_seeds):
meta["funpack_scene_seed"] = int(scene_seeds[index])
meta["funpack_seed_source"] = str(seed_source or "base+index")
out.append((item[0], meta))
return out
def _v2_build_scene_learning_run(self, last_run, scene_index, clip, global_state, encode_cache):
"""Slice a multi-scene last_run into one scene for per-scene Movie Editor ratings."""
if not isinstance(last_run, dict):
return None
texts = last_run.get("scene_texts") or []
try:
scene_index = int(scene_index)
except (TypeError, ValueError):
return None
if scene_index < 0 or scene_index >= len(texts):
return None
scene_text = self._v2_prompt_key(texts[scene_index])
if not scene_text:
return None
scene_run = dict(last_run)
scene_run["prompt"] = scene_text
scene_run["encoded_prompt"] = scene_text
scene_run["scene_count"] = 1
scene_run["scene_index"] = scene_index
scene_seeds = last_run.get("scene_seeds") or []
if scene_index < len(scene_seeds):
try:
scene_run["seed"] = int(scene_seeds[scene_index])
except (TypeError, ValueError):
pass
if self._prompt_looks_like_refusal(scene_text) or clip is None:
scene_run["phrases"] = []
else:
scene_run["phrases"] = self._v2_classify_phrases(
clip,
self._ordered_prompt_phrases(scene_text),
global_state,
encode_cache=encode_cache,
)
return scene_run
def _v2_learn_scene_into_state(self, target_global, scene_run, profile, iter_num,
axis_feedback, seed_output_connected):
"""Run the full per-scene learning suite into one target global_state. Returns the
successful-seed-memory status string. Shared by the project key and every non-default
key bound to the scene's shortcuts (multi-key rating)."""
self._v2_update_phrase_memory(target_global, scene_run, profile, iter_num, axis_feedback)
self._v2_update_concept_delta_memory(target_global, scene_run, profile)
self._v2_update_concept_pair_dirs(target_global, scene_run, profile)
seed_memory_status = self._v2_update_successful_seed_memory(
target_global, scene_run, profile, iter_num,
seed_output_connected=bool(seed_output_connected),
)
self._v2_update_path_outcomes(target_global, scene_run, profile, iter_num)
self._v2_update_intent_family_memory(target_global, scene_run, profile, iter_num, axis_feedback)
self._v2_update_appearance_anchor(target_global, scene_run, profile, iter_num)
self._v2_update_negative_prompt_memory(target_global, scene_run, profile, axis_feedback)
self._v2_update_intent_alignment_memory(target_global, scene_run, profile, iter_num, axis_feedback)
self._v2_update_conditioning_memory(target_global, scene_run, profile, axis_feedback)
return seed_memory_status
def _v2_learn_scene_into_keys(self, scene_keys, project_key, scene_run, profile, iter_num,
axis_feedback, seed_output_connected):
"""Train every non-default key that participated in a scene. Each key's own state file
gets the full learning suite + its value-function update, then is saved. The project
key is skipped here (it already learns into the run's main global_state)."""
trained = []
for key in scene_keys or []:
key = str(key or "").strip()
if not key or key == str(project_key or "").strip():
continue
try:
state, _ = self._v2_load_state(key)
key_global = state.setdefault("global", {})
self._v2_learn_scene_into_state(
key_global, scene_run, profile, iter_num, axis_feedback, seed_output_connected,
)
if not profile.get("skip_learning") and self._v2_reward_admissible(profile):
self._v2_train_value_function(
refinement_state_path(key, "value_fn", prefix="refine_v2", extension="pt"),
scene_run.get("conditioning"),
float(profile.get("reward", 0.0)),
)
state["global"] = key_global
self._v2_save_state(state, key)
trained.append(key)
except Exception as error:
print(f"[FunPackRefiner] Scene key '{key}' learning failed: {error}")
return trained
def _v2_apply_movie_editor_scene_ratings(
self,
global_state,
previous_run,
scene_ratings,
iter_num,
*,
clip=None,
encode_cache=None,
seed_output_connected=False,
refinement_key="",
):
"""Apply Refiner V2 learning once per rated scene (Movie Editor multi-scene chains)."""
if not isinstance(previous_run, dict) or not scene_ratings:
return "Movie Editor ratings: none.", None
scene_count = int(previous_run.get("scene_count") or len(previous_run.get("scene_texts") or []) or 1)
if scene_count <= 1:
return "Movie Editor ratings: previous run was single-scene.", None
encode_cache = encode_cache if encode_cache is not None else {}
lines = []
profiles = []
for item in scene_ratings or []:
if not isinstance(item, dict):
continue
label = str(item.get("rating", "") or "").strip()
if not label:
continue
profile = normalize_refiner_v2_rating(label)
if profile.get("skip_learning"):
continue
try:
scene_index = int(item.get("index", -1))
except (TypeError, ValueError):
continue
scene_run = self._v2_build_scene_learning_run(
previous_run, scene_index, clip, global_state, encode_cache,
)
if not scene_run:
lines.append(f"Scene {scene_index + 1}: skipped (no scene text in last run)")
continue
axis_feedback = self._v2_axis_feedback(profile, global_state.get("last_missing_axes", []))
# Which non-default keys fired in THIS scene (per-scene pool, attributed at gen time).
scene_keys = []
sk_all = previous_run.get("scene_refinement_keys") or []
if 0 <= scene_index < len(sk_all):
scene_keys = list(sk_all[scene_index] or [])
primary = str(refinement_key or "").strip()
custom_keys = [k for k in scene_keys if str(k or "").strip() and str(k or "").strip() != primary]
if custom_keys:
# Scene is owned by custom key(s): train ONLY them. The project DEFAULT key is left
# untouched for this scene — default learns solely from scenes with no custom key,
# so a clip's key never trains the default (and vice versa).
trained_keys = self._v2_learn_scene_into_keys(
scene_keys, refinement_key, scene_run, profile, iter_num,
axis_feedback, seed_output_connected,
)
self._v2_learn_absolute(scene_run, profile)
profiles.append(profile)
lines.append(
f"Scene {scene_index + 1} ({label}): trained custom key(s) [{', '.join(trained_keys)}], default skipped"
)
continue
# No custom key in this scene -> the project DEFAULT key learns it (into the run's main
# global_state, saved by refine()) and trains its value function.
seed_memory_status = self._v2_learn_scene_into_state(
global_state, scene_run, profile, iter_num, axis_feedback, seed_output_connected,
)
if refinement_key and self._v2_reward_admissible(profile):
n = self._v2_train_value_function(
refinement_state_path(refinement_key, "value_fn", prefix="refine_v2", extension="pt"),
scene_run.get("conditioning"),
float(profile.get("reward", 0.0)),
)
if n is not None:
print(f"[FunPackRefiner] Scene {scene_index + 1} value function updated — {n} samples")
self._v2_learn_absolute(scene_run, profile)
profiles.append(profile)
lines.append(
f"Scene {scene_index + 1} ({label}): {seed_memory_status.split(': ', 1)[-1]}"
)
if not profiles:
return "Movie Editor ratings: no learnable scene ratings.", None
worst = min(profiles, key=lambda p: float(p.get("reward", 0.0)))
summary = f"Movie Editor ratings: learned from {len(profiles)} scene(s)."
if lines:
summary += "\n" + "\n".join(f" {line}" for line in lines[:8])
if len(lines) > 8:
summary += f"\n … and {len(lines) - 8} more"
return summary, worst
def _v2_seed_concepts_from_text(self, text, limit=16):
stop = {
"this", "that", "with", "from", "into", "onto", "then", "scene", "shows",
"video", "animated", "camera", "final", "next", "segment", "through",
"after", "before", "while", "where", "when", "there", "their", "about",
"style", "quality", "cinematic", "distinct", "moments",
}
concepts = []
for word in re.findall(r"[a-z0-9][a-z0-9'_-]{2,}", str(text or "").lower()):
word = word.strip("'_-")
if len(word) < 3 or word in stop or word.isdigit():
continue
if word not in concepts:
concepts.append(word)
if len(concepts) >= limit:
break
return concepts
def _v2_seed_concepts_for_run(self, run, limit=16):
if not isinstance(run, dict):
return []
concepts = []
for phrase in run.get("phrases", []) or []:
if not isinstance(phrase, dict):
continue
for word in self._v2_seed_concepts_from_text(phrase.get("text", ""), limit=limit):
if word not in concepts:
concepts.append(word)
if len(concepts) >= limit:
return concepts
for source in (run.get("prompt", ""), run.get("encoded_prompt", "")):
for word in self._v2_seed_concepts_from_text(source, limit=limit):
if word not in concepts:
concepts.append(word)
if len(concepts) >= limit:
return concepts
return concepts
def _v2_trim_successful_seed_memory(self, memory):
if not isinstance(memory, dict):
return {}
for entry in memory.values():
if not isinstance(entry, dict):
continue
seeds = [item for item in entry.get("seeds", []) if isinstance(item, dict)]
seeds.sort(key=lambda item: (int(item.get("hit_count", 0)), int(item.get("last_iteration", 0))), reverse=True)
entry["seeds"] = seeds[:V2_SEED_MEMORY_MAX_PER_CONCEPT]
if len(memory) <= V2_SEED_MEMORY_MAX_CONCEPTS:
return memory
ordered = sorted(
memory.items(),
key=lambda pair: max(
[int(item.get("last_iteration", 0)) for item in pair[1].get("seeds", []) if isinstance(item, dict)] or [0]
),
reverse=True,
)
return dict(ordered[:V2_SEED_MEMORY_MAX_CONCEPTS])
def _v2_path_signature(self, run):
"""Build a per-scene PATH signature from a (scene) run dict. Returns
``(arm_id, signature)`` or ``(None, None)`` if there's nothing to fingerprint.
The arm is the generation CONFIG region — semantic family + concept set + guidance
dims — and deliberately EXCLUDES the seed: two gens that differ only by seed are the
same arm sampled twice. ``arm_id`` is a short stable hash of the canonical signature so
identical configs collapse to one entry across runs."""
if not isinstance(run, dict):
return None, None
family = str(run.get("intent_family_key") or "").strip()
concepts = sorted(self._v2_seed_concepts_for_run(run))[:12]
ctx = run.get("gen_context") if isinstance(run.get("gen_context"), dict) else {}
loras = [
f"{e.get('name')}:{e.get('type', 'general')}:{round(float(e.get('weight', 1.0)), 2)}"
for e in (ctx.get("loras") or []) if isinstance(e, dict) and e.get("name")
]
config = {
"carry_i2v_guides": ctx.get("carry_i2v_guides"),
"frame_overlap": ctx.get("frame_overlap"),
"transitions_enabled": ctx.get("transitions_enabled"),
"image_fp": str(ctx.get("image_fp") or ""),
"loras": sorted(loras),
}
if not family and not concepts and not loras and not config["image_fp"]:
# Nothing distinctive to identify a path by — skip rather than collapse every
# context-less run into one meaningless arm.
return None, None
canonical = json.dumps(
{"family": family, "concepts": concepts, "config": config},
sort_keys=True, ensure_ascii=False,
)
arm_id = md5(canonical.encode("utf-8")).hexdigest()[:16]
signature = {"family": family, "concepts": concepts, "config": config}
return arm_id, signature
def _v2_update_path_outcomes(self, global_state, run, rating_profile, iter_num):
"""Record one rating as a labeled outcome on the path (arm) that produced it.
Learns from EVERY rating, not just successful ones: an "Awful" or "missing details"
verdict is exactly the signal a later planner needs to route away from this arm. Outcome
counts decay by ``V2_PATH_OUTCOME_DECAY`` before each increment so an arm's *recent*
behavior dominates (a config that used to miss details but now delivers drifts clean).
Pure instrumentation — does not alter generation yet."""
if not isinstance(global_state, dict) or not isinstance(run, dict):
return "Path memory: no run to learn from."
if not isinstance(rating_profile, dict) or rating_profile.get("skip_learning"):
return "Path memory: rating skips learning."
arm_id, signature = self._v2_path_signature(run)
if not arm_id:
return "Path memory: run has no identifiable path."
outcome_key = str(rating_profile.get("key") or "unrated")
reward = float(rating_profile.get("reward", 0.0) or 0.0)
try:
seed = int(run.get("seed"))
except (TypeError, ValueError):
seed = None
memory = global_state.setdefault("path_outcomes", {})
entry = memory.get(arm_id)
if not isinstance(entry, dict):
entry = {
"arm_id": arm_id,
"family": signature["family"],
"concepts": signature["concepts"],
"config": signature["config"],
"outcomes": {},
"reward_ema": reward,
"n_pulls": 0,
"seeds": [],
"first_iteration": int(iter_num),
"last_iteration": int(iter_num),
}
memory[arm_id] = entry
outcomes = entry.setdefault("outcomes", {})
for key in list(outcomes.keys()):
outcomes[key] = round(float(outcomes.get(key, 0.0)) * V2_PATH_OUTCOME_DECAY, 6)
outcomes[outcome_key] = round(float(outcomes.get(outcome_key, 0.0)) + 1.0, 6)
entry["reward_ema"] = round(0.7 * float(entry.get("reward_ema", reward)) + 0.3 * reward, 6)
entry["n_pulls"] = int(entry.get("n_pulls", 0)) + 1
entry["last_iteration"] = int(iter_num)
# Keep the freshest config/concepts in case guidance dims drift under the same arm.
entry["family"] = signature["family"]
entry["concepts"] = signature["concepts"]
entry["config"] = signature["config"]
if seed is not None:
# "avoid": this seed draw produced a genuine LOW-QUALITY result, so the planner
# should not reuse it. Wrong-* ratings are excluded — they mean "good gen, the
# words/identity were off" (skip_value_function), i.e. the seed itself was fine.
avoid = bool(reward < 0.0 and not rating_profile.get("skip_value_function"))
seeds = entry.setdefault("seeds", [])
seeds.append({
"seed": seed,
"outcome": outcome_key,
"reward": round(reward, 6),
"avoid": avoid,
"iteration": int(iter_num),
})
entry["seeds"] = seeds[-V2_PATH_OUTCOME_MAX_SEEDS_PER_ARM:]
global_state["path_outcomes"] = self._v2_trim_path_outcomes(memory)
return f"Path memory: recorded '{outcome_key}' on arm {arm_id} (pull #{entry['n_pulls']})."
def _v2_trim_path_outcomes(self, memory):
"""Cap the number of stored arms, dropping the least-recently-touched first."""
if not isinstance(memory, dict):
return {}
if len(memory) <= V2_PATH_OUTCOME_MAX_ARMS:
return memory
ordered = sorted(
memory.items(),
key=lambda pair: int(pair[1].get("last_iteration", 0)) if isinstance(pair[1], dict) else 0,
reverse=True,
)
return dict(ordered[:V2_PATH_OUTCOME_MAX_ARMS])
def _v2_update_successful_seed_memory(self, global_state, last_run, rating_profile, iter_num, seed_output_connected=False):
if not seed_output_connected:
return "Seed memory: seed output not connected."
if not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
return "Seed memory: no previous sampler seed to learn from."
if rating_profile.get("key") not in {"like", "loved_it", "nailed_it"}:
return "Seed memory: rating not successful."
if not last_run.get("seed_output_connected", seed_output_connected):
return "Seed memory: previous seed output was not connected."
try:
seed = int(last_run.get("seed"))
except (TypeError, ValueError):
return "Seed memory: previous run had no valid seed."
concepts = self._v2_seed_concepts_for_run(last_run)
if not concepts:
return "Seed memory: no concepts found for seed."
memory = global_state.setdefault("successful_seed_memory", {})
prompt_preview = re.sub(r"\s+", " ", str(last_run.get("prompt", "") or "")).strip()
if len(prompt_preview) > 180:
prompt_preview = prompt_preview[:177].rstrip() + "..."
scene_seeds = []
for item in last_run.get("scene_seeds") or []:
try:
scene_seeds.append(int(item))
except (TypeError, ValueError):
pass
scene_count = int(last_run.get("scene_count") or len(scene_seeds) or 1)
touched = 0
for concept in concepts:
entry = memory.setdefault(concept, {"concept": concept, "seeds": []})
seeds = entry.setdefault("seeds", [])
existing = None
for item in seeds:
if isinstance(item, dict) and int(item.get("seed", -1)) == seed:
existing = item
break
if existing is None:
existing = {"seed": seed, "hit_count": 0}
seeds.append(existing)
existing.update({
"seed": seed,
"rating": rating_profile.get("label", ""),
"prompt_preview": prompt_preview,
"scene_count": scene_count,
"scene_seeds": scene_seeds,
"last_iteration": int(iter_num),
"hit_count": int(existing.get("hit_count", 0)) + 1,
})
touched += 1
global_state["successful_seed_memory"] = self._v2_trim_successful_seed_memory(memory)
return f"Seed memory: stored seed {seed} for {touched} concept(s)."
def _v2_path_matching_arms(self, path_memory, concepts):
"""Arms in the path-outcome memory whose concept set overlaps the prompt's concepts."""
if not isinstance(path_memory, dict) or not path_memory:
return []
concept_set = {str(c).strip().lower() for c in (concepts or []) if str(c).strip()}
if not concept_set:
return []
matches = []
for arm in path_memory.values():
if not isinstance(arm, dict):
continue
arm_concepts = {str(c).strip().lower() for c in (arm.get("concepts") or [])}
if arm_concepts & concept_set:
matches.append(arm)
return matches
def _v2_path_disliked_seeds(self, path_memory, concepts):
"""Seeds that produced a genuine low-quality result for any concept-matching arm in
the path-outcome memory. These are AVOIDED when routing the next seed — the planner's
'do not repeat what the user disliked' guarantee (path-planner Phase 2)."""
disliked = set()
for arm in self._v2_path_matching_arms(path_memory, concepts):
for sample in arm.get("seeds", []) or []:
if not isinstance(sample, dict) or not sample.get("avoid"):
continue
try:
disliked.add(int(sample.get("seed")))
except (TypeError, ValueError):
continue
return disliked
def _v2_path_arm_beta(self, arm):
"""Beta(alpha, beta) posterior over an arm's liked-rate from its outcome counts.
Each liked outcome (reward >= V2_PATH_LIKED_REWARD) is a success, each disliked
(reward < V2_PATH_DISLIKED_REWARD) a failure; the mild-positive band is ignored.
Laplace prior (1, 1) so an unobserved arm is maximally uncertain."""
alpha, beta = 1.0, 1.0
for key, count in (arm.get("outcomes") or {}).items():
reward = _V2_KEY_REWARD.get(str(key), 0.0)
try:
weight = max(0.0, float(count))
except (TypeError, ValueError):
continue
if reward >= V2_PATH_LIKED_REWARD:
alpha += weight
elif reward < V2_PATH_DISLIKED_REWARD:
beta += weight
return alpha, beta
def _v2_path_explore_decision(self, path_memory, concepts):
"""Thompson sampling over concept-matching arms vs an uninformed 'explore fresh' arm.
Draws θ ~ Beta(α, β) for each matching config arm and one draw for a fresh Beta(1,1)
exploration arm, then picks the max. Arms with a high liked-rate win → exploit (reuse a
good seed); arms dominated by dislikes get a low θ and rarely win → the planner naturally
routes to a fresh seed ('do something else'). Returns ``(explore, reason)``, or
``(None, "")`` when there's no path data so the caller keeps its legacy behavior."""
arms = self._v2_path_matching_arms(path_memory, concepts)
if not arms:
return None, ""
best_theta, best_arm = -1.0, None
for arm in arms:
alpha, beta = self._v2_path_arm_beta(arm)
theta = random.betavariate(alpha, beta)
if theta > best_theta:
best_theta, best_arm = theta, arm
fresh_theta = random.betavariate(1.0, 1.0)
if fresh_theta >= best_theta:
return True, f"path planner: explore fresh (best arm θ={best_theta:.2f} < fresh {fresh_theta:.2f})"
return False, f"path planner: exploit arm θ={best_theta:.2f} (n={int(best_arm.get('n_pulls', 0))})"
def _v2_path_conditioning_plan(self, path_memory, concepts):
"""Route the CONDITIONING variant for the next gen from the path-outcome memory.
Aggregates the Beta posteriors of every concept-matching arm into one liked-rate mean
and returns a directive for ``_v2_ascend_conditioning``:
* "lock" — liked region: keep the conditioning stable (skip the stochastic search)
so a good direction is held in place rather than perturbed away.
* "explore" — disliked region: widen the value-function search to push toward a
different conditioning ("do something else").
* "normal" — ambiguous: today's ascend + standard search.
Returns ``(mode, reason)``, or ``(None, "")`` with no path data (caller keeps default)."""
arms = self._v2_path_matching_arms(path_memory, concepts)
if not arms:
return None, ""
alpha, beta = 1.0, 1.0
for arm in arms:
a, b = self._v2_path_arm_beta(arm)
alpha += a - 1.0
beta += b - 1.0
mean = alpha / (alpha + beta)
if mean >= 0.6:
return "lock", f"path planner: lock conditioning (liked-rate {mean:.2f})"
if mean <= 0.4:
return "explore", f"path planner: explore conditioning (liked-rate {mean:.2f})"
return "normal", f"path planner: normal conditioning (liked-rate {mean:.2f})"
def _v2_choose_successful_seed(self, refinement_key, prompt, fallback_seed, seed_output_connected=False, reset_session=False):
fallback_seed = int(fallback_seed)
if not seed_output_connected:
return fallback_seed, "fresh seed; seed output not connected", None
if reset_session:
return fallback_seed, "fresh seed; session reset", None
state, _ = self._v2_load_state(refinement_key, reset_session=False)
global_state = state.get("global", {}) if isinstance(state, dict) else {}
memory = global_state.get("successful_seed_memory", {}) if isinstance(global_state, dict) else {}
concepts = self._v2_seed_concepts_from_text(prompt)
# Path-planner AVOIDANCE: never reuse a seed that gave a disliked result for these
# concepts, and prefer a fresh seed if the fallback itself is one we should avoid.
disliked = self._v2_path_disliked_seeds(global_state.get("path_outcomes", {}), concepts)
avoided_note = f"; avoiding {len(disliked)} disliked seed(s)" if disliked else ""
while fallback_seed in disliked:
fallback_seed = random.randint(1, 0xffffffffffffffff)
if not isinstance(memory, dict) or not memory:
return fallback_seed, f"fresh seed; no successful seed memory{avoided_note}", None
candidates = []
for concept in concepts:
entry = memory.get(concept)
if not isinstance(entry, dict):
continue
for item in entry.get("seeds", []) or []:
if isinstance(item, dict):
try:
if int(item.get("seed")) in disliked:
continue # liked once, disliked later — drop it
except (TypeError, ValueError):
pass
candidates.append((concept, item))
if not candidates:
return fallback_seed, f"fresh seed; no concept seed match{avoided_note}", None
# Explore/exploit gate. With path-outcome data, a Thompson draw over the matching config
# arms decides — good arms exploit, dislike-dominated arms route to a fresh seed. With no
# path data yet, fall back to the flat reuse chance.
explore, plan_reason = self._v2_path_explore_decision(global_state.get("path_outcomes", {}), concepts)
if explore is None:
explore = random.random() >= V2_SEED_REUSE_CHANCE
plan_reason = "successful seed memory skipped"
if explore:
return fallback_seed, f"fresh seed; {plan_reason}{avoided_note}", None
candidates.sort(
key=lambda pair: (int(pair[1].get("hit_count", 0)), int(pair[1].get("last_iteration", 0))),
reverse=True,
)
concept, item = random.choice(candidates[:min(8, len(candidates))])
try:
seed = int(item.get("seed"))
except (TypeError, ValueError):
return fallback_seed, f"fresh seed; selected memory seed invalid{avoided_note}", None
scene_seeds = []
for value in item.get("scene_seeds") or []:
try:
scene_seeds.append(int(value))
except (TypeError, ValueError):
pass
scene_seeds = scene_seeds or None
return seed, f"successful seed memory: reused {seed} from '{concept}'{avoided_note}", scene_seeds
def _v2_conditioning_vector(self, conditioning):
if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1:
return None
if conditioning.dim() == 3:
return conditioning.detach().float().mean(dim=(0, 1))
return conditioning.detach().float().mean(dim=0)
def _v2_phrase_words(self, text):
return [
word.strip().lower()
for word in re.findall(r"[\w'’.-]+", str(text or ""), flags=re.UNICODE)
if self._is_valuable_token(word.strip())
]
def _v2_has_action_suffix(self, words):
for word in words:
if word.endswith("ing"):
stem = word[:-3]
elif word.endswith("ed"):
stem = word[:-2]
else:
continue
if stem in self.ACTION_SUFFIX_STEMS:
return True
if stem.endswith("n") and stem[:-1] in self.ACTION_SUFFIX_STEMS:
return True
if stem.endswith("p") and stem[:-1] in self.ACTION_SUFFIX_STEMS:
return True
return False
def _v2_rebalance_anchored_scores(self, scores, words):
words = set(words or [])
has_action = bool(words & self.CATEGORY_KEYWORDS["action"]) or self._v2_has_action_suffix(words)
if words & self.APPEARANCE_CONTEXT_WORDS:
scores["appearance"] = max(scores["appearance"], 0.66)
if not has_action:
scores["action"] = min(scores["action"], 0.40)
if words & self.ENVIRONMENT_CONTEXT_WORDS:
scores["environment"] = max(scores["environment"], 0.66)
if not has_action:
scores["action"] = min(scores["action"], 0.36)
if words & self.QUALITY_CONTEXT_WORDS:
scores["quality"] = max(scores["quality"], 0.64)
if not has_action:
scores["action"] = min(scores["action"], 0.34)
if has_action:
scores["action"] = max(scores["action"], 0.66)
return scores
def _v2_heuristic_scores(self, phrase):
words = set(self._v2_phrase_words(phrase))
scores = {category: 0.0 for category in self.CATEGORY_DESCRIPTIONS}
for category, keywords in self.CATEGORY_KEYWORDS.items():
hits = len(words & keywords)
if hits:
scores[category] += min(0.92, 0.48 + hits * 0.18)
text = str(phrase or "").lower()
if self._v2_has_action_suffix(words):
scores["action"] = max(scores["action"], 0.52)
if "camera" in text and re.search(r"\b(move|moves|moving|push|pull|pan|zoom|track|dolly|tilt)", text):
scores["camera"] = max(scores["camera"], 0.78)
scores["action"] = max(scores["action"], 0.48)
if re.search(r"\b(slowly|quickly|gentle|smooth|fast)\b", text) and scores["action"] > 0.0:
scores["action"] = min(0.92, scores["action"] + 0.12)
scores = self._v2_rebalance_anchored_scores(scores, words)
return scores
def _v2_scores_primary(self, scores):
if not scores:
return "details", 0.0
primary, confidence = max(
scores.items(),
key=lambda item: (item[1], self.CATEGORY_PRIORITY.get(item[0], 0.0)),
)
return primary, float(confidence)
def _v2_primary_category_for_text(self, text):
return self._v2_scores_primary(self._v2_heuristic_scores(text))[0]
def _v2_item_primary_category(self, item):
if not isinstance(item, dict):
return self._v2_primary_category_for_text(item)
scores = (
item.get("effective_category_scores") or
item.get("category_scores") or
item.get("clip_heuristic_scores") or
self._v2_heuristic_scores(item.get("text", ""))
)
primary, _ = self._v2_scores_primary(scores)
return str(item.get("primary") or primary or "details").lower()
def _v2_item_has_blocked_auto_category(self, item):
if isinstance(item, dict):
categories = {
str(item.get("primary", "")).lower(),
str(item.get("machine_primary", "")).lower(),
self._v2_item_primary_category(item),
}
scores = item.get("effective_category_scores", item.get("category_scores", {}))
if isinstance(scores, dict):
primary, confidence = self._v2_scores_primary(scores)
categories.add(primary)
for category, score in scores.items():
if float(score or 0.0) >= max(0.42, float(confidence) * 0.82):
categories.add(str(category).lower())
return bool(categories & self.AUTO_INJECT_BLOCKED_CATEGORIES)
return self._v2_primary_category_for_text(item) in self.AUTO_INJECT_BLOCKED_CATEGORIES
def _v2_now_iso(self):
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def _v2_auto_inject_entry_allowed(self, entry, prompt=""):
text = str(entry.get("text", "") if isinstance(entry, dict) else entry).strip()
if not text:
return False
explicit = self._v2_prompt_contains_text(prompt, text)
if isinstance(entry, dict) and bool(entry.get("auto_inject_suppressed")) and not explicit:
return False
if self._v2_item_has_blocked_auto_category(entry) and not explicit:
return False
return True
def _v2_clip_similarity_scores(self, clip, phrase, category_vectors, encode_cache=None):
phrase_cond, _, _ = self._v2_encode_prompt(clip, phrase, encode_cache=encode_cache,
purpose="phrase classification")
phrase_vector = self._v2_conditioning_vector(phrase_cond)
if phrase_vector is None:
return {}
scores = {}
for category, vector in category_vectors.items():
if not isinstance(vector, torch.Tensor) or vector.shape != phrase_vector.shape:
continue
sim = F.cosine_similarity(phrase_vector.unsqueeze(0), vector.to(phrase_vector.device).unsqueeze(0), dim=-1)
scores[category] = float(((sim.item() + 1.0) * 0.5))
return scores
# Eight fixed strings embedded to classify phrases by cosine similarity. They are
# CONSTANT — CATEGORY_DESCRIPTIONS is a class attribute, so the result depends only on the
# encoder. Held for the process rather than the run, keyed by the encoder object, because
# re-embedding them every run costs eight full encoder passes to arrive at the same eight
# vectors. This is not a cache of generated content: nothing user-typed goes into it, and
# a different encoder gets a different key.
_V2_CATEGORY_VECTOR_CACHE = {}
def _v2_category_vectors(self, clip, encode_cache=None):
if isinstance(encode_cache, dict):
cache_key = ("category_vectors", id(clip))
cached = encode_cache.get(cache_key)
if cached is not None:
return cached
held = type(self)._V2_CATEGORY_VECTOR_CACHE.get(id(clip))
if held is not None and held[0]() is clip:
# A WEAK ref, and `is clip` on top of it. Weak so the entry never keeps an encoder
# alive — pinning a text encoder in VRAM to save eight passes would be a bad
# trade. The identity check guards the id() key, since ids are reused once a
# model is freed and a stale entry would classify against another encoder.
if isinstance(encode_cache, dict):
encode_cache[cache_key] = held[1]
return held[1]
vectors = {}
for category, description in self.CATEGORY_DESCRIPTIONS.items():
cond, _, _ = self._v2_encode_prompt(clip, description, encode_cache=encode_cache,
purpose="category vectors")
vector = self._v2_conditioning_vector(cond)
if vector is not None:
vectors[category] = vector.cpu()
if vectors:
try:
ref = weakref.ref(clip)
except TypeError: # not weak-referenceable: hold nothing rather than hold hard
ref = None
if ref is not None:
type(self)._V2_CATEGORY_VECTOR_CACHE.clear()
type(self)._V2_CATEGORY_VECTOR_CACHE[id(clip)] = (ref, vectors)
if isinstance(encode_cache, dict):
encode_cache[cache_key] = vectors
return vectors
def _v2_merge_clip_category_scores(self, heuristic_scores, clip_scores):
if not clip_scores:
return heuristic_scores
merged = dict(heuristic_scores)
heuristic_primary, heuristic_confidence = self._v2_scores_primary(heuristic_scores)
ranked_clip = sorted(clip_scores.items(), key=lambda item: item[1], reverse=True)
if not ranked_clip:
return merged
clip_primary, clip_confidence = ranked_clip[0]
clip_runner_up = ranked_clip[1][1] if len(ranked_clip) > 1 else 0.0
clip_margin = float(clip_confidence) - float(clip_runner_up)
if heuristic_confidence >= 0.60:
return merged
if heuristic_confidence >= 0.44 and clip_primary != heuristic_primary and clip_margin < 0.075:
return merged
mean_clip = sum(float(value) for value in clip_scores.values()) / max(1, len(clip_scores))
for category, score in clip_scores.items():
relative = max(0.0, float(score) - mean_clip)
merged[category] = max(float(merged.get(category, 0.0)), min(0.56, relative * 2.40))
if clip_margin >= 0.045:
merged[clip_primary] = max(float(merged.get(clip_primary, 0.0)), min(0.68, 0.48 + clip_margin * 2.20))
return merged
def _v2_category_template(self, value=0.0):
return {category: float(value) for category in self.CATEGORY_DESCRIPTIONS}
def _v2_clean_category_scores(self, scores, default=0.0):
cleaned = self._v2_category_template(default)
if isinstance(scores, dict):
for category in cleaned:
try:
cleaned[category] = float(scores.get(category, cleaned[category]))
except (TypeError, ValueError):
cleaned[category] = float(default)
return cleaned
def _v2_axis_categories(self, axis):
if axis == "action":
return {"action", "camera"}
if axis == "details":
return {"details", "appearance", "environment", "camera", "action"}
if axis == "quality":
return {"quality", "style"}
return set()
def _v2_axis_category_weight(self, axis, category, scores=None):
if category not in self._v2_axis_categories(axis):
return 0.0
if axis == "details" and category == "details":
return 1.18
if axis == "details" and category == "action":
return 0.58
if axis == "details" and category in {"appearance", "environment"}:
return 0.72
if axis == "action" and category == "camera":
return 0.58
if axis == "details" and category == "camera":
return 0.64
scores = scores or {}
primary, _ = self._v2_scores_primary(scores)
return 1.0 if category == primary else 0.78
def _v2_effective_category_scores(self, machine_scores, category_weights, evidence_count=0):
machine = self._v2_clean_category_scores(machine_scores)
learned = self._v2_clean_category_scores(category_weights)
evidence_count = max(0, int(evidence_count or 0))
learned_power = min(1.85, 0.30 + evidence_count * 0.18)
effective = {}
for category in machine:
value = float(machine.get(category, 0.0)) + float(learned.get(category, 0.0)) * learned_power
effective[category] = round(max(-1.0, min(8.0, value)), 6)
return effective
def _v2_position_bucket(self, index, total):
total = max(1, int(total or 1))
index = max(0, int(index or 0))
if total <= 2:
return "solo" if total == 1 else ("early" if index == 0 else "late")
ratio = index / float(max(1, total - 1))
if ratio < 0.34:
return "early"
if ratio > 0.66:
return "late"
return "mid"
def _v2_context_for_phrase(self, phrases, index, text, window=3):
phrases = phrases or []
index = max(0, min(len(phrases) - 1, int(index or 0))) if phrases else 0
anchor_words = set(self._v2_phrase_words(text))
context_words = []
for offset in range(max(0, index - window), min(len(phrases), index + window + 1)):
if offset == index:
continue
item = phrases[offset]
item_text = item.get("text", "") if isinstance(item, dict) else str(item)
for word in self._v2_phrase_words(item_text):
if word in anchor_words or word in context_words:
continue
context_words.append(word)
bucket = self._v2_position_bucket(index, len(phrases))
return {
"anchor_words": sorted(anchor_words),
"context_words": context_words[:10],
"position_bucket": bucket,
"window": int(window),
}
def _v2_context_signature(self, context):
if not isinstance(context, dict):
return ""
words = sorted({
word
for word in context.get("context_words", [])
if self._is_valuable_token(str(word).strip())
})[:8]
bucket = str(context.get("position_bucket", "mid"))
return f"{bucket}|{','.join(words)}" if words else bucket
def _v2_context_similarity(self, left, right):
if not isinstance(left, dict) or not isinstance(right, dict):
return 0.0
left_words = {
word for word in left.get("context_words", [])
if self._is_valuable_token(str(word).strip())
}
right_words = {
word for word in right.get("context_words", [])
if self._is_valuable_token(str(word).strip())
}
if not left_words and not right_words:
word_score = 0.0
else:
word_score = len(left_words & right_words) / float(max(1, len(left_words | right_words)))
bucket_bonus = 0.12 if left.get("position_bucket") == right.get("position_bucket") else 0.0
return max(0.0, min(1.0, word_score + bucket_bonus))
def _v2_ensure_context_sense(self, entry, context):
signature = self._v2_context_signature(context)
if not signature:
return None
senses = entry.setdefault("context_senses", {})
initial_weights = self._v2_clean_category_scores(entry.get("category_weights", {}))
sense = senses.setdefault(signature, {
"signature": signature,
"context": context,
"category_weights": {
category: round(float(value) * 0.35, 6)
for category, value in initial_weights.items()
},
"effective_category_scores": self._v2_clean_category_scores(entry.get("effective_category_scores", {})),
"category_evidence_count": 0,
"occurrence_count": 0,
"last_seen_iter": 0,
"rating_evidence": {},
})
sense["context"] = context
sense["category_weights"] = self._v2_clean_category_scores(sense.get("category_weights", {}))
sense["effective_category_scores"] = self._v2_clean_category_scores(sense.get("effective_category_scores", {}))
sense.setdefault("category_evidence_count", 0)
sense.setdefault("occurrence_count", 0)
sense.setdefault("last_seen_iter", 0)
sense.setdefault("rating_evidence", {})
return sense
def _v2_prune_context_senses(self, entry, limit=24):
senses = entry.get("context_senses", {})
if not isinstance(senses, dict) or len(senses) <= limit:
return
def rank(item):
_, sense = item
if not isinstance(sense, dict):
return (-1, -1, -1)
return (
int(sense.get("category_evidence_count", 0)),
int(sense.get("occurrence_count", 0)),
int(sense.get("last_seen_iter", 0)),
)
kept = dict(sorted(senses.items(), key=rank, reverse=True)[:limit])
entry["context_senses"] = kept
def _v2_best_context_sense(self, entry, context):
if not isinstance(entry, dict) or not isinstance(context, dict):
return None, 0.0
senses = entry.get("context_senses", {})
if not isinstance(senses, dict):
return None, 0.0
signature = self._v2_context_signature(context)
if signature and isinstance(senses.get(signature), dict):
return senses[signature], 1.0
best_sense = None
best_score = 0.0
for sense in senses.values():
if not isinstance(sense, dict):
continue
score = self._v2_context_similarity(context, sense.get("context", {}))
if score > best_score:
best_sense = sense
best_score = score
if best_score < 0.22:
return None, 0.0
return best_sense, best_score
def _v2_blend_category_weights(self, target, source, scale=1.0):
source = self._v2_clean_category_scores(source)
for category, value in source.items():
target[category] = float(target.get(category, 0.0)) + float(value) * float(scale)
return target
def _v2_ensure_phrase_memory_entry(self, memory, text, phrase=None):
text = str(text or "").strip().lower()
phrase = phrase or {}
primary = phrase.get("primary") or phrase.get("machine_primary") or "details"
entry = memory.setdefault(text, {
"text": text,
"tokens": phrase.get("tokens", []),
"primary": primary,
"machine_primary": primary,
"category_scores": phrase.get("category_scores", {}),
"clip_heuristic_scores": phrase.get("clip_heuristic_scores", phrase.get("category_scores", {})),
"category_weights": self._v2_category_template(0.0),
"effective_category_scores": phrase.get("effective_category_scores", phrase.get("category_scores", {})),
"category_evidence_count": 0,
"rating_evidence": {},
"context_senses": {},
"occurrence_count": 0,
"score": 0.0,
"liked_count": 0,
"missing_count": 0,
"satisfied_count": 0,
"resolved_count": 0,
"regressed_count": 0,
"wrong_count": 0,
"wrong_appearance_count": 0,
"auto_inject_blocked_count": 0,
"auto_inject_suppressed": False,
"bad_count": 0,
"wanted_axes": {},
"satisfied_axes": {},
"resolved_axes": {},
"wrong_axes": {},
"positions": {},
})
entry.setdefault("text", text)
entry.setdefault("tokens", phrase.get("tokens", []))
entry.setdefault("primary", primary)
entry.setdefault("machine_primary", phrase.get("machine_primary", entry.get("primary", primary)))
entry["category_scores"] = self._v2_clean_category_scores(entry.get("category_scores", phrase.get("category_scores", {})))
entry["clip_heuristic_scores"] = self._v2_clean_category_scores(
entry.get("clip_heuristic_scores", entry.get("category_scores", {}))
)
entry["category_weights"] = self._v2_clean_category_scores(entry.get("category_weights", {}))
entry["effective_category_scores"] = self._v2_clean_category_scores(
entry.get("effective_category_scores", entry.get("category_scores", {}))
)
entry.setdefault("category_evidence_count", 0)
entry.setdefault("rating_evidence", {})
entry.setdefault("context_senses", {})
entry.setdefault("occurrence_count", 0)
entry.setdefault("score", 0.0)
entry.setdefault("liked_count", 0)
entry.setdefault("missing_count", 0)
entry.setdefault("satisfied_count", 0)
entry.setdefault("resolved_count", 0)
entry.setdefault("regressed_count", 0)
entry.setdefault("wrong_count", 0)
entry.setdefault("wrong_appearance_count", 0)
entry.setdefault("auto_inject_blocked_count", 0)
entry.setdefault("auto_inject_suppressed", False)
entry.setdefault("bad_count", 0)
entry.setdefault("wanted_axes", {})
entry.setdefault("satisfied_axes", {})
entry.setdefault("resolved_axes", {})
entry.setdefault("wrong_axes", {})
entry.setdefault("positions", {})
return entry
def _v2_apply_learned_category_scores(self, item, phrase_memory):
if not isinstance(phrase_memory, dict):
return item
text = str(item.get("text", "")).strip().lower()
context = item.get("context", {})
entry = phrase_memory.get(text)
machine_scores = self._v2_clean_category_scores(item.get("category_scores", {}))
weights = self._v2_category_template(0.0)
evidence_count = 0
context_status = "none"
if isinstance(entry, dict):
sense, sense_similarity = self._v2_best_context_sense(entry, context)
entry_scale = 0.35 if isinstance(sense, dict) else 1.0
self._v2_blend_category_weights(weights, entry.get("category_weights", {}), entry_scale)
evidence_count += int(int(entry.get("category_evidence_count", 0)) * entry_scale)
if isinstance(sense, dict):
self._v2_blend_category_weights(
weights,
sense.get("category_weights", {}),
1.20 + sense_similarity * 0.65,
)
evidence_count += int(int(sense.get("category_evidence_count", 0)) * max(0.35, sense_similarity))
context_status = "exact" if sense_similarity >= 0.99 else f"near:{sense_similarity:.2f}"
for word in self._v2_phrase_words(text):
token_entry = phrase_memory.get(word)
if not isinstance(token_entry, dict) or token_entry is entry:
continue
self._v2_blend_category_weights(weights, token_entry.get("category_weights", {}), 0.32)
evidence_count += int(int(token_entry.get("category_evidence_count", 0)) * 0.22)
sense, sense_similarity = self._v2_best_context_sense(token_entry, context)
if isinstance(sense, dict):
self._v2_blend_category_weights(
weights,
sense.get("category_weights", {}),
0.65 + sense_similarity * 0.55,
)
evidence_count += int(int(sense.get("category_evidence_count", 0)) * max(0.42, sense_similarity))
context_status = "exact" if sense_similarity >= 0.99 else f"near:{sense_similarity:.2f}"
effective = self._v2_effective_category_scores(machine_scores, weights, evidence_count)
primary, confidence = self._v2_scores_primary(effective)
machine_primary, machine_confidence = self._v2_scores_primary(machine_scores)
item["clip_heuristic_scores"] = machine_scores
item["machine_primary"] = machine_primary
item["machine_confidence"] = round(float(machine_confidence), 4)
item["category_weights"] = weights
item["category_evidence_count"] = evidence_count
item["effective_category_scores"] = effective
item["category_scores"] = effective
item["primary"] = primary
item["confidence"] = round(float(confidence), 4)
item["source"] = "rating_weighted" if evidence_count > 0 else item.get("source", "heuristic")
item["context_signature"] = self._v2_context_signature(context)
item["context_source"] = context_status
return item
def _v2_concept_units_for_run(self, last_run):
units = []
seen = set()
def add(text, kind, phrase=None, position=0, context=None):
clean = str(text or "").strip().lower()
context = context or {}
seen_key = f"{clean}:{self._v2_context_signature(context)}"
if len(clean) < 3 or seen_key in seen or not any(c.isalpha() for c in clean):
return
words = self._v2_phrase_words(clean)
if not words:
return
seen.add(seen_key)
source = phrase or {}
scores = source.get("clip_heuristic_scores", source.get("category_scores", self._v2_heuristic_scores(clean)))
primary, confidence = self._v2_scores_primary(scores)
units.append({
"text": clean,
"kind": kind,
"tokens": source.get("tokens", words),
"position": int(position),
"context": context,
"context_signature": self._v2_context_signature(context),
"category_scores": self._v2_clean_category_scores(scores),
"clip_heuristic_scores": self._v2_clean_category_scores(scores),
"effective_category_scores": self._v2_clean_category_scores(source.get("effective_category_scores", scores)),
"primary": source.get("primary", primary),
"machine_primary": source.get("machine_primary", primary),
"confidence": source.get("confidence", round(float(confidence), 4)),
})
for index, phrase in enumerate(last_run.get("phrases", []) or []):
if not isinstance(phrase, dict):
continue
text = str(phrase.get("text", "")).strip().lower()
phrase_context = phrase.get("context") or self._v2_context_for_phrase(last_run.get("phrases", []), index, text)
add(text, "phrase", phrase, phrase.get("position", index), phrase_context)
words = self._v2_phrase_words(text)
for word_index, word in enumerate(words):
token_context = self._v2_context_for_phrase(last_run.get("phrases", []), index, word)
add(word, "token", phrase, phrase.get("position", index) + word_index, token_context)
for size in (2, 3):
for start in range(0, max(0, len(words) - size + 1)):
add(" ".join(words[start:start + size]), "ngram", phrase, phrase.get("position", index) + start, phrase_context)
prompt = str(last_run.get("prompt", "") or "")
encoded_prompt = str(last_run.get("encoded_prompt", "") or "")
def add_prompt_segments(source_prompt, kind, offset=0):
prompt_segments = [
{"text": segment.strip()}
for segment in re.split(r"[,.;\n]+", source_prompt)
if segment.strip()
]
for position, segment in enumerate(prompt_segments):
add(
segment.get("text", ""),
kind,
None,
offset + position,
self._v2_context_for_phrase(prompt_segments, position, segment.get("text", "")),
)
add_prompt_segments(prompt, "prompt_phrase")
if encoded_prompt and encoded_prompt != prompt:
add_prompt_segments(encoded_prompt, "auto_phrase", offset=128)
for position, candidate in enumerate(last_run.get("repair_candidates", []) or []):
if not isinstance(candidate, dict):
continue
add(candidate.get("text", ""), "repair_candidate", candidate, 256 + position, {})
return units[:96]
def _v2_classify_phrases(self, clip, phrases, global_state=None, encode_cache=None):
phrase_items = []
uncertain = []
phrase_memory = (global_state or {}).get("phrase_memory", {}) if isinstance(global_state, dict) else {}
phrases = phrases or []
for index, phrase in enumerate(phrases):
text = str(phrase.get("text", "")).strip().lower()
if not text:
continue
scores = self._v2_heuristic_scores(text)
primary, confidence = self._v2_scores_primary(scores)
context = self._v2_context_for_phrase(phrases, index, text)
item = {
"text": text,
"tokens": list(phrase.get("tokens", [])),
"position": index,
"context": context,
"context_signature": self._v2_context_signature(context),
"category_scores": scores,
"clip_heuristic_scores": dict(scores),
"machine_primary": primary,
"machine_confidence": round(float(confidence), 4),
"category_weights": self._v2_category_template(0.0),
"category_evidence_count": 0,
"effective_category_scores": dict(scores),
"primary": primary,
"confidence": round(float(confidence), 4),
"source": "heuristic",
}
if confidence < 0.62:
uncertain.append(item)
phrase_items.append(item)
if uncertain and clip is not None:
category_vectors = self._v2_category_vectors(clip, encode_cache=encode_cache)
for item in uncertain:
clip_scores = self._v2_clip_similarity_scores(clip, item["text"], category_vectors, encode_cache=encode_cache)
if not clip_scores:
continue
merged = self._v2_merge_clip_category_scores(item["category_scores"], clip_scores)
primary, confidence = self._v2_scores_primary(merged)
item["category_scores"] = merged
item["clip_heuristic_scores"] = dict(merged)
item["machine_primary"] = primary
item["machine_confidence"] = round(float(confidence), 4)
item["effective_category_scores"] = dict(merged)
item["primary"] = primary
item["confidence"] = round(float(confidence), 4)
item["source"] = "clip_similarity"
for item in phrase_items:
self._v2_apply_learned_category_scores(item, phrase_memory)
return phrase_items
def _v2_axis_matches_category(self, axis, category):
category = (category or "details").lower()
return category in self._v2_axis_categories(axis)
def _v2_order_axes(self, axes):
axis_set = {axis for axis in axes or [] if axis in V2_FEEDBACK_AXES}
return [axis for axis in V2_FEEDBACK_AXES if axis in axis_set]
def _v2_axes_for_category(self, category):
return {
axis
for axis in V2_FEEDBACK_AXES
if self._v2_axis_matches_category(axis, category)
}
def _v2_axes_for_scores(self, scores):
primary, confidence = self._v2_scores_primary(scores)
axes = set(self._v2_axes_for_category(primary))
for category, score in (scores or {}).items():
if float(score or 0.0) >= max(0.28, float(confidence) * 0.72):
axes |= self._v2_axes_for_category(category)
return axes
def _v2_axis_feedback(self, rating_profile, previous_missing_axes=None):
key = rating_profile.get("key", "")
if key == "continue":
all_axes = set(V2_FEEDBACK_AXES)
has_prev = previous_missing_axes is not None
previous_missing = set(previous_missing_axes or []) & all_axes if has_prev else set()
missing_axes = previous_missing
satisfied_axes = all_axes - missing_axes
resolved_axes = set() if not has_prev else (previous_missing - missing_axes)
regressed_axes = set()
return {
"missing_axes": self._v2_order_axes(missing_axes),
"satisfied_axes": self._v2_order_axes(satisfied_axes),
"resolved_axes": self._v2_order_axes(resolved_axes),
"regressed_axes": self._v2_order_axes(regressed_axes),
"wrong_axes": [],
}
if rating_profile.get("skip_learning") or key == "discover":
return {
"missing_axes": [],
"satisfied_axes": [],
"resolved_axes": [],
"regressed_axes": [],
"wrong_axes": [],
}
all_axes = set(V2_FEEDBACK_AXES)
missing_axes = set(rating_profile.get("missing_axes", [])) & all_axes
wrong_axes = set(rating_profile.get("wrong_axes", [])) & all_axes
if key == "awful":
missing_axes = set(all_axes)
elif key in {"like", "loved_it", "nailed_it"}:
missing_axes = set()
wrong_axes = set()
has_previous_axis_signal = previous_missing_axes is not None
previous_missing = set(previous_missing_axes or []) & all_axes
satisfied_axes = all_axes - missing_axes
resolved_axes = previous_missing - missing_axes
regressed_axes = missing_axes - previous_missing if has_previous_axis_signal else set()
return {
"missing_axes": self._v2_order_axes(missing_axes),
"satisfied_axes": self._v2_order_axes(satisfied_axes),
"resolved_axes": self._v2_order_axes(resolved_axes),
"regressed_axes": self._v2_order_axes(regressed_axes),
"wrong_axes": self._v2_order_axes(wrong_axes),
}
def _v2_axis_feedback_status(self, axis_feedback):
if not isinstance(axis_feedback, dict):
return "Axis feedback: none."
def fmt(name):
values = self._v2_order_axes(axis_feedback.get(name, []))
return ", ".join(values) if values else "none"
return (
f"Axis feedback: missing {fmt('missing_axes')} | "
f"satisfied {fmt('satisfied_axes')} | "
f"resolved {fmt('resolved_axes')} | "
f"regressed {fmt('regressed_axes')} | "
f"wrong {fmt('wrong_axes')}."
)
def _v2_active_repair_feedback(self, global_state, axis_feedback, learning_profile):
if not isinstance(global_state, dict):
return axis_feedback, "Repair persistence: unavailable."
active = set(global_state.get("active_repair_axes", []) or []) & set(V2_FEEDBACK_AXES)
key = learning_profile.get("key", "") if isinstance(learning_profile, dict) else ""
if key in {"like", "loved_it", "nailed_it"}:
active = set()
global_state["active_repair_axes"] = []
feedback = dict(axis_feedback or {})
feedback["missing_axes"] = []
return feedback, "Repair persistence: cleared by Perfect."
if key == "fresh_prompt":
active = set()
global_state["active_repair_axes"] = []
feedback = dict(axis_feedback or {})
feedback["missing_axes"] = []
return feedback, "Repair persistence: cleared for prompt edit (session training kept)."
if key == "continue":
active |= set(axis_feedback.get("missing_axes", [])) & set(V2_FEEDBACK_AXES)
elif not learning_profile.get("skip_learning") and key != "discover":
active |= set(axis_feedback.get("missing_axes", [])) & set(V2_FEEDBACK_AXES)
active |= set(axis_feedback.get("wrong_axes", [])) & set(V2_FEEDBACK_AXES)
ordered = self._v2_order_axes(active)
global_state["active_repair_axes"] = ordered
feedback = dict(axis_feedback or {})
feedback["missing_axes"] = ordered
if ordered:
return feedback, f"Repair persistence: active until Perfect ({', '.join(ordered)})."
return feedback, "Repair persistence: no active repairs."
def _v2_top_category_summary(self, scores, limit=3):
cleaned = self._v2_clean_category_scores(scores)
ranked = [
(category, value)
for category, value in sorted(cleaned.items(), key=lambda item: item[1], reverse=True)
if abs(float(value)) > 0.0001
]
if not ranked:
return "neutral"
return ", ".join(f"{category}:{value:.2f}" for category, value in ranked[:limit])
def _v2_training_guidance(self, has_previous_run, learning_profile, axis_feedback, phrases, memory_status, lora_status):
if not has_previous_run:
return (
"Guidance: first V2 run only seeds the current prompt. "
"Set the rating after reviewing this output, then run again with the same refinement key to train."
)
if learning_profile.get("key") == "continue":
return (
"Guidance: no new rating — applying existing session memory and active repairs. "
"Rate the last render when you want Studio to learn from it."
)
if learning_profile.get("key") == "fresh_prompt":
return (
"Guidance: prompt changed — encoding the new text with session training kept; "
"stale action/detail repairs were cleared. Rate when you want new learning."
)
if learning_profile.get("skip_learning"):
return (
"Guidance: '-Just forget it-' skipped learning. Use it for workflow/model/seed failures; "
"use a missing-axis rating when the prompt should teach the node."
)
if learning_profile.get("key") == "wrong_appearance":
return (
"Guidance: wrong-appearance rating suppresses auto-injected character, clothing, "
"subject, and background memory without training unrelated action/detail/quality ratings."
)
if learning_profile.get("wrong_axes"):
return (
"Guidance: wrong-intent rating preserved satisfied axes but marked the current "
"action/detail context as wrong for this request. The next run will repair from intent "
"and preferred context clusters."
)
missing_axes = set(axis_feedback.get("missing_axes", [])) if isinstance(axis_feedback, dict) else set()
resolved_axes = set(axis_feedback.get("resolved_axes", [])) if isinstance(axis_feedback, dict) else set()
regressed_axes = set(axis_feedback.get("regressed_axes", [])) if isinstance(axis_feedback, dict) else set()
phrase_axes = set()
context_sources = set()
weak_contexts = 0
for phrase in phrases or []:
phrase_axes |= self._v2_axes_for_scores(
phrase.get("effective_category_scores", phrase.get("category_scores", {})) or
{phrase.get("primary", "details"): 1.0}
)
context_source = str(phrase.get("context_source", "none"))
context_sources.add(context_source)
if context_source == "none":
weak_contexts += 1
notes = []
if missing_axes:
if not (missing_axes & phrase_axes):
notes.append(
"missing axes did not strongly match current prompt concepts; make the missing action/detail/quality words more explicit"
)
else:
notes.append(
"missing axes matched prompt concepts; repeated consistent ratings will strengthen those category weights"
)
if resolved_axes:
notes.append("resolved axes were preserved as positive evidence")
if regressed_axes:
notes.append("regressed axes were marked as needing repair")
if weak_contexts and phrases:
notes.append("some concepts have no learned context yet; repeat similar phrasing if you want stable sense learning")
if "exact" in context_sources:
notes.append("at least one concept matched an exact learned context")
if lora_status.startswith("LoRA suggestions: no"):
notes.append("no LoRA stack connected, so only prompt/conditioning memory trained")
if "no prompt phrases stored" in memory_status:
notes.append("no valuable prompt concepts were found; add concrete descriptive words")
return "Guidance: " + "; ".join(notes[:4]) + "." if notes else "Guidance: training signal looks usable."
def _v2_category_diagnostics(self, phrases, limit=8):
parts = []
for item in (phrases or [])[:limit]:
text = item.get("text", "")
machine = item.get("machine_primary", item.get("primary", "details"))
effective = item.get("primary", "details")
context = item.get("context_source", "none")
evidence = int(item.get("category_evidence_count", 0))
top = self._v2_top_category_summary(item.get("effective_category_scores", item.get("category_scores", {})), limit=2)
parts.append(f"- {text}: {machine}->{effective}, ctx={context}, evidence={evidence}, top={top}")
return "Category diagnostics:\n" + ("\n".join(parts) if parts else "none")
def _v2_phrase_axes(self, phrase):
if not isinstance(phrase, dict):
return set()
scores = phrase.get("effective_category_scores", phrase.get("category_scores", {}))
axes = self._v2_axes_for_scores(scores) if isinstance(scores, dict) else set()
primary = phrase.get("primary")
if primary:
axes |= self._v2_axes_for_category(primary)
return axes
def _v2_clean_phrase_text(self, text):
text = re.sub(r"\s+", " ", str(text or "").strip().lower()).strip(" ,;:.")
if len(text) < 3 or not any(char.isalpha() for char in text):
return ""
if not self._v2_phrase_words(text):
return ""
return text
def _v2_phrase_root_set(self, text):
return self._v2_repair_intent_roots(text)
def _v2_phrase_texts_match(self, left, right):
left_clean = self._v2_clean_phrase_text(left)
right_clean = self._v2_clean_phrase_text(right)
if not left_clean or not right_clean:
return False
if self._v2_prompt_contains_text(left_clean, right_clean) or self._v2_prompt_contains_text(right_clean, left_clean):
return True
left_roots = self._v2_phrase_root_set(left_clean)
right_roots = self._v2_phrase_root_set(right_clean)
if not left_roots or not right_roots:
return False
overlap = left_roots & right_roots
if not overlap:
return False
if min(len(left_roots), len(right_roots)) == 1:
return True
needed = min(2, len(left_roots))
return len(overlap) >= needed or (len(overlap) / max(1, len(left_roots))) >= 0.58
def _v2_phrase_represented_by(self, text, phrases):
return any(
self._v2_phrase_texts_match(text, phrase.get("text", "") if isinstance(phrase, dict) else phrase)
for phrase in phrases or []
)
def _v2_alignment_token_units(self, text):
words = self._v2_phrase_words(text)
units = [{"token": word, "kind": "word", "words": [word]} for word in words]
for index in range(0, max(0, len(words) - 1)):
pair = " ".join(words[index:index + 2])
units.append({"token": pair, "kind": "pair", "words": words[index:index + 2]})
return units
def _v2_intent_alignment_key(self, intent_prompt):
clean = self._v2_prompt_key(intent_prompt).lower()
return md5(clean.encode("utf-8")).hexdigest()
def _v2_intent_source_prompt(self, prompt, intent_prompt="", intent_is_vague=False):
intent_prompt = self._v2_prompt_key(intent_prompt)
if intent_prompt and not intent_is_vague and not self._v2_user_intent_prompt_is_vague(intent_prompt):
return intent_prompt
return self._v2_prompt_key(prompt)
def _v2_prompt_body_similarity(self, left, right):
left_roots = self._v2_repair_intent_roots(left)
right_roots = self._v2_repair_intent_roots(right)
if not left_roots or not right_roots:
return 0.0
overlap = left_roots & right_roots
if not overlap:
return 0.0
jaccard = len(overlap) / float(max(1, len(left_roots | right_roots)))
containment = len(overlap) / float(max(1, min(len(left_roots), len(right_roots))))
return max(jaccard, containment * 0.88)
def _v2_intent_family_slot(self, global_state, intent_prompt, create=True):
intent_prompt = self._v2_prompt_key(intent_prompt)
if not isinstance(global_state, dict) or not intent_prompt:
return "", None, 0.0
memory = global_state.setdefault("intent_family_memory", {})
if not isinstance(memory, dict):
memory = {}
global_state["intent_family_memory"] = memory
best_key = ""
best_slot = None
best_similarity = 0.0
for key, slot in memory.items():
if not isinstance(slot, dict):
continue
candidates = [slot.get("intent_prompt", "")]
candidates.extend(slot.get("intent_prompts", []) if isinstance(slot.get("intent_prompts"), list) else [])
for candidate in candidates:
similarity = self._v2_prompt_body_similarity(intent_prompt, candidate)
if similarity > best_similarity:
best_key = key
best_slot = slot
best_similarity = similarity
if best_slot is not None and best_similarity >= 0.58:
return best_key, best_slot, best_similarity
if not create:
return "", None, best_similarity
family_key = self._v2_intent_alignment_key(intent_prompt)
slot = memory.setdefault(family_key, {
"family_key": family_key,
"intent_prompt": intent_prompt,
"intent_prompts": [],
"total_seen": 0,
"perfect_anchors": {},
"loved_variants": {},
"variant_evidence": {},
"intent_preference_phrases": {},
"conditioning_deltas": {},
"negative_tags": {},
"last_seen_iter": 0,
})
return family_key, slot, 1.0
def _v2_sync_intent_family_aliases(self, global_state, family_key, slot):
if not family_key or not isinstance(slot, dict) or not isinstance(global_state, dict):
return
global_state.setdefault("perfect_anchors", {})[family_key] = slot.get("perfect_anchors", {})
global_state.setdefault("variant_evidence", {})[family_key] = slot.get("variant_evidence", {})
global_state.setdefault("intent_preference_phrases", {})[family_key] = slot.get("intent_preference_phrases", {})
global_state.setdefault("conditioning_deltas", {})[family_key] = slot.get("conditioning_deltas", {})
def _v2_intent_texts_for_family(self, intent_prompt, intent_phrases, family_slot=None):
texts = []
intent_prompt = self._v2_clean_phrase_text(intent_prompt)
if intent_prompt:
texts.append(intent_prompt)
for phrase in intent_phrases or []:
text = self._v2_clean_phrase_text(phrase.get("text", "") if isinstance(phrase, dict) else phrase)
if text and text not in texts:
texts.append(text)
if isinstance(family_slot, dict):
for entry in family_slot.get("intent_preference_phrases", {}).values():
if not isinstance(entry, dict) or int(entry.get("seen_count", 0)) < 2:
continue
text = self._v2_clean_phrase_text(entry.get("text", ""))
if text and text not in texts:
texts.append(text)
return texts
def _v2_text_represented_by_intent(self, text, intent_prompt="", intent_phrases=None, family_slot=None):
clean = self._v2_clean_phrase_text(text)
if not clean:
return False
for candidate in self._v2_intent_texts_for_family(intent_prompt, intent_phrases or [], family_slot):
if self._v2_phrase_texts_match(clean, candidate):
return True
return False
def _v2_text_axes(self, text):
scores = self._v2_heuristic_scores(text)
axes = self._v2_axes_for_scores(scores)
primary = self._v2_primary_category_for_text(text)
if primary:
axes |= self._v2_axes_for_category(primary)
return axes
def _v2_text_semantic_similarity(self, clip, left, right, encode_cache=None):
if clip is None:
return 0.0
if not hasattr(clip, "encode_from_tokens_scheduled"):
# Generation-only clip (e.g. FunPackAdvisorLLM) can't encode; skip semantic gate.
return 1.0
left = self._v2_clean_phrase_text(left)
right = self._v2_clean_phrase_text(right)
if not left or not right:
return 0.0
left_cond, _, _ = self._v2_encode_prompt(clip, left, encode_cache=encode_cache,
purpose="memory/scene matching")
right_cond, _, _ = self._v2_encode_prompt(clip, right, encode_cache=encode_cache,
purpose="memory/scene matching")
left_vector = self._v2_conditioning_vector(left_cond)
right_vector = self._v2_conditioning_vector(right_cond)
if left_vector is None or right_vector is None or left_vector.shape != right_vector.shape:
return 0.0
try:
sim = F.cosine_similarity(left_vector.unsqueeze(0), right_vector.unsqueeze(0), dim=-1)
return float((sim.item() + 1.0) * 0.5)
except Exception:
return 0.0
def _v2_short_intent_expansions(self, intent_prompt):
key = self._v2_clean_phrase_text(intent_prompt)
expansions = {
"defenestration": [
"thrown out of window",
"throwing a person out of a window",
"human thrown through a window",
],
"defenstration": [
"defenestration",
"thrown out of window",
"throwing a person out of a window",
"human thrown through a window",
],
}
return expansions.get(key, [])
def _v2_mark_semantic_intent_locks(self, clip, phrases, intent_prompt="", intent_phrases=None, encode_cache=None):
intent_prompt = self._v2_clean_phrase_text(intent_prompt)
if not phrases or not intent_prompt:
return phrases
intent_words = self._v2_phrase_words(intent_prompt)
if len(intent_words) > 3:
return phrases
semantic_sources = [intent_prompt]
for expansion in self._v2_short_intent_expansions(intent_prompt):
clean_expansion = self._v2_clean_phrase_text(expansion)
if clean_expansion and clean_expansion not in semantic_sources:
semantic_sources.append(clean_expansion)
for phrase in intent_phrases or []:
text = self._v2_clean_phrase_text(phrase.get("text", "") if isinstance(phrase, dict) else phrase)
if text and text not in semantic_sources:
semantic_sources.append(text)
for phrase in phrases:
if not isinstance(phrase, dict) or phrase.get("intent_locked"):
continue
text = self._v2_clean_phrase_text(phrase.get("text", ""))
if not text:
continue
lexical_match = any(self._v2_phrase_texts_match(text, source) for source in semantic_sources)
if lexical_match:
phrase["intent_locked"] = True
phrase["semantic_intent_locked"] = True
phrase["intent_similarity"] = 1.0
continue
best = max(
(
self._v2_text_semantic_similarity(clip, source, text, encode_cache=encode_cache)
for source in semantic_sources
),
default=0.0,
)
if best >= 0.62:
phrase["intent_locked"] = True
phrase["semantic_intent_locked"] = True
phrase["intent_similarity"] = round(float(best), 4)
return phrases
def _v2_requested_negative_block_texts(self, current_prompt="", intent_prompt="", intent_phrases=None, family_slot=None):
texts = []
for source in (current_prompt, intent_prompt):
clean = self._v2_clean_phrase_text(source)
if clean and clean not in texts:
texts.append(clean)
for phrase in self._ordered_prompt_phrases(source):
text = self._v2_clean_phrase_text(phrase.get("text", ""))
if text and text not in texts:
texts.append(text)
for text in self._v2_intent_texts_for_family(intent_prompt, intent_phrases or [], family_slot):
if text and text not in texts:
texts.append(text)
return texts
def _v2_negative_text_conflicts_with_request(self, text, current_prompt="", intent_prompt="", intent_phrases=None, family_slot=None):
clean = self._v2_clean_phrase_text(text)
if not clean:
return False
clean_roots = self._v2_repair_intent_roots(clean)
clean_axes = self._v2_text_axes(clean)
for requested in self._v2_requested_negative_block_texts(current_prompt, intent_prompt, intent_phrases, family_slot):
if (
self._v2_prompt_contains_text(current_prompt, clean) or
self._v2_prompt_contains_text(requested, clean) or
self._v2_prompt_contains_text(clean, requested) or
self._v2_phrase_texts_match(clean, requested)
):
return True
requested_roots = self._v2_repair_intent_roots(requested)
overlap = clean_roots & requested_roots
if not overlap:
continue
requested_axes = self._v2_text_axes(requested)
if "action" in clean_axes and "action" in requested_axes:
return True
if len(overlap) >= 2 and (
len(overlap) / float(max(1, min(len(clean_roots), len(requested_roots)))) >= 0.45
):
return True
return False
def _v2_mark_intent_locks(self, phrases, intent_prompt="", intent_phrases=None, family_slot=None):
if not phrases:
return phrases
for phrase in phrases:
if not isinstance(phrase, dict):
continue
text = phrase.get("text", "")
locked = self._v2_text_represented_by_intent(text, intent_prompt, intent_phrases, None)
preference_locked = self._v2_text_represented_by_intent(text, "", [], family_slot) and not locked
phrase["intent_locked"] = bool(locked)
phrase["preference_locked"] = bool(preference_locked)
return phrases
def _v2_serialized_delta(self, target_payload, source_payload):
if not isinstance(target_payload, dict) or not isinstance(source_payload, dict):
return None
try:
target = serializable_to_tensor(target_payload).detach().float()
source = serializable_to_tensor(source_payload).detach().float()
if list(target.shape) != list(source.shape):
return None
return tensor_to_serializable((target - source).cpu())
except Exception:
return None
def _v2_store_conditioning_delta_average(self, slot, delta_payload):
if not isinstance(slot, dict) or not isinstance(delta_payload, dict):
return False
count = int(slot.get("count", 0))
existing = slot.get("delta")
if count <= 0 or not isinstance(existing, dict):
slot["delta"] = delta_payload
slot["count"] = 1
return True
try:
previous = serializable_to_tensor(existing)
current = serializable_to_tensor(delta_payload).to(previous.device)
if list(previous.shape) != list(current.shape):
slot["delta"] = delta_payload
slot["count"] = 1
return True
averaged = (previous * count + current) / float(count + 1)
slot["delta"] = tensor_to_serializable(averaged.cpu())
slot["count"] = count + 1
return True
except Exception:
slot["delta"] = delta_payload
slot["count"] = 1
return True
def _v2_update_appearance_anchor(self, global_state, previous_run, rating_profile, iter_num):
"""Consistency anchor for the "Wrong appearance" rating. global_state is already
per-refinement-key, so these live with that key automatically.
On a good rating, remember the rated run's conditioning as the "blessed appearance"
(the last good character). On a Wrong-appearance rating, remember the rejected run's
conditioning as the "drift" to repel from. _v2_apply_conditioning_memory_impl pulls the
next gen toward the blessed appearance and away from the drift when the user rated the
previous gen Wrong appearance."""
if not isinstance(global_state, dict) or not isinstance(previous_run, dict):
return "Appearance anchor: no update."
if rating_profile.get("skip_learning"):
return "Appearance anchor: no update."
conditioning = previous_run.get("conditioning")
if not isinstance(conditioning, dict):
return "Appearance anchor: run carried no conditioning."
key = rating_profile.get("key", "")
is_good = key in {"like", "nailed_it", "loved_it"} or bool(rating_profile.get("loved_modifier"))
if is_good:
existing = global_state.get("appearance_anchor") or {}
global_state["appearance_anchor"] = {
"conditioning": conditioning,
"prompt": self._v2_prompt_key(previous_run.get("prompt", "")),
"count": int(existing.get("count", 0)) + 1,
"last_seen_iter": int(iter_num),
}
return "Appearance anchor: blessed appearance stored."
if key == "wrong_appearance":
global_state["appearance_drift"] = {
"conditioning": conditioning,
"prompt": self._v2_prompt_key(previous_run.get("prompt", "")),
"last_seen_iter": int(iter_num),
}
return "Appearance anchor: drift stored (repel on next gen)."
return "Appearance anchor: no update."
def _v2_update_intent_family_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
if not isinstance(global_state, dict) or not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
return "Intent family: no learning update.", ""
source_intent = self._v2_intent_source_prompt(
last_run.get("prompt", ""),
last_run.get("intent_prompt", ""),
bool(last_run.get("intent_prompt_is_vague")),
)
if not source_intent:
return "Intent family: no prompt body.", ""
family_key, slot, similarity = self._v2_intent_family_slot(global_state, source_intent, create=True)
if not isinstance(slot, dict):
return "Intent family: unavailable.", ""
slot["family_key"] = family_key
slot["intent_prompt"] = slot.get("intent_prompt") or source_intent
prompts = list(slot.get("intent_prompts", [])) if isinstance(slot.get("intent_prompts"), list) else []
if source_intent not in prompts:
prompts.append(source_intent)
slot["intent_prompts"] = prompts[-12:]
slot["total_seen"] = int(slot.get("total_seen", 0)) + 1
slot["last_seen_iter"] = int(iter_num)
intent_phrases = [phrase for phrase in last_run.get("intent_phrases", []) or [] if isinstance(phrase, dict)]
if not intent_phrases:
intent_phrases = self._v2_classify_phrases(None, self._ordered_prompt_phrases(source_intent), global_state)
positive_phrases = [phrase for phrase in last_run.get("phrases", []) or [] if isinstance(phrase, dict)]
self._v2_mark_intent_locks(positive_phrases, source_intent, intent_phrases, slot)
preferences = slot.setdefault("intent_preference_phrases", {})
preference_updates = 0
for phrase in intent_phrases:
payload = self._v2_intent_alignment_phrase_payload(phrase)
if not payload:
continue
entry = preferences.setdefault(payload["text"], {
**payload,
"seen_count": 0,
"repair_count": 0,
"score": 0.0,
"last_seen_iter": 0,
})
entry.update(payload)
entry["seen_count"] = int(entry.get("seen_count", 0)) + 1
entry["last_seen_iter"] = int(iter_num)
if int(entry.get("seen_count", 0)) >= 2:
entry["score"] = round(min(4.0, float(entry.get("score", 0.0)) + 0.36), 4)
preference_updates += 1
else:
entry["score"] = round(min(4.0, float(entry.get("score", 0.0)) + 0.12), 4)
has_semantic_intent_lock = any(
bool(phrase.get("semantic_intent_locked"))
for phrase in positive_phrases
if isinstance(phrase, dict)
)
missing_intent = [
phrase for phrase in intent_phrases
if (
not has_semantic_intent_lock and
not self._v2_phrase_represented_by(phrase.get("text", ""), positive_phrases)
)
]
extra_positive = [
phrase for phrase in positive_phrases
if (
not phrase.get("intent_locked") and
not phrase.get("semantic_intent_locked") and
not self._v2_text_represented_by_intent(phrase.get("text", ""), source_intent, intent_phrases, None)
)
]
repairs = [
item for item in last_run.get("repair_candidates", []) or []
if isinstance(item, dict) and self._v2_clean_phrase_text(item.get("text", ""))
]
variant_key = md5(self._v2_prompt_key(last_run.get("prompt", "")).encode("utf-8")).hexdigest()
variants = slot.setdefault("variant_evidence", {})
variant = variants.setdefault(variant_key, {
"positive_prompt": self._v2_prompt_key(last_run.get("prompt", "")),
"encoded_prompt": self._v2_prompt_key(last_run.get("encoded_prompt", "")),
"rating_count": 0,
"avg_reward": 0.0,
"accepted_count": 0,
"rejected_count": 0,
"missing_intent": [],
"extra_positive": [],
"repair_candidates": [],
"last_rating_label": "",
"last_seen_iter": 0,
})
count = int(variant.get("rating_count", 0))
reward = float(rating_profile.get("reward", 0.0))
variant["avg_reward"] = round((float(variant.get("avg_reward", 0.0)) * count + reward) / float(count + 1), 6)
variant["rating_count"] = count + 1
variant["last_rating_label"] = rating_profile.get("label", "")
variant["last_seen_iter"] = int(iter_num)
variant["missing_intent"] = [item.get("text", "") for item in missing_intent]
variant["extra_positive"] = [item.get("text", "") for item in extra_positive]
variant["repair_candidates"] = [item.get("text", "") for item in repairs]
if rating_profile.get("key") == "like":
variant["accepted_count"] = int(variant.get("accepted_count", 0)) + 1
elif reward < 0.0 or rating_profile.get("wrong_axes") or rating_profile.get("key") == "awful":
variant["rejected_count"] = int(variant.get("rejected_count", 0)) + 1
anchor_count = 0
delta_updates = 0
if rating_profile.get("key") == "like" and isinstance(last_run.get("conditioning"), dict):
anchor_payload = {
"intent_prompt": source_intent,
"positive_prompt": self._v2_prompt_key(last_run.get("prompt", "")),
"encoded_prompt": self._v2_prompt_key(last_run.get("encoded_prompt", "")),
"conditioning": last_run.get("conditioning"),
"source_conditioning": last_run.get("source_conditioning"),
"repair_candidates": [item.get("text", "") for item in repairs],
"last_seen_iter": int(iter_num),
}
anchors = slot.setdefault("perfect_anchors", {})
if "base" not in anchors:
anchors["base"] = dict(anchor_payload, anchor_role="base")
else:
loved = slot.setdefault("loved_variants", {})
loved[variant_key] = dict(anchor_payload, anchor_role="variant")
if len(loved) > 16:
slot["loved_variants"] = dict(sorted(
loved.items(),
key=lambda item: int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
reverse=True,
)[:16])
anchor_count = 1 + len(slot.get("loved_variants", {}))
delta = self._v2_serialized_delta(last_run.get("conditioning"), last_run.get("source_conditioning"))
if isinstance(delta, dict):
deltas = slot.setdefault("conditioning_deltas", {})
positive_delta = deltas.setdefault("positive", {})
if self._v2_store_conditioning_delta_average(positive_delta, delta):
positive_delta["last_seen_iter"] = int(iter_num)
delta_updates += 1
if len(variants) > 40:
slot["variant_evidence"] = dict(sorted(
variants.items(),
key=lambda item: (
float(item[1].get("avg_reward", 0.0)) if isinstance(item[1], dict) else -99.0,
int(item[1].get("rating_count", 0)) if isinstance(item[1], dict) else 0,
int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
),
reverse=True,
)[:40])
self._v2_sync_intent_family_aliases(global_state, family_key, slot)
return (
"Intent family learned: "
f"family={family_key[:8]} sim={similarity:.2f}, repeated preference(s)={preference_updates}, "
f"anchors={anchor_count or len(slot.get('perfect_anchors', {})) + len(slot.get('loved_variants', {}))}, "
f"variant reward {variant['avg_reward']:+.2f}, delta update(s)={delta_updates}."
), family_key
def _v2_apply_intent_family_delta(self, conditioning, family_slot, strength):
if not isinstance(conditioning, torch.Tensor) or not isinstance(family_slot, dict):
return conditioning, "intent-family idle"
positive_delta = family_slot.get("conditioning_deltas", {}).get("positive", {})
payload = positive_delta.get("delta") if isinstance(positive_delta, dict) else None
if not self._v2_shape_compatible(payload, conditioning):
return conditioning, "intent-family idle"
try:
delta = serializable_to_tensor(payload).to(device=conditioning.device, dtype=conditioning.dtype)
if list(delta.shape) != list(conditioning.shape):
return conditioning, "intent-family idle"
original_norm = conditioning.norm(dim=-1, keepdim=True).clamp_min(1e-8)
max_delta = original_norm * min(0.030, max(0.006, float(strength) * 0.35))
delta_norm = delta.norm(dim=-1, keepdim=True).clamp_min(1e-8)
scale = torch.minimum(torch.ones_like(delta_norm), max_delta / delta_norm)
return conditioning + delta * scale, f"intent-family positive delta x{int(positive_delta.get('count', 0))}"
except Exception:
return conditioning, "intent-family delta failed"
def _v2_intent_alignment_phrase_payload(self, phrase):
text = self._v2_clean_phrase_text(phrase.get("text", "") if isinstance(phrase, dict) else phrase)
if not text:
return None
source = phrase if isinstance(phrase, dict) else {}
scores = source.get("effective_category_scores", source.get("category_scores", self._v2_heuristic_scores(text)))
axes = self._v2_order_axes(self._v2_axes_for_scores(scores))
return {
"text": text,
"axes": axes,
"primary": source.get("primary", self._v2_primary_category_for_text(text)),
"effective_category_scores": self._v2_clean_category_scores(scores),
}
def _v2_update_intent_alignment_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
if not isinstance(global_state, dict) or not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
return "Intent alignment: no learning update."
intent_prompt = self._v2_prompt_key(last_run.get("intent_prompt", ""))
if not intent_prompt or last_run.get("intent_prompt_is_vague") or self._v2_user_intent_prompt_is_vague(intent_prompt):
return "Intent alignment: no explicit original intent."
positive_prompt = self._v2_prompt_key(last_run.get("prompt", ""))
positive_phrases = [phrase for phrase in last_run.get("phrases", []) or [] if isinstance(phrase, dict)]
intent_phrases = [phrase for phrase in last_run.get("intent_phrases", []) or [] if isinstance(phrase, dict)]
if not intent_phrases:
intent_phrases = self._v2_classify_phrases(None, self._ordered_prompt_phrases(intent_prompt), global_state)
if not positive_phrases or not intent_phrases:
return "Intent alignment: no comparable prompt phrases."
has_semantic_intent_lock = any(
bool(phrase.get("semantic_intent_locked"))
for phrase in positive_phrases
if isinstance(phrase, dict)
)
missing_intent = [
phrase for phrase in intent_phrases
if (
not has_semantic_intent_lock and
not self._v2_phrase_represented_by(phrase.get("text", ""), positive_phrases)
)
]
extra_positive = [
phrase for phrase in positive_phrases
if (
not phrase.get("intent_locked") and
not phrase.get("semantic_intent_locked") and
not self._v2_phrase_represented_by(phrase.get("text", ""), intent_phrases) and
not self._v2_prompt_contains_text(intent_prompt, phrase.get("text", ""))
)
]
memory = global_state.setdefault("intent_alignment_memory", {})
intent_key = self._v2_intent_alignment_key(intent_prompt)
slot = memory.setdefault(intent_key, {
"intent_prompt": intent_prompt,
"variants": {},
"intent_enhance_pairs": {},
"intent_tokens": {},
"provided_tokens": {},
"bad_tokens": {},
"missing_intent_phrases": {},
"extra_positive_phrases": {},
"last_seen_iter": 0,
})
slot["intent_prompt"] = intent_prompt
slot["last_seen_iter"] = int(iter_num)
_, family_slot, _ = self._v2_intent_family_slot(global_state, intent_prompt, create=False)
has_perfect_anchor = bool(
isinstance(family_slot, dict) and (
family_slot.get("perfect_anchors") or family_slot.get("loved_variants")
)
)
reward = float(rating_profile.get("reward", 0.0))
rating_key = rating_profile.get("key", "")
axis_feedback = axis_feedback or self._v2_axis_feedback(
rating_profile,
global_state.get("last_missing_axes", []),
)
missing_axes = set(axis_feedback.get("missing_axes", []))
wrong_axes = set(axis_feedback.get("wrong_axes", []))
variant_key = md5(positive_prompt.encode("utf-8")).hexdigest()
variants = slot.setdefault("variants", {})
variant = variants.setdefault(variant_key, {
"positive_prompt": positive_prompt,
"rating_count": 0,
"avg_reward": 0.0,
"missing_intent_count": 0,
"extra_positive_count": 0,
"last_rating_label": "",
"last_seen_iter": 0,
})
count = int(variant.get("rating_count", 0))
variant["avg_reward"] = round((float(variant.get("avg_reward", 0.0)) * count + reward) / float(count + 1), 6)
variant["rating_count"] = count + 1
variant["missing_intent_count"] = len(missing_intent)
variant["extra_positive_count"] = len(extra_positive)
variant["last_rating_label"] = rating_profile.get("label", "")
variant["last_seen_iter"] = int(iter_num)
pairs = slot.setdefault("intent_enhance_pairs", {})
pairs[variant_key] = {
"intent_prompt": intent_prompt,
"positive_prompt": positive_prompt,
"rating_count": int(variant["rating_count"]),
"avg_reward": float(variant["avg_reward"]),
"missing_intent_count": int(variant["missing_intent_count"]),
"extra_positive_count": int(variant["extra_positive_count"]),
"last_rating_label": variant["last_rating_label"],
"last_seen_iter": int(iter_num),
}
intent_token_memory = slot.setdefault("intent_tokens", {})
intent_token_set = set()
for unit in self._v2_alignment_token_units(intent_prompt):
token = unit["token"]
intent_token_set.add(token)
entry = intent_token_memory.setdefault(token, {
"token": token,
"kind": unit["kind"],
"words": list(unit["words"]),
"seen_count": 0,
"last_seen_iter": 0,
})
entry["kind"] = unit["kind"]
entry["words"] = list(unit["words"])
entry["seen_count"] = int(entry.get("seen_count", 0)) + 1
entry["last_seen_iter"] = int(iter_num)
provided_token_memory = slot.setdefault("provided_tokens", {})
bad_token_memory = slot.setdefault("bad_tokens", {})
token_updates = 0
for phrase in positive_phrases:
text = self._v2_clean_phrase_text(phrase.get("text", ""))
if not text:
continue
phrase_axes = self._v2_phrase_axes(phrase)
phrase_is_blocked_extra = (
not phrase.get("intent_locked") and
not phrase.get("semantic_intent_locked") and
not self._v2_phrase_represented_by(text, intent_phrases) and
not self._v2_prompt_contains_text(intent_prompt, text)
)
phrase_is_bad_extra = (
rating_key == "awful" or
(rating_key == "wrong_appearance" and self._v2_item_has_blocked_auto_category(phrase)) or
bool(wrong_axes & phrase_axes) or
(float(reward) < -0.35 and phrase_is_blocked_extra)
)
for unit in self._v2_alignment_token_units(text):
token = unit["token"]
entry = provided_token_memory.setdefault(token, {
"token": token,
"kind": unit["kind"],
"words": list(unit["words"]),
"score": 0.0,
"seen_count": 0,
"accepted_count": 0,
"rejected_count": 0,
"intent_count": 0,
"enhancer_only_count": 0,
"phrases": [],
"variants": {},
"omit": False,
"last_seen_iter": 0,
})
entry["kind"] = unit["kind"]
entry["words"] = list(unit["words"])
entry["seen_count"] = int(entry.get("seen_count", 0)) + 1
entry["last_seen_iter"] = int(iter_num)
entry.setdefault("variants", {})[variant_key] = int(entry.setdefault("variants", {}).get(variant_key, 0)) + 1
phrases_for_token = list(entry.get("phrases", []))
if text not in phrases_for_token:
phrases_for_token.append(text)
entry["phrases"] = phrases_for_token[-12:]
if token in intent_token_set:
entry["intent_count"] = int(entry.get("intent_count", 0)) + 1
delta = 0.06 if rating_key == "like" else (0.03 if rating_key == "nailed_it" else 0.0)
else:
entry["enhancer_only_count"] = int(entry.get("enhancer_only_count", 0)) + 1
if rating_key == "like":
delta = 0.12
entry["accepted_count"] = int(entry.get("accepted_count", 0)) + 1
elif rating_key == "nailed_it":
delta = 0.07
elif rating_key == "loved_it":
delta = 0.07
elif phrase_is_bad_extra:
delta = -0.60 if rating_key == "wrong_appearance" else -0.36
entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
else:
delta = max(-0.06, min(0.08, reward * 0.06))
entry["score"] = round(max(-6.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
entry["omit"] = (
token not in intent_token_set and
float(entry.get("score", 0.0)) <= -0.50 and
int(entry.get("rejected_count", 0)) > int(entry.get("accepted_count", 0))
)
if entry["omit"]:
bad_token_memory[token] = {
"token": token,
"kind": unit["kind"],
"words": list(unit["words"]),
"score": float(entry.get("score", 0.0)),
"rejected_count": int(entry.get("rejected_count", 0)),
"accepted_count": int(entry.get("accepted_count", 0)),
"phrases": list(entry.get("phrases", []))[-8:],
"last_seen_iter": int(iter_num),
}
else:
bad_token_memory.pop(token, None)
token_updates += 1
missing_updates = 0
for phrase in missing_intent:
payload = self._v2_intent_alignment_phrase_payload(phrase)
if not payload:
continue
axes = set(payload.get("axes", []))
entry = slot.setdefault("missing_intent_phrases", {}).setdefault(payload["text"], {
**payload,
"score": 0.0,
"missing_count": 0,
"forgiven_count": 0,
"last_seen_iter": 0,
})
if rating_key == "like":
delta = -0.18
entry["forgiven_count"] = int(entry.get("forgiven_count", 0)) + 1
elif rating_key == "nailed_it":
delta = -0.10
entry["forgiven_count"] = int(entry.get("forgiven_count", 0)) + 1
elif rating_profile.get("loved_modifier"):
pass # loved_modifier: full axis pressure still applies — fix the issue, quality signal comes from reward/conditioning
elif rating_key == "loved_it":
continue # axis-blind quality endorsement — don't learn prompt adherence
elif rating_key == "awful":
delta = 0.70 if has_perfect_anchor else 0.44
entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
elif missing_axes:
delta = 0.95 if axes & missing_axes else 0.28
if not has_perfect_anchor:
delta = min(delta, 0.58)
entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
elif wrong_axes:
delta = 0.42 if axes & wrong_axes else 0.16
if not has_perfect_anchor:
delta = min(delta, 0.30)
entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
else:
delta = -0.10 if reward > 0.35 else 0.18 if reward < -0.20 else 0.0
entry.update(payload)
entry["score"] = round(max(-2.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
entry["last_seen_iter"] = int(iter_num)
missing_updates += 1
extra_updates = 0
for phrase in extra_positive:
payload = self._v2_intent_alignment_phrase_payload(phrase)
if not payload:
continue
axes = set(payload.get("axes", []))
entry = slot.setdefault("extra_positive_phrases", {}).setdefault(payload["text"], {
**payload,
"score": 0.0,
"accepted_count": 0,
"rejected_count": 0,
"last_seen_iter": 0,
})
if rating_key == "like":
delta = 0.42
entry["accepted_count"] = int(entry.get("accepted_count", 0)) + 1
elif rating_key == "wrong_appearance" and self._v2_item_has_blocked_auto_category(payload):
delta = -0.95
entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
elif rating_key == "awful":
delta = -0.78
entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
elif wrong_axes:
delta = -0.70 if axes & wrong_axes else -0.25
entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
elif reward < -0.35:
delta = -0.46
entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
else:
delta = max(-0.08, min(0.12, reward * 0.10))
entry.update(payload)
entry["score"] = round(max(-6.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
entry["last_seen_iter"] = int(iter_num)
extra_updates += 1
if len(variants) > 40:
slot["variants"] = dict(sorted(
variants.items(),
key=lambda item: (
float(item[1].get("avg_reward", 0.0)) if isinstance(item[1], dict) else -99.0,
int(item[1].get("rating_count", 0)) if isinstance(item[1], dict) else 0,
int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
),
reverse=True,
)[:40])
if memory and len(memory) > 80:
global_state["intent_alignment_memory"] = dict(sorted(
memory.items(),
key=lambda item: int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
reverse=True,
)[:80])
return (
"Intent alignment learned: "
f"{missing_updates} omitted original phrase(s), {extra_updates} enhancer-only phrase(s), "
f"{token_updates} provided token(s), {len(bad_token_memory)} omit token(s), "
f"variant reward {variant['avg_reward']:+.2f}."
)
def _v2_apply_intent_alignment_memory(self, prompt, current_phrases, intent_prompt, intent_phrases, global_state):
intent_prompt = self._v2_prompt_key(intent_prompt)
if not intent_prompt or self._v2_user_intent_prompt_is_vague(intent_prompt):
return prompt, "Intent alignment: no explicit original intent.", []
memory = global_state.get("intent_alignment_memory", {}) if isinstance(global_state, dict) else {}
slot = memory.get(self._v2_intent_alignment_key(intent_prompt)) if isinstance(memory, dict) else None
if not isinstance(slot, dict):
return prompt, "Intent alignment: no learned enhancer variants for this original prompt.", []
intent_phrases = [phrase for phrase in intent_phrases or [] if isinstance(phrase, dict)]
current_phrases = [phrase for phrase in current_phrases or [] if isinstance(phrase, dict)]
additions = []
for entry in sorted(
slot.get("missing_intent_phrases", {}).values(),
key=lambda item: (
float(item.get("score", 0.0)) if isinstance(item, dict) else -99.0,
int(item.get("missing_count", 0)) if isinstance(item, dict) else 0,
),
reverse=True,
):
if not isinstance(entry, dict):
continue
text = self._v2_clean_phrase_text(entry.get("text", ""))
if (
not text or
float(entry.get("score", 0.0)) < 0.55 or
self._v2_prompt_contains_text(prompt, text) or
self._v2_phrase_represented_by(text, current_phrases) or
not self._v2_phrase_represented_by(text, intent_phrases)
):
continue
additions.append(text)
if len(additions) >= 4:
break
bad_extra_texts = []
for entry in slot.get("extra_positive_phrases", {}).values():
if not isinstance(entry, dict):
continue
text = self._v2_clean_phrase_text(entry.get("text", ""))
if (
text and
float(entry.get("score", 0.0)) <= -0.55 and
int(entry.get("rejected_count", 0)) > int(entry.get("accepted_count", 0)) and
not self._v2_phrase_represented_by(text, intent_phrases)
):
bad_extra_texts.append(text)
bad_extra_tokens = {
str(token)
for token, entry in slot.get("bad_tokens", {}).items()
if isinstance(entry, dict) and bool(entry.get("token"))
}
intent_token_set = {unit["token"] for unit in self._v2_alignment_token_units(intent_prompt)}
removed = []
segments = [segment.strip() for segment in re.split(r"[,;\n]+", str(prompt or "")) if segment.strip()]
if segments and (bad_extra_texts or bad_extra_tokens):
kept = []
for segment in segments:
segment_is_intent = self._v2_phrase_represented_by(segment, intent_phrases)
segment_is_bad_extra = any(
self._v2_phrase_texts_match(segment, extra_text) or self._v2_prompt_contains_text(segment, extra_text)
for extra_text in bad_extra_texts
)
segment_tokens = {unit["token"] for unit in self._v2_alignment_token_units(segment)}
segment_has_bad_token = bool((segment_tokens - intent_token_set) & bad_extra_tokens)
if (segment_is_bad_extra or segment_has_bad_token) and not segment_is_intent:
removed.append(segment)
continue
kept.append(segment)
prompt = ", ".join(kept) if kept else str(prompt or "").strip()
if additions:
prompt = f"{prompt}, {', '.join(additions)}" if str(prompt or "").strip() else ", ".join(additions)
if not additions and not removed:
return prompt, "Intent alignment: learned variants present, no prompt changes needed.", []
adjustments = [
{"text": text, "source": "intent_missing", "action": "added"}
for text in additions
] + [
{"text": text, "source": "enhancer_extra", "action": "removed"}
for text in removed
]
return (
prompt,
f"Intent alignment: restored {len(additions)} original phrase(s), removed {len(removed)} rejected enhancer-only phrase(s).",
adjustments,
)
def _v2_context_cluster_key(self, text, context):
signature = self._v2_context_signature(context)
key = f"{text}|{signature}" if signature else text
return md5(key.encode("utf-8")).hexdigest()
def _v2_memory_kind_scale(self, kind):
if kind in {"prompt_phrase", "phrase"}:
return 1.0
if kind == "auto_phrase":
return 0.72
if kind == "repair_candidate":
return 0.64
if kind == "ngram":
return 0.62
if kind == "token":
return 0.24
return 0.50
def _v2_update_preferred_context_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
if not isinstance(global_state, dict) or not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
return "Preferred context: no update."
if rating_profile.get("key") == "wrong_appearance":
return "Preferred context: skipped for wrong appearance."
phrases = [phrase for phrase in last_run.get("phrases", []) or [] if isinstance(phrase, dict)]
if not phrases:
return "Preferred context: no prompt phrases."
axis_feedback = axis_feedback or self._v2_axis_feedback(
rating_profile,
global_state.get("last_missing_axes", []),
)
positive_axes = set()
if rating_profile.get("key") == "like":
positive_axes = {"action", "details"}
base_delta = 1.0
elif rating_profile.get("key") == "nailed_it":
positive_axes = {"action", "details"}
base_delta = 0.65
elif rating_profile.get("loved_modifier"):
positive_axes = {"action", "details"}
base_delta = 0.30
elif rating_profile.get("key") == "loved_it":
return "Preferred context: skipped for axis-blind quality endorsement."
else:
positive_axes = (set(axis_feedback.get("satisfied_axes", [])) | set(axis_feedback.get("resolved_axes", []))) & {"action", "details"}
base_delta = 0.38
if not positive_axes:
return "Preferred context: no liked action/detail axes."
memory = global_state.setdefault("preferred_context_memory", {})
touched = []
for index, phrase in enumerate(phrases):
text = self._v2_clean_phrase_text(phrase.get("text", ""))
axes = self._v2_phrase_axes(phrase) & positive_axes
if not text or not axes:
continue
start = max(0, index - 3)
end = min(len(phrases), index + 4)
cluster = []
for pos in range(start, end):
item_text = self._v2_clean_phrase_text(phrases[pos].get("text", ""))
if item_text and item_text not in cluster:
cluster.append(item_text)
if text not in cluster:
cluster.insert(min(index - start, len(cluster)), text)
context = phrase.get("context") or self._v2_context_for_phrase(phrases, index, text, window=3)
key = self._v2_context_cluster_key(text, context)
entry = memory.setdefault(key, {
"anchor": text,
"axes": {},
"context": context,
"phrases": [],
"neighbors": [],
"score": 0.0,
"liked_count": 0,
"satisfied_count": 0,
"last_seen_iter": 0,
})
entry["anchor"] = text
entry["context"] = context
entry["score"] = round(max(-2.0, min(12.0, float(entry.get("score", 0.0)) + base_delta)), 4)
if rating_profile.get("key") == "like":
entry["liked_count"] = int(entry.get("liked_count", 0)) + 1
else:
entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
entry["last_seen_iter"] = int(iter_num)
axis_counts = entry.setdefault("axes", {})
for axis in self._v2_order_axes(axes):
axis_counts[axis] = int(axis_counts.get(axis, 0)) + 1
existing_phrases = list(entry.get("phrases", []))
for item_text in cluster:
if item_text not in existing_phrases:
existing_phrases.append(item_text)
entry["phrases"] = existing_phrases[:12]
entry["neighbors"] = [item for item in entry["phrases"] if item != text][:10]
memory[key] = entry
touched.append(text)
if len(memory) > 160:
memory = dict(sorted(
memory.items(),
key=lambda item: (
float(item[1].get("score", 0.0)) if isinstance(item[1], dict) else -99.0,
int(item[1].get("liked_count", 0)) if isinstance(item[1], dict) else 0,
int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
),
reverse=True,
)[:160])
global_state["preferred_context_memory"] = memory
if not touched:
return "Preferred context: no action/detail clusters matched."
return f"Preferred context stored: {len(touched)} action/detail cluster(s): {', '.join(touched[:5])}{'...' if len(touched) > 5 else ''}."
def _v2_update_phrase_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
if not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
return "Lucky memory: no learning update."
memory = global_state.setdefault("phrase_memory", {})
concepts = self._v2_concept_units_for_run(last_run)
axis_feedback = axis_feedback or self._v2_axis_feedback(
rating_profile,
global_state.get("last_missing_axes", []),
)
missing_axes = set(axis_feedback.get("missing_axes", []))
satisfied_axes = set(axis_feedback.get("satisfied_axes", []))
resolved_axes = set(axis_feedback.get("resolved_axes", []))
regressed_axes = set(axis_feedback.get("regressed_axes", []))
wrong_axes = set(axis_feedback.get("wrong_axes", []))
reward = float(rating_profile.get("reward", 0.0))
wrong_appearance = rating_profile.get("key") == "wrong_appearance"
original_prompt = str(last_run.get("prompt", "") or "")
preferred_context_status = self._v2_update_preferred_context_memory(
global_state,
last_run,
rating_profile,
iter_num,
axis_feedback,
)
touched = []
trained = []
kind_counts = {}
suppressed = []
for phrase in concepts:
text = str(phrase.get("text", "")).strip().lower()
if not text:
continue
entry = self._v2_ensure_phrase_memory_entry(memory, text, phrase)
entry["kind"] = phrase.get("kind", entry.get("kind", "phrase"))
kind_counts[entry["kind"]] = int(kind_counts.get(entry["kind"], 0)) + 1
entry["machine_primary"] = phrase.get("machine_primary", entry.get("machine_primary", entry.get("primary", "details")))
entry["clip_heuristic_scores"] = self._v2_clean_category_scores(
phrase.get("clip_heuristic_scores", phrase.get("category_scores", entry.get("clip_heuristic_scores", {})))
)
context = phrase.get("context", {})
context_sense = self._v2_ensure_context_sense(entry, context)
if isinstance(context_sense, dict):
context_sense["occurrence_count"] = int(context_sense.get("occurrence_count", 0)) + 1
context_sense["last_seen_iter"] = int(iter_num)
entry["tokens"] = phrase.get("tokens", entry.get("tokens", []))
entry["occurrence_count"] = int(entry.get("occurrence_count", 0)) + 1
entry.setdefault("first_seen_iter", int(iter_num))
entry.setdefault("positions", {})[str(int(phrase.get("position", 0)))] = (
int(entry.setdefault("positions", {}).get(str(int(phrase.get("position", 0))), 0)) + 1
)
kind_scale = self._v2_memory_kind_scale(entry.get("kind", "phrase"))
machine_scores = self._v2_clean_category_scores(entry.get("clip_heuristic_scores", {}))
weights = self._v2_clean_category_scores(entry.get("category_weights", {}))
effective_before = self._v2_effective_category_scores(
machine_scores,
weights,
int(entry.get("category_evidence_count", 0)),
)
phrase_axes = self._v2_axes_for_scores(effective_before)
matched_missing = phrase_axes & missing_axes
matched_satisfied = phrase_axes & satisfied_axes
matched_resolved = phrase_axes & resolved_axes
matched_regressed = phrase_axes & regressed_axes
matched_wrong = phrase_axes & wrong_axes
rating_evidence = entry.setdefault("rating_evidence", {})
weights_before = dict(weights)
count_category_evidence = rating_profile.get("key") != "discover"
def bump_evidence(name, axes):
axis_counts = rating_evidence.setdefault(name, {})
for axis in self._v2_order_axes(axes):
axis_counts[axis] = int(axis_counts.get(axis, 0)) + 1
def train_axis(axis, amount, prefer_current=True):
for category in self._v2_axis_categories(axis):
relation = self._v2_axis_category_weight(axis, category, effective_before if prefer_current else machine_scores)
weights[category] = float(weights.get(category, 0.0)) + amount * relation
if wrong_appearance:
explicit = self._v2_prompt_contains_text(original_prompt, text)
auto_source = phrase.get("kind") in {"auto_phrase", "repair_candidate"}
should_suppress = self._v2_item_has_blocked_auto_category(phrase) and (auto_source or not explicit)
if should_suppress:
delta = -0.95 * kind_scale
entry["wrong_appearance_count"] = int(entry.get("wrong_appearance_count", 0)) + 1
entry["auto_inject_blocked_count"] = int(entry.get("auto_inject_blocked_count", 0)) + 1
entry["auto_inject_suppressed"] = True
rating_evidence["wrong_appearance"] = int(rating_evidence.get("wrong_appearance", 0)) + 1
for category in self.AUTO_INJECT_BLOCKED_CATEGORIES:
weights[category] = float(weights.get(category, 0.0)) - 0.30 * kind_scale
suppressed.append(text)
else:
delta = 0.0
count_category_evidence = False
elif rating_profile.get("loved_modifier"):
delta = 0.45 * kind_scale
entry["liked_count"] = int(entry.get("liked_count", 0)) + 1
rating_evidence["liked"] = int(rating_evidence.get("liked", 0)) + 1
for category, score in effective_before.items():
if float(score) >= 0.28:
weights[category] = float(weights.get(category, 0.0)) + 0.10 * kind_scale
elif rating_profile.get("key") == "nailed_it":
delta = 0.35 * kind_scale
entry["liked_count"] = int(entry.get("liked_count", 0)) + 1
rating_evidence["liked"] = int(rating_evidence.get("liked", 0)) + 1
entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
for axis in V2_FEEDBACK_AXES:
satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
bump_evidence("satisfied_axes", V2_FEEDBACK_AXES)
for category, score in effective_before.items():
if float(score) >= 0.28:
weights[category] = float(weights.get(category, 0.0)) + 0.08 * kind_scale
elif rating_profile.get("key") == "loved_it":
delta = 0.35 * kind_scale
entry["liked_count"] = int(entry.get("liked_count", 0)) + 1
rating_evidence["liked"] = int(rating_evidence.get("liked", 0)) + 1
entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
for axis in V2_FEEDBACK_AXES:
satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
bump_evidence("satisfied_axes", V2_FEEDBACK_AXES)
for category, score in effective_before.items():
if float(score) >= 0.28:
weights[category] = float(weights.get(category, 0.0)) + 0.08 * kind_scale
elif rating_profile.get("key") == "like":
delta = 0.55 * kind_scale
entry["liked_count"] = int(entry.get("liked_count", 0)) + 1
rating_evidence["liked"] = int(rating_evidence.get("liked", 0)) + 1
entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
for axis in V2_FEEDBACK_AXES:
satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
bump_evidence("satisfied_axes", V2_FEEDBACK_AXES)
for category, score in effective_before.items():
if float(score) >= 0.28:
weights[category] = float(weights.get(category, 0.0)) + 0.13 * kind_scale
for axis in V2_FEEDBACK_AXES:
train_axis(axis, 0.035 * kind_scale)
elif rating_profile.get("key") == "awful":
delta = -0.90 * kind_scale
entry["bad_count"] = int(entry.get("bad_count", 0)) + 1
rating_evidence["bad"] = int(rating_evidence.get("bad", 0)) + 1
primary_before, _ = self._v2_scores_primary(effective_before)
weights[primary_before] = float(weights.get(primary_before, 0.0)) - 0.045 * kind_scale
bump_evidence("missing_axes", missing_axes)
for axis in self._v2_order_axes(missing_axes):
train_axis(axis, 0.045 * kind_scale, prefer_current=False)
elif wrong_axes:
if matched_wrong:
delta = -0.18 * kind_scale
entry["wrong_count"] = int(entry.get("wrong_count", 0)) + 1
wrong_axis_counts = entry.setdefault("wrong_axes", {})
for axis in self._v2_order_axes(matched_wrong):
wrong_axis_counts[axis] = int(wrong_axis_counts.get(axis, 0)) + 1
train_axis(axis, 0.12 * kind_scale, prefer_current=False)
primary_before, _ = self._v2_scores_primary(effective_before)
weights[primary_before] = float(weights.get(primary_before, 0.0)) - 0.055 * kind_scale
elif matched_satisfied:
delta = 0.18 * kind_scale
entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
for axis in self._v2_order_axes(matched_satisfied):
train_axis(axis, 0.070 * kind_scale)
else:
delta = 0.02 * kind_scale
wanted_axes = entry.setdefault("wanted_axes", {})
for axis in self._v2_order_axes(wrong_axes):
wanted_axes[axis] = int(wanted_axes.get(axis, 0)) + 1
satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
for axis in self._v2_order_axes(matched_satisfied):
satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
bump_evidence("wrong_axes", wrong_axes)
bump_evidence("satisfied_axes", satisfied_axes)
elif missing_axes:
for axis in self._v2_order_axes(missing_axes - matched_missing):
train_axis(axis, 0.14 * kind_scale, prefer_current=False)
if matched_missing:
delta = (0.34 + (0.08 if matched_regressed else 0.0)) * kind_scale
entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
for axis in self._v2_order_axes(matched_missing):
train_axis(axis, (0.18 + (0.05 if axis in matched_regressed else 0.0)) * kind_scale)
elif matched_resolved:
delta = 0.46 * kind_scale
entry["resolved_count"] = int(entry.get("resolved_count", 0)) + 1
for axis in self._v2_order_axes(matched_resolved):
train_axis(axis, 0.15 * kind_scale)
elif matched_satisfied:
delta = 0.20 * kind_scale
entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
for axis in self._v2_order_axes(matched_satisfied):
train_axis(axis, 0.075 * kind_scale)
else:
delta = 0.04 * kind_scale
wanted_axes = entry.setdefault("wanted_axes", {})
for axis in self._v2_order_axes(matched_missing):
wanted_axes[axis] = int(wanted_axes.get(axis, 0)) + 1
satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
for axis in self._v2_order_axes(matched_satisfied):
satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
resolved_axis_counts = entry.setdefault("resolved_axes", {})
for axis in self._v2_order_axes(matched_resolved):
resolved_axis_counts[axis] = int(resolved_axis_counts.get(axis, 0)) + 1
bump_evidence("missing_axes", missing_axes)
bump_evidence("satisfied_axes", satisfied_axes)
bump_evidence("resolved_axes", resolved_axes)
bump_evidence("regressed_axes", regressed_axes)
else:
delta = reward * 0.20 * kind_scale
for category, score in effective_before.items():
if float(score) >= 0.32:
weights[category] = float(weights.get(category, 0.0)) + reward * 0.035 * kind_scale
entry["category_weights"] = {
category: round(max(-1.5, min(2.5, float(value))), 6)
for category, value in weights.items()
}
entry["category_evidence_count"] = int(entry.get("category_evidence_count", 0)) + (
1 if count_category_evidence else 0
)
if isinstance(context_sense, dict) and count_category_evidence:
sense_weights = self._v2_clean_category_scores(context_sense.get("category_weights", {}))
for category, value in entry["category_weights"].items():
sense_weights[category] = float(sense_weights.get(category, 0.0)) + (
float(value) - float(weights_before.get(category, 0.0))
) * (1.15 * kind_scale)
context_sense["category_weights"] = {
category: round(max(-1.5, min(2.5, float(value))), 6)
for category, value in sense_weights.items()
}
context_sense["category_evidence_count"] = int(context_sense.get("category_evidence_count", 0)) + 1
context_sense["effective_category_scores"] = self._v2_effective_category_scores(
machine_scores,
context_sense["category_weights"],
int(context_sense.get("category_evidence_count", 0)),
)
sense_evidence = context_sense.setdefault("rating_evidence", {})
sense_evidence[rating_profile.get("key", "unknown")] = (
int(sense_evidence.get(rating_profile.get("key", "unknown"), 0)) + 1
)
self._v2_prune_context_senses(entry)
entry["effective_category_scores"] = self._v2_effective_category_scores(
machine_scores,
entry["category_weights"],
int(entry.get("category_evidence_count", 0)),
)
primary, confidence = self._v2_scores_primary(entry["effective_category_scores"])
entry["primary"] = primary
entry["confidence"] = round(float(confidence), 4)
entry["category_scores"] = dict(entry["effective_category_scores"])
entry["score"] = round(max(-3.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
entry["last_seen_iter"] = int(iter_num)
memory[text] = entry
touched.append(text)
if len(trained) < 8 and entry.get("kind") in {"phrase", "token", "ngram"}:
trained.append(
f"{text}[{entry.get('kind')}:{entry['machine_primary']}->{primary}, "
f"ctx={phrase.get('context_signature', 'none') or 'none'}, "
f"e={int(entry.get('category_evidence_count', 0))}, "
f"top={self._v2_top_category_summary(entry['effective_category_scores'], limit=2)}]"
)
if not touched:
return "Lucky memory: no prompt phrases stored."
kind_summary = ", ".join(f"{name}:{count}" for name, count in sorted(kind_counts.items())) or "none"
suppression_summary = (
f" Appearance auto-injection suppressed: {', '.join(suppressed[:8])}{'...' if len(suppressed) > 8 else ''}."
if suppressed else
""
)
return (
f"Category memory trained: {len(touched)} concept unit(s) ({kind_summary}). "
f"Sample: {', '.join(trained) if trained else ', '.join(touched[:6])}. "
f"{preferred_context_status}{suppression_summary} "
f"{self._v2_axis_feedback_status(axis_feedback)}"
)
def _v2_shape_compatible(self, payload, conditioning):
return self._serialized_conditioning_compatible(payload, conditioning)
def _v2_find_phrase_token_ranges(self, clip, full_cond, phrase_texts, encode_cache=None):
"""Find token positions of phrases in full_cond via cosine similarity.
Model-agnostic: works with any text encoder by comparing mean-pooled phrase
encodings against each position in the full conditioning sequence."""
if not isinstance(full_cond, torch.Tensor) or not phrase_texts:
return []
seq_len = full_cond.shape[1] if full_cond.dim() >= 2 else 0
if seq_len < 2:
return []
ranges = []
try:
full_norm = full_cond[0].float()
full_norm = full_norm / full_norm.norm(dim=-1, keepdim=True).clamp_min(1e-8)
for text in phrase_texts[:8]:
phrase_cond, _, _ = self._v2_encode_prompt(clip, str(text), encode_cache=encode_cache,
purpose="emphasis ranges")
if not isinstance(phrase_cond, torch.Tensor):
continue
phrase_mean = phrase_cond[0].float().mean(dim=0)
phrase_norm = phrase_mean / phrase_mean.norm().clamp_min(1e-8)
sims = (full_norm * phrase_norm).sum(dim=-1)
mean_s = float(sims.mean().item())
std_s = float(sims.std().item())
threshold = mean_s + std_s * 0.6
above = (sims > threshold).cpu().tolist()
start = None
for i, v in enumerate(above):
if v and start is None:
start = i
elif not v and start is not None:
if i - start >= 1:
ranges.append((start, i))
start = None
if start is not None and seq_len - start >= 1:
ranges.append((start, seq_len))
except Exception:
pass
return ranges[:16]
def _v2_build_attn2_patch(self, direction_slots, emphasis_ranges):
"""Build attn2_patch for layer-level direction injection and phrase K/V emphasis.
Operates in text encoder space (pre-projection) so directions are directly compatible.
Per-layer scale is small (0.02) so effect accumulates gradually across all layers."""
loaded = []
for dir_payload, scale, negate in direction_slots:
try:
t = serializable_to_tensor(dir_payload).float().cpu()
loaded.append((t, float(scale), bool(negate)))
except Exception:
continue
if not loaded and not emphasis_ranges:
return None
def patch(q, k, v, extra_options):
k_out, v_out = k, v
try:
for direction_cpu, scale, negate in loaded:
d = direction_cpu.to(device=k.device, dtype=k.dtype)
if d.shape[-1] != k.shape[-1]:
continue
while d.dim() < k.dim():
d = d.unsqueeze(0)
d = d.expand_as(k)
if negate:
d = -d
norms = k.norm(dim=-1, keepdim=True)
mx = norms.amax().clamp_min(1e-8)
active = norms > mx * 0.01
avg_n = float(norms[active].mean().item() if active.any() else mx.item())
delta = d * (scale * 0.02 * 0.3 * avg_n)
k_out = torch.where(active, k_out + delta, k_out)
v_out = torch.where(active, v_out + delta, v_out)
except Exception:
pass
if emphasis_ranges:
try:
k_out = k_out.clone()
v_out = v_out.clone()
for start, end in emphasis_ranges:
end = min(end, k_out.shape[1])
if start < end:
k_out[:, start:end] = k_out[:, start:end] * 1.25
v_out[:, start:end] = v_out[:, start:end] * 1.25
except Exception:
pass
# If this fires on an AV-concat context, keep the audio channel-slice unmodified
# (no-op on already-projected / single-stream widths). Protects audio_attn2.
k_out = protect_audio_channels(k_out, k)
v_out = protect_audio_channels(v_out, v)
return q, k_out, v_out
return patch
def _v2_model_reads_attn2(self, model):
"""Whether this model has a cross-attention for the direction patch to hook.
Asked BEFORE the emphasis ranges are computed, not only inside the patch. Locating
those ranges costs one full text-encoder pass per phrase — on H3 that is the same
multi-billion-parameter encoder the prompt goes through, spent on ranges for a patch
that is about to decline. Same question, asked early enough to skip the work.
"""
if model is None:
return False
try:
from .minimax_h3 import is_h3_model
except ImportError:
from minimax_h3 import is_h3_model
return not is_h3_model(model)
def _v2_apply_model_patches(self, model, global_state, axis_feedback, rating_profile, emphasis_ranges, strength):
"""Clone model and apply attn2 patch for layer-level direction injection + phrase emphasis."""
if model is None:
return None
# MiniMax H3 has no cross-attention at all: text, audio and video are rows of ONE packed
# self-attention stream, so there is no attn2 for `set_model_attn2_patch` to hook. The
# patch would install and never fire. Say so instead of reporting a patch that is doing
# nothing — the conditioning-space steering (embed guidance, score slider, the direction
# applied to the tensor itself) still works on H3 and carries the same intent.
try:
from .minimax_h3 import is_h3_model
except ImportError:
from minimax_h3 import is_h3_model
if is_h3_model(model):
_log.note_on_change("studio:h3attn2", "FunPackStudio",
"MiniMax H3: skipping the attn2 K/V direction patch — H3 is a single packed "
"self-attention stream with no cross-attention to hook. Learned directions "
"still apply to the conditioning tensor itself.")
return None
axis_memory = global_state.get("axis_conditioning_memory", {})
missing_axes = set(axis_feedback.get("missing_axes", []))
satisfied_axes = set(axis_feedback.get("satisfied_axes", []))
liked_dir_slot = global_state.get("liked_dir", {})
bad_dir_slot = global_state.get("bad_dir", {})
reward = float(rating_profile.get("reward", 0.0)) if isinstance(rating_profile, dict) else 0.0
slots = []
if int(liked_dir_slot.get("direction_count", 0)) >= 3:
slots.append((liked_dir_slot.get("direction"), strength, False))
for axis in V2_FEEDBACK_AXES:
axis_slot = axis_memory.get(axis, {}) if isinstance(axis_memory, dict) else {}
if not isinstance(axis_slot, dict):
continue
pos_slot = axis_slot.get("positive", {})
neg_slot = axis_slot.get("negative", {})
if axis in missing_axes:
if int(pos_slot.get("direction_count", 0)) >= 3:
slots.append((pos_slot.get("direction"), strength * 1.1, False))
if int(neg_slot.get("direction_count", 0)) >= 3:
slots.append((neg_slot.get("direction"), strength * 0.82, True))
elif axis in satisfied_axes:
if int(pos_slot.get("direction_count", 0)) >= 3:
slots.append((pos_slot.get("direction"), strength * 0.34, False))
if reward < 0.0 and int(bad_dir_slot.get("direction_count", 0)) >= 3:
slots.append((bad_dir_slot.get("direction"), strength * 0.72, True))
valid_slots = [(p, s, n) for p, s, n in slots if isinstance(p, dict)]
patch_fn = self._v2_build_attn2_patch(valid_slots, emphasis_ranges)
if patch_fn is None:
return None
patched = model.clone()
patched.set_model_attn2_patch(patch_fn)
return patched
def _v2_pool_conditioning(self, payload):
"""Deserialize and mean-pool [batch?, seq, dim] → [dim]. Shape-invariant across prompt lengths."""
if not isinstance(payload, dict):
return None
try:
t = serializable_to_tensor(payload).float()
while t.dim() > 1:
t = t.mean(dim=0)
return t
except Exception:
return None
def _v2_update_session_mean(self, global_state, source_payload):
"""Running mean of raw source conditioning (pre-memory), pooled to [dim] for shape invariance."""
pooled = self._v2_pool_conditioning(source_payload)
if pooled is None:
return
count = int(global_state.get("session_source_mean_count", 0))
existing = global_state.get("session_source_mean")
if count <= 0 or not isinstance(existing, dict):
global_state["session_source_mean"] = tensor_to_serializable(pooled.cpu())
global_state["session_source_mean_count"] = 1
return
try:
prev = serializable_to_tensor(existing).float().to(pooled.device)
if list(prev.shape) != list(pooled.shape):
global_state["session_source_mean"] = tensor_to_serializable(pooled.cpu())
global_state["session_source_mean_count"] = 1
return
averaged = (prev * count + pooled) / float(count + 1)
global_state["session_source_mean"] = tensor_to_serializable(averaged.cpu())
global_state["session_source_mean_count"] = count + 1
except Exception:
pass
def _v2_store_direction(self, slot, payload, session_mean_payload):
"""Store running-averaged unit-norm direction (delta from session mean) in slot.
Shape-invariant: works across any sequence length because both are pooled to [dim]."""
pooled = self._v2_pool_conditioning(payload)
session_mean = self._v2_pool_conditioning(session_mean_payload)
if pooled is None or session_mean is None:
return
try:
mean = session_mean.to(pooled.device)
if list(mean.shape) != list(pooled.shape):
return
delta = pooled - mean
magnitude = float(delta.norm().item())
if magnitude < 1e-6:
return
unit = delta / magnitude
count = int(slot.get("direction_count", 0))
existing = slot.get("direction")
if count <= 0 or not isinstance(existing, dict):
slot["direction"] = tensor_to_serializable(unit.cpu())
slot["direction_magnitude"] = magnitude
slot["direction_count"] = 1
return
prev = serializable_to_tensor(existing).float().to(unit.device)
if list(prev.shape) != list(unit.shape):
slot["direction"] = tensor_to_serializable(unit.cpu())
slot["direction_magnitude"] = magnitude
slot["direction_count"] = 1
return
avg_dir = (prev * count + unit) / float(count + 1)
avg_dir = avg_dir / avg_dir.norm().clamp_min(1e-8)
avg_mag = (float(slot.get("direction_magnitude", 0.0)) * count + magnitude) / float(count + 1)
slot["direction"] = tensor_to_serializable(avg_dir.cpu())
slot["direction_magnitude"] = float(avg_mag)
slot["direction_count"] = count + 1
except Exception:
pass
def _v2_store_prompt_keyed_direction(self, global_state, payload, session_mean_payload,
max_entries=32):
"""Append a (prompt fingerprint -> liked direction) entry to a retrieval index.
Stores the SAME unit-delta direction that feeds the global liked_dir slot
(pool(payload) minus the session source mean), paired with this run's pooled
conditioning as the prompt fingerprint, so the sampler can later retrieve the
direction learned on the NEAREST prompts instead of the one global average (see
samplers._resolve_prompt_keyed_direction). The fingerprint is pool(payload) — the
processed conditioning, which is exactly what the Chain Sampler pools from its
scene input at retrieval time, so cosine matching lines up on the same domain.
Non-parametric memory: a plain ring buffer, no model to train, so it can't collapse
into a spurious attractor the way a small value function can. Oldest rolls off;
rides the same clip-state file as liked_dir, so delete_refinement_key removes it
with the key. Silent no-op on any shape/None issue."""
pooled = self._v2_pool_conditioning(payload)
session_mean = self._v2_pool_conditioning(session_mean_payload)
if pooled is None or session_mean is None:
return
try:
if list(session_mean.shape) != list(pooled.shape):
return
delta = pooled - session_mean.to(pooled.device)
magnitude = float(delta.norm().item())
if magnitude < 1e-6:
return
unit = delta / magnitude
index = global_state.get("prompt_dir_index")
if not isinstance(index, list):
index = []
index.append({
"prompt": tensor_to_serializable(pooled.cpu()),
"direction": tensor_to_serializable(unit.cpu()),
"direction_magnitude": magnitude,
})
global_state["prompt_dir_index"] = index[-int(max_entries):]
except Exception:
pass
def _v2_apply_direction(self, mixed, slot, strength, negate=False):
"""Apply stored unit direction to active tokens using NORM_SCALE calibration.
Requires direction_count >= 3. Falls back silently if not ready."""
if not isinstance(slot, dict):
return mixed
count = int(slot.get("direction_count", 0))
if count < 3:
return mixed
direction_payload = slot.get("direction")
magnitude = float(slot.get("direction_magnitude", 0.0))
if not isinstance(direction_payload, dict) or magnitude < 1e-6:
return mixed
try:
direction = serializable_to_tensor(direction_payload).to(device=mixed.device, dtype=mixed.dtype)
while direction.dim() < mixed.dim():
direction = direction.unsqueeze(0)
direction = direction.expand_as(mixed)
if negate:
direction = -direction
token_norms = mixed.norm(dim=-1, keepdim=True)
max_norm = token_norms.amax().clamp_min(1e-8)
active_mask = token_norms > max_norm * 0.01
avg_active_norm = float(
token_norms[active_mask].mean().item() if active_mask.any() else max_norm.item()
)
# NORM_SCALE=0.3 calibration: strength=1.0 is visible but non-destructive
delta = direction * (strength * 0.3 * avg_active_norm)
return torch.where(active_mask, mixed + delta, mixed)
except Exception as _e:
_log.failed("FunPackStudio", "learned direction", _e,
"the conditioning is used unsteered — ratings had no effect on this run")
return mixed
def _v2_store_conditioning_average(self, slot, payload):
count = int(slot.get("count", 0))
existing = slot.get("conditioning")
if count <= 0 or not isinstance(existing, dict):
slot["conditioning"] = payload
slot["count"] = 1
return
try:
previous = serializable_to_tensor(existing)
current = serializable_to_tensor(payload).to(previous.device)
if list(previous.shape) != list(current.shape):
slot["conditioning"] = payload
slot["count"] = 1
return
averaged = (previous * count + current) / float(count + 1)
slot["conditioning"] = tensor_to_serializable(averaged.cpu())
slot["count"] = count + 1
except Exception:
slot["conditioning"] = payload
slot["count"] = 1
def _v2_update_axis_conditioning_memory(self, global_state, payload, axis_feedback, session_mean_payload=None):
if not isinstance(payload, dict):
return
memory = global_state.setdefault("axis_conditioning_memory", {})
for axis in self._v2_order_axes(axis_feedback.get("satisfied_axes", [])):
axis_slot = memory.setdefault(axis, {})
positive = axis_slot.setdefault("positive", {})
self._v2_store_conditioning_average(positive, payload)
self._v2_store_direction(positive, payload, session_mean_payload)
for axis in self._v2_order_axes(axis_feedback.get("resolved_axes", [])):
axis_slot = memory.setdefault(axis, {})
positive = axis_slot.setdefault("positive", {})
self._v2_store_conditioning_average(positive, payload)
self._v2_store_direction(positive, payload, session_mean_payload)
axis_slot["resolved_count"] = int(axis_slot.get("resolved_count", 0)) + 1
for axis in self._v2_order_axes(axis_feedback.get("missing_axes", [])):
axis_slot = memory.setdefault(axis, {})
negative = axis_slot.setdefault("negative", {})
self._v2_store_conditioning_average(negative, payload)
self._v2_store_direction(negative, payload, session_mean_payload)
for axis in self._v2_order_axes(axis_feedback.get("regressed_axes", [])):
axis_slot = memory.setdefault(axis, {})
axis_slot["regressed_count"] = int(axis_slot.get("regressed_count", 0)) + 1
def _v2_update_conditioning_memory(self, global_state, last_run, rating_profile, axis_feedback=None):
if not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
return
if rating_profile.get("key") == "wrong_appearance":
return
payload = last_run.get("conditioning")
if not isinstance(payload, dict):
return
source_payload = last_run.get("source_conditioning")
self._v2_update_session_mean(global_state, source_payload)
session_mean = global_state.get("session_source_mean")
key = rating_profile.get("key", "")
axis_feedback = axis_feedback or self._v2_axis_feedback(
rating_profile,
global_state.get("last_missing_axes", []),
)
self._v2_update_axis_conditioning_memory(global_state, payload, axis_feedback,
session_mean_payload=session_mean)
if key in {"like", "loved_it", "nailed_it"}:
liked_dir_slot = global_state.setdefault("liked_dir", {})
self._v2_store_direction(liked_dir_slot, payload, session_mean)
self._v2_store_prompt_keyed_direction(global_state, payload, session_mean)
count = int(global_state.get("liked_conditioning_count", 0))
if count <= 0 or not isinstance(global_state.get("liked_conditioning"), dict):
global_state["liked_conditioning"] = payload
global_state["liked_conditioning_count"] = 1
return
try:
previous = serializable_to_tensor(global_state["liked_conditioning"])
current = serializable_to_tensor(payload).to(previous.device)
if list(previous.shape) != list(current.shape):
global_state["liked_conditioning"] = payload
global_state["liked_conditioning_count"] = 1
return
averaged = (previous * count + current) / float(count + 1)
global_state["liked_conditioning"] = tensor_to_serializable(averaged.cpu())
global_state["liked_conditioning_count"] = count + 1
except Exception:
global_state["liked_conditioning"] = payload
global_state["liked_conditioning_count"] = 1
elif key == "awful" or float(rating_profile.get("reward", 0.0)) < -0.35:
bad_dir_slot = global_state.setdefault("bad_dir", {})
self._v2_store_direction(bad_dir_slot, payload, session_mean)
global_state["bad_conditioning"] = payload
global_state["bad_conditioning_count"] = int(global_state.get("bad_conditioning_count", 0)) + 1
def _v2_learn_from_batch(self, refinement_key, rated_items):
"""Phase 3b: learn from a controlled batch. Every entry shared frozen tokens, so axes
rated missing/wrong across many entries are a HIGH-CONFIDENCE signal (not seed luck).
Feeds each rated entry's conditioning into the per-axis direction memory (same machinery
as a normal rated run) AND records axes that are consistently bad as a persistent repair
bias the next generation consumes. rated_items: [{"rating": label, "conditioning": payload}]."""
try:
state, _ = self._v2_load_state(refinement_key)
except Exception as e:
return f"batch axis learning skipped: state load failed ({e})"
global_state = state.setdefault("global", {})
rated = 0
missing_votes, wrong_votes = {}, {}
for it in rated_items or []:
label = it.get("rating")
profile = normalize_refiner_v2_rating(label) if label else None
if not profile or profile.get("skip_learning"):
continue
rated += 1
payload = it.get("conditioning")
if isinstance(payload, dict):
try:
axis_feedback = self._v2_axis_feedback(profile, global_state.get("last_missing_axes", []))
self._v2_update_conditioning_memory(global_state, {"conditioning": payload}, profile, axis_feedback)
except Exception as e:
print(f"[FunPackVideoRefinerV2] batch cond-memory update failed: {e}")
for ax in profile.get("missing_axes", []):
missing_votes[ax] = missing_votes.get(ax, 0) + 1
for ax in profile.get("wrong_axes", []):
wrong_votes[ax] = wrong_votes.get(ax, 0) + 1
if rated == 0:
return "batch axis learning: no rated entries."
threshold = max(2, (rated + 1) // 2)
consistent = [ax for ax, c in missing_votes.items() if c >= threshold]
consistent += [ax for ax, c in wrong_votes.items() if c >= threshold and ax not in consistent]
consistent = [ax for ax in consistent if ax in V2_FEEDBACK_AXES]
if consistent:
repair = set(global_state.get("active_repair_axes", []) or []) | set(consistent)
global_state["active_repair_axes"] = self._v2_order_axes(repair & set(V2_FEEDBACK_AXES))
global_state["last_batch_insight"] = {
"rated": rated, "missing": missing_votes, "wrong": wrong_votes, "consistent": consistent,
}
try:
self._v2_save_state(state, refinement_key)
except Exception as e:
return f"batch axis learning: state save failed ({e})"
return (f"batch axis learning: {rated} rated; consistent problem axes = "
f"{', '.join(consistent) if consistent else 'none (results varied by seed)'}.")
def _v2_update_streaks(self, global_state, rating_profile, update_conditioning_strength=True):
reward = float(rating_profile.get("reward", 0.0))
if update_conditioning_strength:
avg = float(global_state.get("avg_reward_ema", 0.0))
avg = 0.86 * avg + 0.14 * reward
global_state["avg_reward_ema"] = round(avg, 6)
if rating_profile.get("key") in {"like", "loved_it", "nailed_it"}:
global_state["good_streak"] = int(global_state.get("good_streak", 0)) + 1
global_state["bad_streak"] = 0
elif reward < -0.25:
global_state["bad_streak"] = int(global_state.get("bad_streak", 0)) + 1
global_state["good_streak"] = 0
else:
global_state["good_streak"] = 0
global_state["bad_streak"] = 0
global_state["last_rating_label"] = rating_profile.get("label", "")
global_state["last_missing_axes"] = list(rating_profile.get("missing_axes", []))
def _v2_update_concept_delta_memory(self, global_state, previous_run, learning_profile):
"""Learn which final-prompt phrases correlate with reward changes when added/removed."""
if not isinstance(previous_run, dict) or learning_profile.get("skip_learning"):
return "Concept delta: skipped."
reward = float(learning_profile.get("reward", 0.0))
current_texts = set(previous_run.get("final_phrase_texts", []) or [])
prev_texts = set(previous_run.get("prev_final_phrase_texts", []) or [])
if not current_texts or not prev_texts:
return "Concept delta: need 2 consecutive final prompts."
added = current_texts - prev_texts
removed = prev_texts - current_texts
if not added and not removed:
return "Concept delta: no changes between final prompts."
memory = global_state.setdefault("concept_delta_memory", {})
for text in added:
e = memory.setdefault(text, {"add_sum": 0.0, "add_count": 0, "remove_sum": 0.0, "remove_count": 0})
e["add_sum"] = round(e["add_sum"] + reward, 6)
e["add_count"] += 1
for text in removed:
e = memory.setdefault(text, {"add_sum": 0.0, "add_count": 0, "remove_sum": 0.0, "remove_count": 0})
e["remove_sum"] = round(e["remove_sum"] + reward, 6)
e["remove_count"] += 1
return f"Concept delta: +{len(added)} -{len(removed)} phrases tracked (reward={reward:+.2f})."
def _v2_concept_delta_direction(self, global_state, current_final_texts, prev_final_texts):
"""Weighted conditioning direction from concept delta memory for new phrases in this run."""
delta_memory = global_state.get("concept_delta_memory", {})
phrase_memory = global_state.get("phrase_memory", {})
if not delta_memory or not phrase_memory:
return None, 0.0
added = set(current_final_texts) - set(prev_final_texts)
if not added:
return None, 0.0
MIN_COUNT = 3
directions, weights = [], []
for text in added:
entry = delta_memory.get(text)
if not entry or entry.get("add_count", 0) < MIN_COUNT:
continue
avg_reward = entry["add_sum"] / entry["add_count"]
if abs(avg_reward) < 0.10:
continue
# Find embedding — match by phrase text or individual tokens
pm_entry = phrase_memory.get(text) or next(
(phrase_memory[k] for k in phrase_memory if text in k or k in text), None
)
if not pm_entry or "embedding" not in pm_entry:
continue
try:
emb = serializable_to_tensor(pm_entry["embedding"]).float()
directions.append(emb)
weights.append(float(avg_reward))
except Exception:
continue
if not directions:
return None, 0.0
total = torch.zeros_like(directions[0])
for d, w in zip(directions, weights):
total = total + d.to(total.device) * w
norm = total.norm()
if norm < 1e-8:
return None, 0.0
avg_signal = sum(abs(w) for w in weights) / len(weights)
strength = min(0.035, avg_signal * 0.035 * len(weights))
return torch.nn.functional.normalize(total, dim=-1), strength
def _v2_update_concept_pair_dirs(self, global_state, previous_run, learning_profile):
"""Store bad conditioning direction for each phrase pair present in a bad generation."""
if not isinstance(previous_run, dict) or learning_profile.get("skip_learning"):
return
if learning_profile.get("loved_modifier"):
return # Loved despite issues — don't penalize this combination
reward = float(learning_profile.get("reward", 0.0))
key = learning_profile.get("key", "")
# Learn from anatomy/quality failures — wrong_appearance explicitly included
# as it signals anatomy artifacts even though reward=0.0
bad_anatomy = reward < -0.10 or key in {"wrong_appearance", "wrong_action_quality"}
if not bad_anatomy:
return
payload = previous_run.get("conditioning")
if not isinstance(payload, dict):
return
final_texts = previous_run.get("final_phrase_texts", []) or []
if len(final_texts) < 2:
return
session_mean = global_state.get("session_source_mean")
pair_dirs = global_state.setdefault("concept_pair_dirs", {})
top = final_texts[:8] # Cap to avoid O(n²) explosion
for i, a in enumerate(top):
for b in top[i + 1:]:
key = md5(f"{min(a,b)}|{max(a,b)}".encode()).hexdigest()[:12]
slot = pair_dirs.setdefault(key, {"phrases": [min(a, b), max(a, b)]})
self._v2_store_direction(slot, payload, session_mean)
# Prune if too large — remove pairs with fewest observations
if len(pair_dirs) > 300:
pruned = sorted(pair_dirs, key=lambda k: int(pair_dirs[k].get("direction_count", 0)))
for k in pruned[:len(pair_dirs) - 200]:
del pair_dirs[k]
def _v2_auto_strength(self, global_state):
avg = float(global_state.get("avg_reward_ema", 0.0))
good = int(global_state.get("good_streak", 0))
bad = int(global_state.get("bad_streak", 0))
strength = 0.030
if good >= 3 or avg > 0.55:
strength *= 0.42
elif good >= 1 or avg > 0.25:
strength *= 0.68
if bad >= 3 or avg < -0.45:
strength *= 2.20
elif bad >= 1 or avg < -0.15:
strength *= 1.45
return max(0.008, min(0.085, strength))
# --- reference vision rows are not steerable -----------------------------------
# MiniMax H3 tokenizes a reference image INTO the conditioning: comfy's
# token_tags_from_embeds_info marks the whole vision block (and its flanking
# <|vision_start|>/<|vision_end|>) as 0 and leaves text at 1. Those rows are Qwen's
# encoding of the picture itself.
#
# Every steering path here — learned directions, taste pull, concept deltas, the absolute
# store, negative_erase — is a lerp or an add over the WHOLE tensor. Applied to the vision
# rows it moves the encoded picture toward a direction learned from TEXT, and the
# character comes out looking like someone else. Nothing read the tag map before this.
REFERENCE_ROWS_KEY = "funpack_pristine_reference_rows"
def _v2_stash_reference_rows(self, cond, meta):
"""Keep a pristine copy of everything BEFORE the prompt.
An r2v conditioning is laid out reference-first — `<Picture n>: ` label, vision
block, then the encoded prompt — so the boundary is all that is needed: rows before
it are the picture and are not Studio's to touch, rows from it on are the prompt and
are. Protecting by that boundary rather than by the image tags alone also covers the
label, which sits between the two and is not part of anything Studio wrote.
Cheap: a conditioning is a few hundred rows. Dropped again in
`_v2_restore_reference_rows`. No-op on a text-only or non-H3 conditioning, where the
prompt starts at row 0 and there is nothing in front of it.
"""
try:
if not isinstance(cond, torch.Tensor) or cond.dim() < 2 or not isinstance(meta, dict):
return meta
try:
from . import h3_token_weights as _tw
except ImportError:
import h3_token_weights as _tw
region = _tw.prompt_region(meta.get("minimax_token_tags"), int(cond.shape[1]))
if not region or region[0] <= 0:
return meta
# The WHOLE tensor, not just the rows before `region[0]`. Where the prompt really
# begins cannot be known yet: "<Audio n>: " labels carry no vision block, so they
# sit inside the same run of text tags as the prompt, and the boundary only
# becomes exact once the encoded text has been identified — which happens later,
# in finalize. A conditioning is a few hundred rows, so keeping all of it costs
# single-digit megabytes and is dropped on restore.
meta = dict(meta)
meta[self.REFERENCE_ROWS_KEY] = cond.clone()
return meta
except Exception as _e: # noqa: BLE001
_log.failed("FunPackStudio", "reference protection", _e,
"a reference image's rows CAN be moved by steering, which shows up "
"as the character's appearance drifting")
return meta
def _v2_restore_reference_rows(self, conditioning_list, clip=None, candidates=()):
"""Put everything that is not the PROMPT back exactly as it arrived.
The user writes the prompt. They do not write "<Picture 1>: " or "<Audio 1>: " —
those are the tokenizer's, and neither they nor the vision block are Studio's to
steer. The prompt is the tokenizer's last segment, so the boundary is measured back
from the end using the text that was actually encoded; without that the vision
block's end is used, which leaves a label steerable but never the picture.
"""
try:
from . import h3_token_weights as _tw
except ImportError:
import h3_token_weights as _tw
tokenizer = _tw.h3_tokenizer(clip) if clip is not None else None
out = []
for entry in conditioning_list or []:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2
and isinstance(entry[1], dict) and self.REFERENCE_ROWS_KEY in entry[1]):
out.append(entry)
continue
cond, meta = entry[0], dict(entry[1])
rows = meta.pop(self.REFERENCE_ROWS_KEY)
try:
cond_len = int(cond.shape[1]) if isinstance(cond, torch.Tensor) and cond.dim() >= 2 else 0
tags = meta.get("minimax_token_tags")
start = None
if tokenizer is not None:
_t, _n, verified = _tw.choose_encoded_text(
tokenizer, candidates, tags, cond_len)
start = verified
if start is None:
region = _tw.prompt_region(tags, cond_len)
start = region[0] if region else None
n = int(start) if start else 0
if n > 0 and cond_len >= n and rows.shape[-1] == cond.shape[-1] \
and int(rows.shape[1]) >= n:
cond = cond.clone()
cond[:, :n] = rows[:, :n].to(device=cond.device, dtype=cond.dtype)
except Exception as _e: # noqa: BLE001
_log.failed("FunPackStudio", "reference protection", _e,
"this scene's reference rows keep whatever steering did to them")
out.append([cond, meta] if isinstance(entry, list) else (cond, meta))
return out
def _v2_payload_like(self, payload, mixed):
"""A stored conditioning as a tensor the CURRENT one can be blended with, or None.
`_v2_shape_compatible` checks the channel width and the batch, deliberately not the
sequence length — and on H3 that length changes with every prompt edit, because Qwen
does not pad (`pad_to_max_length=False`). So a payload learned at 499 positions met a
conditioning of 492 and the blend threw, taking the whole learned direction with it.
On LTX the encoder pads to a fixed length and this never came up.
`_resize_conditioning_sequence_like` was written for exactly this and never wired to
anything. It interpolates along the sequence, which is what keeps a learned direction
applicable to a prompt whose length has since changed.
"""
try:
target = serializable_to_tensor(payload).to(device=mixed.device, dtype=mixed.dtype)
except Exception: # noqa: BLE001
return None
if not isinstance(target, torch.Tensor):
return None
if list(target.shape) == list(mixed.shape):
return target
resized = self._resize_conditioning_sequence_like(target, mixed)
if resized is None:
return None
_log.note_on_change(
"studio:payload_resize", "FunPackStudio",
f"a stored conditioning was learned at {int(target.shape[-2])} positions and this "
f"prompt is {int(mixed.shape[-2])} — resampled along the sequence so the learned "
f"direction still applies. H3's encoder does not pad, so this happens on any "
f"prompt edit.")
return resized
def _v2_apply_conditioning_payload(self, mixed, payload, strength):
if not self._v2_shape_compatible(payload, mixed):
return mixed
try:
target = self._v2_payload_like(payload, mixed)
if target is None:
return mixed
return mixed.lerp(target, strength)
except Exception as _e:
_log.failed("FunPackStudio", "conditioning memory", _e,
"the conditioning is used unblended")
return mixed
def _v2_repel_conditioning_payload(self, mixed, payload, strength):
if not self._v2_shape_compatible(payload, mixed):
return mixed
try:
target = self._v2_payload_like(payload, mixed)
if target is None:
return mixed
return mixed + (mixed - target) * strength
except Exception as _e:
_log.failed("FunPackStudio", "conditioning repel", _e,
"the conditioning is not pushed away from the disliked payload")
return mixed
def _vf_direction_boost(self, slot, vf_grad_dir, alpha=0.5):
"""Cosine similarity between a direction slot and VF gradient → boost factor in [0.5, 1.5].
Phrases pointing with the reward gradient get amplified; against get dampened."""
if vf_grad_dir is None:
return 1.0
try:
direction = slot.get("direction")
if direction is None:
return 1.0
d = serializable_to_tensor(direction).float()
d_mean = d.mean(dim=-2) if d.dim() > 1 else d
d_dir = torch.nn.functional.normalize(d_mean.flatten(), dim=0)
vf_dir = vf_grad_dir.flatten().to(d_dir.device)
sim = float(torch.dot(d_dir, vf_dir).clamp(-1.0, 1.0).item())
return max(0.5, min(1.5, 1.0 + alpha * sim))
except Exception:
return 1.0
def _vf_gradient_direction(self, conditioning, vf):
"""Unit vector of ∂reward/∂conditioning, mean-pooled across sequence. Returns None on failure."""
try:
grad = vf.gradient(conditioning) # same shape as conditioning
dir_vec = grad.float().mean(dim=1).squeeze(0) # [C]
return torch.nn.functional.normalize(dir_vec, dim=-1)
except Exception:
return None
def _v2_apply_conditioning_memory(self, conditioning, *args, **kwargs):
"""Audio-safe wrapper: confine all per-prompt conditioning steering to the video
channel-slice, restoring the audio channels from the unsteered input (LTXAV splits the
conditioning into video|audio channels; see protect_audio_channels)."""
result = self._v2_apply_conditioning_memory_impl(conditioning, *args, **kwargs)
if isinstance(result, tuple) and result and isinstance(result[0], torch.Tensor):
return (protect_audio_channels(result[0], conditioning),) + tuple(result[1:])
return result
def _v2_apply_conditioning_memory_impl(self, conditioning, global_state, rating_profile, axis_feedback=None, intent_family_slot=None, vf=None, concept_delta_dir=None, concept_delta_strength=0.0, current_final_texts=None):
if not isinstance(conditioning, torch.Tensor):
return conditioning, "Adaptation: unavailable."
original = conditioning.clone()
mixed = conditioning.clone()
strength = self._v2_auto_strength(global_state)
# --- Value function: global confidence + gradient direction ---
vf_score = None
vf_grad_dir = None
if vf is not None:
try:
with torch.inference_mode(False), torch.no_grad():
vf_score = float(vf.forward(vf.compress(original.float()).unsqueeze(0)).item())
# Global confidence: scale all memory modifications by VF predicted reward
# High score → full strength (conditioning direction is good)
# Low score → reduced (be cautious, current direction looks bad)
confidence_scale = max(0.40, (vf_score + 1.0) * 0.50) # [-1,1] → [0.4, 1.0]
strength = strength * confidence_scale
# Gradient direction for phrase-level alignment
vf_grad_dir = self._vf_gradient_direction(original, vf)
except Exception as _e:
_log.failed("FunPackStudio", "value-function confidence scaling", _e,
"steering runs at full strength with no confidence discount")
axis_feedback = axis_feedback or self._v2_axis_feedback(
rating_profile,
global_state.get("last_missing_axes", []),
)
# Liked conditioning: direction-based when enough data (cross-prompt compatible),
# otherwise legacy lerp toward averaged full tensor.
liked_dir_slot = global_state.get("liked_dir", {})
liked_payload = global_state.get("liked_conditioning")
liked_boost = self._vf_direction_boost(liked_dir_slot, vf_grad_dir)
if int(liked_dir_slot.get("direction_count", 0)) >= 3:
mixed = self._v2_apply_direction(mixed, liked_dir_slot, strength * liked_boost)
elif self._v2_shape_compatible(liked_payload, mixed):
try:
liked = serializable_to_tensor(liked_payload).to(device=mixed.device, dtype=mixed.dtype)
mixed = mixed.lerp(liked, strength)
except Exception as _e:
_log.failed("FunPackStudio", "liked-conditioning blend", _e,
"the conditioning is NOT pulled toward what you rated well")
mixed, family_delta_status = self._v2_apply_intent_family_delta(mixed, intent_family_slot, strength)
# --- Consistency anchor (Wrong appearance) ---
# "Wrong appearance" means the established character drifted. Pull this run gently back
# toward the last good-rated conditioning for this key (the "blessed appearance") and
# repel from the just-rejected drifted look. Gentle fixed strengths; the global 7.5%
# delta cap below bounds the net effect. No-op when nothing is stored or shapes mismatch.
appearance_status = "appearance-anchor idle"
if rating_profile.get("key") == "wrong_appearance":
anchor = global_state.get("appearance_anchor") or {}
drift = global_state.get("appearance_drift") or {}
anchor_actions = []
if self._v2_shape_compatible(anchor.get("conditioning"), mixed):
mixed = self._v2_apply_conditioning_payload(mixed, anchor.get("conditioning"), 0.05)
anchor_actions.append("pull→blessed")
if self._v2_shape_compatible(drift.get("conditioning"), mixed):
mixed = self._v2_repel_conditioning_payload(mixed, drift.get("conditioning"), 0.04)
anchor_actions.append("repel←drift")
if anchor_actions:
appearance_status = "appearance-anchor: " + " + ".join(anchor_actions)
axis_memory = global_state.get("axis_conditioning_memory", {})
axis_actions = []
missing_axes = set(axis_feedback.get("missing_axes", []))
satisfied_axes = set(axis_feedback.get("satisfied_axes", []))
for axis in V2_FEEDBACK_AXES:
axis_slot = axis_memory.get(axis, {}) if isinstance(axis_memory, dict) else {}
if not isinstance(axis_slot, dict):
continue
positive_slot = axis_slot.get("positive", {})
negative_slot = axis_slot.get("negative", {})
pos_dir_ready = int(positive_slot.get("direction_count", 0)) >= 3
neg_dir_ready = int(negative_slot.get("direction_count", 0)) >= 3
if axis in missing_axes:
boost = self._vf_direction_boost(positive_slot, vf_grad_dir)
before = mixed
if pos_dir_ready:
mixed = self._v2_apply_direction(mixed, positive_slot, min(0.070, strength * 1.10 * boost))
else:
mixed = self._v2_apply_conditioning_payload(
mixed, positive_slot.get("conditioning"), min(0.070, strength * 1.10))
if neg_dir_ready:
mixed = self._v2_apply_direction(mixed, negative_slot, min(0.055, strength * 0.82), negate=True)
else:
mixed = self._v2_repel_conditioning_payload(
mixed, negative_slot.get("conditioning"), min(0.055, strength * 0.82))
if not torch.equal(mixed, before):
axis_actions.append(f"{axis}:repair({'dir' if pos_dir_ready else 'lerp'}, boost={boost:.2f})")
elif axis in satisfied_axes:
boost = self._vf_direction_boost(positive_slot, vf_grad_dir)
before = mixed
if pos_dir_ready:
mixed = self._v2_apply_direction(mixed, positive_slot, min(0.026, strength * 0.34 * boost))
else:
mixed = self._v2_apply_conditioning_payload(
mixed, positive_slot.get("conditioning"), min(0.026, strength * 0.34))
if not torch.equal(mixed, before):
axis_actions.append(f"{axis}:preserve({'dir' if pos_dir_ready else 'lerp'}, boost={boost:.2f})")
# --- POSITIVE SIGNALS: specific (new phrase delta) ---
# Applied after broad/context positives so specific phrase knowledge
# layers on top rather than being overwritten by broader signals.
if concept_delta_dir is not None and concept_delta_strength > 0:
try:
d = concept_delta_dir.to(device=mixed.device, dtype=mixed.dtype)
mixed = mixed + concept_delta_strength * d.reshape(mixed.shape[-1]).unsqueeze(0).unsqueeze(0).expand_as(mixed)
axis_actions.append(f"concept-delta:{concept_delta_strength:.3f}")
except Exception as _e:
_log.failed("FunPackStudio", "concept delta", _e,
"the concept adjustment is not applied to this conditioning")
# --- NEGATIVE SIGNALS: broad bad direction ---
# Applied after all positive signals so we remove bad directions
# from the fully built-up positive state.
bad_dir_slot = global_state.get("bad_dir", {})
bad_payload = global_state.get("bad_conditioning")
if float(rating_profile.get("reward", 0.0)) < 0.0:
if int(bad_dir_slot.get("direction_count", 0)) >= 3:
mixed = self._v2_apply_direction(mixed, bad_dir_slot, min(0.055, strength * 0.72), negate=True)
elif self._v2_shape_compatible(bad_payload, mixed):
try:
bad = serializable_to_tensor(bad_payload).to(device=mixed.device, dtype=mixed.dtype)
mixed = mixed + (mixed - bad) * min(0.055, strength * 0.72)
except Exception as _e:
_log.failed("FunPackStudio", "disliked-conditioning repel", _e,
"the conditioning is NOT pushed away from what you rated badly")
# --- NEGATIVE SIGNALS: specific pair repulsion ---
# Applied last among negatives — most targeted, has final say on
# known-bad concept combinations.
if current_final_texts:
pair_dirs = global_state.get("concept_pair_dirs", {})
if pair_dirs:
top = current_final_texts[:8]
applied = 0
for i, a in enumerate(top):
for b in top[i + 1:]:
key = md5(f"{min(a,b)}|{max(a,b)}".encode()).hexdigest()[:12]
slot = pair_dirs.get(key)
if slot and int(slot.get("direction_count", 0)) >= 3:
mixed = self._v2_apply_direction(mixed, slot, min(0.030, strength * 0.40), negate=True)
applied += 1
if applied >= 3:
break
if applied >= 3:
break
if applied:
axis_actions.append(f"pair-repulsion:{applied}")
# --- Delta cap (7.5% of conditioning norm) ---
delta = mixed - original
original_norm = original.norm(dim=-1, keepdim=True).clamp_min(1e-8)
max_delta = original_norm * 0.075
delta_norm = delta.norm(dim=-1, keepdim=True).clamp_min(1e-8)
scale = torch.minimum(torch.ones_like(delta_norm), max_delta / delta_norm)
mixed = original + delta * scale
mixed = torch.clamp(mixed, min=-60.0, max=60.0)
mixed = mixed / mixed.norm(dim=-1, keepdim=True).clamp_min(1e-8) * original_norm
# --- VF final gate: roll back if net effect lowered predicted reward ---
# Catches cases where patches collectively moved conditioning in a
# worse direction despite each being individually well-intentioned.
if vf is not None and vf_score is not None:
try:
with torch.inference_mode(False), torch.no_grad():
new_score = float(vf.forward(vf.compress(mixed.float()).unsqueeze(0)).item())
if new_score < vf_score - 0.05:
rollback = min(0.75, (vf_score - new_score) / max(0.1, abs(vf_score) + 0.1))
mixed = mixed.lerp(original, rollback)
axis_actions.append(f"vf-rollback:{rollback:.2f}")
except Exception as _e:
# The rollback is the safety net on the whole memory blend: without it a blend
# the value function scores WORSE than the original is kept anyway.
_log.failed("FunPackStudio", "value-function rollback check", _e,
"the blended conditioning is kept unchecked — it may score worse "
"than the unblended one")
liked_count = int(liked_dir_slot.get("direction_count", 0))
liked_mode = f"direction ({liked_count} runs)" if liked_count >= 3 else f"lerp fallback ({liked_count}/3 runs)"
bad_count = int(bad_dir_slot.get("direction_count", 0))
ema = float(global_state.get("avg_reward_ema", 0.0))
good_s = int(global_state.get("good_streak", 0))
bad_s = int(global_state.get("bad_streak", 0))
streak_str = f"good streak {good_s}" if good_s > 0 else (f"bad streak {bad_s}" if bad_s > 0 else "no streak")
axis_str = ", ".join(axis_actions) if axis_actions else "none"
vf_str = f" | VF score {vf_score:+.3f}" if vf_score is not None else ""
return mixed, (
f"Strength {strength:.3f}{vf_str} | reward trend {ema:+.3f} | {streak_str}\n"
f" Liked conditioning: {liked_mode}\n"
f" Bad conditioning: {'direction' if bad_count >= 3 else f'lerp fallback ({bad_count}/3 runs)'}\n"
f" Axis adjustments: {axis_str}\n"
f" {family_delta_status}\n"
f" {appearance_status}\n"
f" Total delta clamped to 7.5% of conditioning norm"
)
def _v2_prompt_contains_text(self, prompt, text):
prompt_key = re.sub(r"\s+", " ", str(prompt or "").strip().lower())
text_key = re.sub(r"\s+", " ", str(text or "").strip().lower())
if not prompt_key or not text_key:
return False
if text_key in prompt_key:
return True
prompt_words = set(self._v2_phrase_words(prompt_key))
text_words = set(self._v2_phrase_words(text_key))
return bool(text_words) and text_words <= prompt_words
def _v2_repair_intent_roots(self, text):
roots = set()
for word in self._v2_phrase_words(text):
roots.add(word)
if len(word) > 4 and word.endswith("ing"):
stem = word[:-3]
roots.add(stem)
if len(stem) > 2 and stem[-1] == stem[-2]:
roots.add(stem[:-1])
roots.add(stem + "e")
elif len(word) > 3 and word.endswith("ed"):
stem = word[:-2]
roots.add(stem)
roots.add(stem + "e")
elif len(word) > 3 and word.endswith("s"):
roots.add(word[:-1])
return {root for root in roots if len(root) >= 3}
def _v2_repair_candidate_matches_intent(self, text, intent_roots):
if not intent_roots:
return False
candidate_roots = self._v2_repair_intent_roots(text)
return bool(candidate_roots & intent_roots)
def _v2_user_intent_prompt_is_vague(self, prompt):
text = re.sub(r"\s+", " ", str(prompt or "").strip().lower())
if not text:
return True
words = set(self._v2_phrase_words(text))
if not words:
return True
vague_words = {
"figure", "out", "something", "anything", "whatever", "surprise", "random",
"choice", "choose", "decide", "good", "nice", "cool", "best", "you", "your",
"it", "make", "do",
}
if len(words) <= 5 and words <= vague_words:
return True
if re.fullmatch(r"(figure it out|surprise me|anything|whatever|your choice|you decide|do something|make it good|make it cool)", text):
return True
scores = self._v2_heuristic_scores(text)
return len(words) <= 3 and not self._v2_axes_for_scores(scores)
def _v2_serializable_repair_candidates(self, candidates):
serializable = []
for item in candidates or []:
if not isinstance(item, dict):
continue
axes = set(item.get("axes", []))
serializable.append({
"text": str(item.get("text", "")),
"axes": self._v2_order_axes(axes),
"score": round(float(item.get("score", 0.0)), 6),
"source": str(item.get("source", "")),
"cluster": [
str(text)
for text in item.get("cluster", [])
if str(text).strip()
],
})
return serializable
def _v2_update_intent_expansion(self, global_state, intent_prompt, feedback_prompt, limit=16):
feedback = str(feedback_prompt or "").strip()
key = self._v2_prompt_key(intent_prompt)
if not feedback or not key:
return
memory = global_state.setdefault("intent_expansion_memory", {})
entries = list(memory.get(key, []))
if feedback not in entries:
entries.append(feedback)
memory[key] = entries[-limit:]
def _v2_format_intent_expansions(self, global_state, intent_prompt):
key = self._v2_prompt_key(intent_prompt)
if not key:
return ""
entries = global_state.get("intent_expansion_memory", {}).get(key, [])
return "; ".join(entries) if entries else ""
def _v2_rating_is_discard(self, learning_profile):
"""True when the run must leave NO trace in refiner state — the user's
'-Just forget it-' ('Just ignore it') rating or an auto-discarded enhancer
refusal. Distinct from the other skip_learning states (continue / fresh_prompt),
which are active workflow modes that still legitimately capture user feedback."""
if not isinstance(learning_profile, dict):
return False
return learning_profile.get("key") == "forget" or bool(learning_profile.get("refusal_filtered"))
def _v2_update_advisor_feedback_history(self, global_state, feedback_prompt, rating_label, iter_num, limit=10):
feedback = str(feedback_prompt or "").strip()
if not feedback:
return
history = list(global_state.setdefault("advisor_feedback_history", []))
history.append({
"feedback": feedback,
"rating": str(rating_label or "").strip(),
"iteration": int(iter_num),
"source": "user",
})
global_state["advisor_feedback_history"] = history[-limit:]
def _v2_record_advisor_diagnostic(self, global_state, diagnostic, rating_label, iter_num, limit=10):
diagnostic = str(diagnostic or "").strip()
if not diagnostic:
return
history = list(global_state.setdefault("advisor_feedback_history", []))
history.append({
"feedback": diagnostic,
"rating": str(rating_label or "").strip(),
"iteration": int(iter_num),
"source": "advisor",
})
global_state["advisor_feedback_history"] = history[-limit:]
def _v2_format_advisor_feedback_history(self, global_state, limit=10):
history = list(global_state.get("advisor_feedback_history", []))
if not history:
return ""
recent = history[-limit:][::-1]
lines = []
for i, entry in enumerate(recent, 1):
feedback = str(entry.get("feedback", "")).strip()
if not feedback:
continue
rating = str(entry.get("rating", "")).strip()
source = str(entry.get("source", "user")).strip()
if source == "advisor":
entry_text = f"Advisor note: {feedback}"
else:
entry_text = f"{rating}: {feedback}" if rating else feedback
lines.append(f"{i}. {entry_text}")
return "\n".join(lines)
def _v2_direction_readout(self, global_state, axis_feedback=None):
"""Human-readable summary of all direction memory slots and their status."""
def slot_line(slot, label, negate=False):
count = int(slot.get("direction_count", 0)) if isinstance(slot, dict) else 0
if count == 0:
return f" {label}: no data yet"
mag = float(slot.get("direction_magnitude", 0.0)) if isinstance(slot, dict) else 0.0
if count < 3:
return f" {label}: warming up ({count}/3 runs needed)"
arrow = "away from" if negate else "toward"
return f" {label}: active ({count} runs, mag {mag:.3f}) - pulling {arrow} learned pattern"
missing_axes = set((axis_feedback or {}).get("missing_axes", []))
satisfied_axes = set((axis_feedback or {}).get("satisfied_axes", []))
axis_memory = global_state.get("axis_conditioning_memory", {}) if isinstance(global_state, dict) else {}
lines = [
slot_line(global_state.get("liked_dir", {}), "Liked"),
slot_line(global_state.get("bad_dir", {}), "Bad", negate=True),
]
for axis in V2_FEEDBACK_AXES:
axis_slot = axis_memory.get(axis, {}) if isinstance(axis_memory, dict) else {}
if not isinstance(axis_slot, dict):
continue
pos_slot = axis_slot.get("positive", {})
neg_slot = axis_slot.get("negative", {})
pos_count = int(pos_slot.get("direction_count", 0))
neg_count = int(neg_slot.get("direction_count", 0))
if pos_count == 0 and neg_count == 0:
continue
role = "REPAIR" if axis in missing_axes else ("preserve" if axis in satisfied_axes else "idle")
pos_str = f"dir({pos_count})" if pos_count >= 3 else f"lerp({pos_count}/3)"
neg_str = f"dir({neg_count})" if neg_count >= 3 else f"lerp({neg_count}/3)"
lines.append(f" Axis {axis}: {role} | + {pos_str} | - {neg_str}")
return "\n".join(lines)
def _v2_find_chat_tokenizer(self, clip):
"""BFS through CLIP wrapper hierarchy to find a HuggingFace tokenizer
that exposes apply_chat_template. Returns None if not found."""
seen = set()
queue = [clip]
attrs = ("tokenizer", "processor", "cond_stage_model", "patcher", "model", "transformer")
depth = 0
while queue and depth < 5:
next_q = []
for obj in queue:
if obj is None or id(obj) in seen:
continue
seen.add(id(obj))
try:
if hasattr(obj, "apply_chat_template"):
return obj
for attr in attrs:
child = getattr(obj, attr, None)
if child is not None:
next_q.append(child)
tkns = getattr(obj, "tokenizers", None)
if isinstance(tkns, dict):
next_q.extend(tkns.values())
except Exception:
continue
queue = next_q
depth += 1
return None
def _v2_advisor_analysis_prompt(
self,
prompt,
intent_prompt,
repair_candidates,
previous_run=None,
feedback_prompt="",
feedback_history="",
axis_feedback=None,
rating_label="",
):
previous_run = previous_run if isinstance(previous_run, dict) else {}
previous_prompt = self._v2_prompt_key(previous_run.get("encoded_prompt", "") or previous_run.get("prompt", ""))
axis_feedback = axis_feedback if isinstance(axis_feedback, dict) else {}
missing = axis_feedback.get("missing_axes", [])
resolved = axis_feedback.get("resolved_axes", [])
regressed = axis_feedback.get("regressed_axes", [])
wrong = axis_feedback.get("wrong_axes", [])
has_history = bool(str(feedback_history or "").strip())
if has_history:
analysis_system = (
"You are diagnosing a video generation prompt session. "
"Look at the session history and identify the root cause of why the same issues keep returning.\n\n"
"Output exactly one line:\n"
"DIAGNOSTIC: <what the session history reveals — why fixes are not sticking, "
"or what structural problem in the prompt causes recurring failures>"
)
else:
analysis_system = (
"You are analyzing a video generation prompt. "
"Identify what is specifically wrong or missing that caused this rating.\n\n"
"Output exactly one line:\n"
"DIAGNOSTIC: <what is wrong or missing in the prompt, referencing actual words from it>"
)
user_lines = []
user_lines.append(f"Rating: {rating_label or 'unrated'}")
if missing:
user_lines.append(f"Missing: {', '.join(missing)}")
if regressed:
user_lines.append(f"Regressed: {', '.join(regressed)}")
if resolved:
user_lines.append(f"Resolved: {', '.join(resolved)}")
if wrong:
user_lines.append(f"Wrong: {', '.join(wrong)}")
user_lines.append(f"Intent: {self._v2_prompt_key(intent_prompt) or 'same as prompt'}")
user_lines.append(f"Prompt: {self._v2_prompt_key(prompt)}")
if previous_prompt and previous_prompt != self._v2_prompt_key(prompt):
user_lines.append(f"Previous prompt: {previous_prompt}")
candidates = [
str(c.get("text", "")) for c in (repair_candidates or [])[:4]
if isinstance(c, dict) and str(c.get("text", "")).strip()
]
if candidates:
user_lines.append(f"Memory suggests: {', '.join(candidates)}")
if has_history:
user_lines.append(f"Session history:\n{feedback_history}")
user_lines.append("\nOne sentence.")
return analysis_system, "\n".join(user_lines)
def _v2_advisor_prompt(
self,
prompt,
intent_prompt,
repair_candidates,
previous_run=None,
mode="Full",
feedback_prompt="",
feedback_history="",
analysis="",
rating_label="",
intent_expansions="",
):
has_feedback = bool(str(feedback_prompt or "").strip())
previous_run = previous_run if isinstance(previous_run, dict) else {}
previous_prompt = self._v2_prompt_key(previous_run.get("encoded_prompt", "") or previous_run.get("prompt", ""))
user_lines = []
user_lines.append(f"Issue with the previous output: {rating_label or 'needs improvement'}")
if has_feedback:
user_lines.append(f"User's note: {str(feedback_prompt).strip()}")
if intent_expansions:
user_lines.append(f"About this scene: {intent_expansions}")
if feedback_history:
user_lines.append(f"Past feedback:\n{feedback_history}")
candidates = [
str(c.get("text", "")) for c in (repair_candidates or [])[:4]
if isinstance(c, dict) and str(c.get("text", "")).strip()
]
if candidates and not has_feedback:
user_lines.append(f"Suggested additions: {', '.join(candidates)}")
if analysis:
user_lines.append(f"Analysis: {analysis}")
user_lines.append("")
user_lines.append(f"Original intent: {self._v2_prompt_key(intent_prompt) or 'same as prompt'}")
user_lines.append(f"Prompt to fix: {previous_prompt or self._v2_prompt_key(prompt) or 'none'}")
user_lines.append("")
user_lines.append("Write the corrected prompt.")
system = V2_PROMPT_ADVISOR_SYSTEM_PROMPT.strip()
if analysis:
system += f"\n\nSession diagnostic: {analysis}"
return system, "\n".join(user_lines)
def _v2_generate_advisor_text(self, clip, system_prompt, user_prompt, seed=None, image=None, thinking=True, max_length=800):
if clip is None or not hasattr(clip, "generate") or not hasattr(clip, "decode"):
return "", "Advisor: unavailable; connected CLIP does not expose text generation."
max_length = max(128, int(max_length or 800))
try:
# Tokenization: three layers in order of preference.
# Layer 1: wrapper natively accepts system_prompt kwarg.
tokens = None
try:
tokens = clip.tokenize(str(user_prompt), image=image, min_length=1, thinking=bool(thinking), system_prompt=str(system_prompt))
except TypeError:
try:
tokens = clip.tokenize(str(user_prompt), image=image, system_prompt=str(system_prompt))
except TypeError:
pass
if tokens is None:
# Layer 2: apply chat template via underlying HuggingFace tokenizer.
chat_tokenizer = self._v2_find_chat_tokenizer(clip)
advisor_prompt = None
if chat_tokenizer is not None:
try:
messages = [
{"role": "system", "content": str(system_prompt)},
{"role": "user", "content": str(user_prompt)},
]
advisor_prompt = chat_tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
except Exception:
pass
if advisor_prompt is None:
# Layer 3: flat string with completion anchor.
advisor_prompt = str(system_prompt) + "\n\n" + str(user_prompt) + "\n\nOutput:"
try:
tokens = clip.tokenize(advisor_prompt, image=image, min_length=1, thinking=bool(thinking))
except TypeError:
try:
tokens = clip.tokenize(advisor_prompt, image=image)
except TypeError:
tokens = clip.tokenize(advisor_prompt)
generate_kwargs = dict(
do_sample=True,
max_length=max_length,
temperature=0.7,
top_k=50,
top_p=0.92,
min_p=0.05,
repetition_penalty=1.3,
no_repeat_ngram_size=5,
presence_penalty=0.0,
seed=seed if seed else None,
)
try:
generated_ids = clip.generate(tokens, **generate_kwargs)
except TypeError:
generate_kwargs.pop("no_repeat_ngram_size", None)
generated_ids = clip.generate(tokens, **generate_kwargs)
text = clip.decode(generated_ids, skip_special_tokens=True)
return str(text or "").strip(), "Advisor: generated."
except Exception as error:
return "", f"Advisor: generation failed: {error}"
def _v2_parse_advisor_response(self, text, prompt_labels=("REPAIRED_PROMPT", "PROMPT")):
text = re.sub(r"```.*?```", "", str(text or ""), flags=re.DOTALL).strip()
text = re.sub(r"</?start_of_turn>|<end_of_turn>", "", text).strip()
diagnostic = ""
repaired = ""
label_pattern = "|".join(re.escape(str(label)) for label in (prompt_labels or ("PROMPT",)))
diagnostic_match = re.search(r"(?im)^\s*DIAGNOSTIC\s*:\s*(.+)$", text)
repaired_match = re.search(rf"(?ims)^\s*(?:{label_pattern})\s*:\s*(.+)$", text)
known_label_match = re.search(r"(?im)^\s*(?:DIAGNOSTIC|REPAIRED_PROMPT|NEGATIVE_PROMPT|PROMPT)\s*:", text)
if diagnostic_match:
diagnostic = re.sub(r"\s+", " ", diagnostic_match.group(1)).strip()
if repaired_match:
repaired = repaired_match.group(1).strip()
repaired = re.split(r"(?im)^\s*(?:DIAGNOSTIC|REPAIRED_PROMPT|NEGATIVE_PROMPT|PROMPT)\s*:", repaired)[0].strip()
elif text and not diagnostic_match and not known_label_match:
repaired = text.strip()
repaired = re.sub(r"(?im)^\s*(?:DIAGNOSTIC|REPAIRED_PROMPT|NEGATIVE_PROMPT|PROMPT)\s*:\s*", "", repaired).strip()
repaired = re.sub(r"\s+", " ", repaired).strip(" \t\r\n\"'")
if len(repaired) > 1600:
repaired = repaired[:1600].rsplit(" ", 1)[0].strip()
return diagnostic, repaired
def _v2_validate_advisor_prompt(self, current_prompt, advised_prompt, intent_prompt, intent_phrases, clip=None, encode_cache=None, has_user_feedback=False):
current = self._v2_prompt_key(current_prompt)
advised = self._v2_prompt_key(advised_prompt)
intent = self._v2_prompt_key(intent_prompt) or current
if not advised:
return False, "empty prompt"
if self._prompt_looks_like_refusal(advised):
return False, "refusal text"
if advised == current:
return True, "unchanged"
# When the user provided explicit feedback, skip the intent-distance and
# protected-category checks — the user is asking for specific changes and
# those guards would silently block what was requested.
if not has_user_feedback:
body_similarity = self._v2_prompt_body_similarity(advised, intent)
semantic_similarity = self._v2_text_semantic_similarity(clip, advised, intent, encode_cache=encode_cache) if clip is not None else 0.0
if body_similarity < 0.30 and semantic_similarity < 0.58:
return False, "too far from intent"
generated_phrases = self._ordered_prompt_phrases(advised)
current_phrases = self._ordered_prompt_phrases(current)
protected_categories = {"appearance", "subject", "environment"}
for phrase in generated_phrases[:24]:
text = phrase.get("text", "")
if self._v2_phrase_represented_by(text, current_phrases) or self._v2_phrase_represented_by(text, intent_phrases or []):
continue
category = self._v2_primary_category_for_text(text)
if category in protected_categories:
return False, f"new protected {category} phrase"
return True, "validated"
def _v2_prompt_advisor(
self,
clip,
mode,
current_prompt,
intent_prompt,
axis_feedback,
repair_candidates,
previous_run=None,
intent_phrases=None,
allow_prompt_change=False,
seed=0,
image=None,
thinking=True,
encode_cache=None,
feedback_prompt="",
feedback_history="",
max_length=800,
rating_profile=None,
intent_expansions="",
):
mode = self._v2_advisor_mode(mode)
if mode == "Off":
return current_prompt, "Advisor: off.", "", False, ""
if not self._v2_prompt_key(current_prompt):
return current_prompt, "Advisor: skipped; prompt empty.", "", False, ""
rating_profile = rating_profile if isinstance(rating_profile, dict) else {}
is_perfect = rating_profile.get("key") == "like"
repair_signal = bool(axis_feedback.get("missing_axes") or axis_feedback.get("wrong_axes"))
has_feedback = bool(str(feedback_prompt or "").strip())
previous_run = previous_run if isinstance(previous_run, dict) else {}
prev_intent = self._v2_prompt_key(previous_run.get("intent_source_prompt", "") or previous_run.get("intent_prompt", ""))
current_intent = self._v2_prompt_key(intent_prompt)
intent_changed = bool(prev_intent and current_intent and self._v2_prompt_body_similarity(prev_intent, current_intent) < 0.50)
if mode in {"Only prompt", "Full"} and not repair_signal and not is_perfect and not has_feedback:
return current_prompt, "Advisor: skipped; no repair signal and no user feedback.", "", False, ""
# Pass 1: diagnostic analysis — runs for Full and Only diagnostics when no explicit feedback
# was given. Explicit feedback already names the problem; the diagnostic is for identifying
# patterns and root causes across the session when the user hasn't said what's wrong.
analysis = ""
if mode in {"Full", "Only diagnostics"} and not has_feedback:
analysis_system, analysis_user = self._v2_advisor_analysis_prompt(
current_prompt,
intent_prompt,
repair_candidates,
previous_run=previous_run,
feedback_prompt=feedback_prompt,
feedback_history=feedback_history,
axis_feedback=axis_feedback,
rating_label=rating_profile.get("label", ""),
)
analysis_raw, analysis_status = self._v2_generate_advisor_text(
clip, analysis_system, analysis_user, seed=seed, image=image, thinking=thinking,
max_length=1200,
)
analysis, _ = self._v2_parse_advisor_response(analysis_raw, prompt_labels=("ANALYSIS",))
if not analysis and analysis_raw:
analysis = re.sub(r"\s+", " ", analysis_raw).strip()[:400]
if not analysis:
# LLM failed or produced empty output - build a rule-based fallback so
# the diagnostic history always grows and future runs have context.
parts = []
if axis_feedback.get("regressed_axes"):
parts.append(f"{', '.join(axis_feedback['regressed_axes'])} regressing after repair")
if axis_feedback.get("missing_axes"):
parts.append(f"missing {', '.join(axis_feedback['missing_axes'])}")
if axis_feedback.get("wrong_axes"):
parts.append(f"wrong {', '.join(axis_feedback['wrong_axes'])}")
analysis = f"Session note: {'; '.join(parts)}." if parts else f"Rating: {rating_profile.get('label', 'unrated')}."
if mode == "Only diagnostics":
return current_prompt, f"Advisor: diagnostics only. {analysis or 'No analysis returned.'}", analysis, False, ""
# Guard checks before pass 2.
if not allow_prompt_change and not has_feedback:
return current_prompt, f"Advisor: prompt changes disabled. {analysis}".strip(), analysis, False, ""
if intent_changed and not has_feedback:
return current_prompt, f"Advisor: intent changed; repair skipped. {analysis}".strip(), analysis, False, ""
# Pass 2: repair — builds on the analysis from pass 1 (or runs standalone for Only prompt).
repair_system, repair_user = self._v2_advisor_prompt(
current_prompt,
intent_prompt,
repair_candidates,
previous_run=previous_run,
mode=mode,
feedback_prompt=feedback_prompt,
feedback_history=feedback_history,
analysis=analysis,
rating_label=rating_profile.get("label", "") if isinstance(rating_profile, dict) else "",
intent_expansions=intent_expansions,
)
raw, gen_status = self._v2_generate_advisor_text(
clip, repair_system, repair_user, seed=seed, image=image, thinking=thinking, max_length=max_length
)
if not raw:
return current_prompt, gen_status, analysis, False, ""
prompt_labels = ("PROMPT",) if mode == "Only prompt" else ("REPAIRED_PROMPT", "PROMPT")
_, advised = self._v2_parse_advisor_response(raw, prompt_labels=prompt_labels)
valid, reason = self._v2_validate_advisor_prompt(
current_prompt,
advised,
intent_prompt,
intent_phrases or [],
clip=clip,
encode_cache=encode_cache,
has_user_feedback=has_feedback,
)
if not valid:
return current_prompt, f"Advisor: rejected ({reason}). {analysis}".strip(), analysis, False, advised
if self._v2_prompt_key(advised) == self._v2_prompt_key(current_prompt):
return current_prompt, f"Advisor: no change. {analysis}".strip(), analysis, False, ""
return advised, f"Advisor: applied repair. {analysis}".strip(), analysis, True, advised
def _v2_encoded_prompts_output(self, positive_prompt, advisor_diagnostic="", pre_advisor_prompt="", advisor_suggested=""):
parts = [f"Positive prompt: {str(positive_prompt or '').strip()}"]
positive_key = self._v2_prompt_key(positive_prompt)
suggested_key = self._v2_prompt_key(advisor_suggested) if advisor_suggested else ""
if advisor_suggested:
label = "Advisor suggestion (applied)" if suggested_key == positive_key else "Advisor suggestion (rejected)"
parts.append(f"{label}: {str(advisor_suggested).strip()}")
if advisor_diagnostic:
parts.append(f"Advisor analysis: {str(advisor_diagnostic).strip()}")
if pre_advisor_prompt and self._v2_prompt_key(pre_advisor_prompt) != positive_key:
parts.append(f"Pre-advisor prompt: {str(pre_advisor_prompt).strip()}")
return "\n\n".join(parts)
def _v2_memory_entry_matches_current_scene(self, text, prompt, intent_phrases=None, clip=None, encode_cache=None):
clean = self._v2_clean_phrase_text(text)
prompt_clean = self._v2_clean_phrase_text(prompt)
if not clean:
return False
if self._v2_user_intent_prompt_is_vague(prompt_clean):
return True
if self._v2_repair_candidate_matches_intent(clean, self._v2_repair_intent_roots(prompt_clean)):
return True
if self._v2_phrase_represented_by(clean, intent_phrases or []):
return True
if clip is not None and prompt_clean:
return self._v2_text_semantic_similarity(clip, clean, prompt_clean, encode_cache=encode_cache) >= 0.62
return False
def _v2_compose_lucky_prompt(
self,
prompt,
phrases,
global_state,
intent_prompt="",
intent_phrases=None,
clip=None,
encode_cache=None,
):
memory = global_state.get("phrase_memory", {})
scored = []
for text, entry in memory.items():
if not isinstance(entry, dict):
continue
if not self._v2_auto_inject_entry_allowed(entry, prompt):
continue
primary = str(entry.get("primary", "details")).lower()
if primary in {"action", "camera", "details"} and not self._v2_memory_entry_matches_current_scene(
entry.get("text", text),
intent_prompt or prompt,
intent_phrases,
clip=clip,
encode_cache=encode_cache,
):
continue
score = float(entry.get("score", 0.0))
score += min(1.0, int(entry.get("liked_count", 0)) * 0.14)
score += min(1.2, sum(int(v) for v in entry.get("wanted_axes", {}).values()) * 0.10)
score -= min(1.6, int(entry.get("bad_count", 0)) * 0.22)
score -= min(1.1, int(entry.get("wrong_count", 0)) * 0.18)
score -= min(1.8, int(entry.get("auto_inject_blocked_count", 0)) * 0.65)
if score <= 0.10:
continue
scored.append((score, entry.get("primary", "details"), str(entry.get("text", text)).strip()))
scored = sorted(scored, key=lambda item: (item[0], item[1] == "action"), reverse=True)
current_texts = [str(item.get("text", "")).strip().lower() for item in phrases]
selected = []
seen = set()
for phrase in current_texts:
if phrase and phrase not in seen:
selected.append(phrase)
seen.add(phrase)
for _, _, text in scored:
key = text.lower()
if not key or key in seen:
continue
selected.append(text)
seen.add(key)
if len(selected) >= 14:
break
if not selected:
return prompt, "Lucky: on | memory empty, used current prompt."
lucky_prompt = ", ".join(selected)
return lucky_prompt, f"Lucky: on | composed {len(selected)} phrase(s): {', '.join(selected[:8])}{'...' if len(selected) > 8 else ''}."
def _v2_aspect_bucket(self, width, height):
width = max(1, int(width or 1))
height = max(1, int(height or 1))
ratio = width / float(height)
if ratio >= 1.95:
return "ultrawide"
if ratio >= 1.20:
return "landscape"
if ratio <= 0.52:
return "vertical"
if ratio <= 0.83:
return "portrait"
return "square"
def _v2_image_fingerprint(self, image):
if not isinstance(image, torch.Tensor) or image.numel() == 0:
return ""
try:
frame = image.detach().float()
if frame.dim() == 4:
frame = frame[0]
if frame.dim() != 3:
return ""
if frame.shape[-1] in {1, 3, 4}:
frame = frame.permute(2, 0, 1).unsqueeze(0)
else:
frame = frame.unsqueeze(0)
sample = F.interpolate(frame[:, :3], size=(8, 8), mode="bilinear", align_corners=False)
quantized = torch.clamp(sample[0].detach().cpu(), 0.0, 1.0).mul(255).round().to(torch.uint8)
return md5(quantized.numpy().tobytes()).hexdigest()[:16]
except Exception:
return ""
def _v2_image_metadata(self, source_image, previous_metadata=None):
if not isinstance(source_image, torch.Tensor) or source_image.numel() == 0:
return {}, "Vision image: none."
shape = list(source_image.shape)
if len(shape) < 3:
return {}, "Vision image: unsupported shape."
if len(shape) >= 4:
batch = int(shape[0])
height = int(shape[1])
width = int(shape[2])
else:
batch = 1
height = int(shape[0])
width = int(shape[1])
fingerprint = self._v2_image_fingerprint(source_image)
previous_fingerprint = ""
if isinstance(previous_metadata, dict):
previous_fingerprint = str(previous_metadata.get("fingerprint", ""))
changed = bool(previous_fingerprint and fingerprint and previous_fingerprint != fingerprint)
metadata = {
"width": width,
"height": height,
"batch": batch,
"aspect_ratio": round(width / float(max(1, height)), 6),
"aspect_bucket": self._v2_aspect_bucket(width, height),
"fingerprint": fingerprint,
"changed_from_previous": changed,
}
changed_text = "changed" if changed else ("same/first" if fingerprint else "unknown")
return metadata, (
f"Vision image: {width}x{height} {metadata['aspect_bucket']} "
f"ratio={metadata['aspect_ratio']:.3f} fingerprint={fingerprint or 'none'} {changed_text}."
)
def _v2_clip_vision_summary(self, clip_vision_output):
tensors = []
def walk(value, path="root", depth=0):
if depth > 3 or value is None:
return
if isinstance(value, torch.Tensor):
tensors.append((path, value))
return
if isinstance(value, dict):
for key, item in value.items():
walk(item, f"{path}.{key}", depth + 1)
return
for key in ("image_embeds", "last_hidden_state", "penultimate_hidden_states", "hidden_states"):
if hasattr(value, key):
walk(getattr(value, key), f"{path}.{key}", depth + 1)
walk(clip_vision_output)
summaries = []
for key, tensor in tensors[:8]:
try:
detached = tensor.detach().float().cpu()
summaries.append({
"key": key,
"shape": list(detached.shape),
"mean": round(float(detached.mean().item()), 6),
"std": round(float(detached.std().item()) if detached.numel() > 1 else 0.0, 6),
"norm": round(float(detached.norm().item()), 6),
})
except Exception:
continue
if not summaries:
return {}, "CLIP Vision: none."
digest_source = json.dumps(summaries, sort_keys=True)
summary = {
"tensor_count": len(tensors),
"fields": summaries,
"fingerprint": md5(digest_source.encode("utf-8")).hexdigest()[:16],
}
return summary, f"CLIP Vision: {len(tensors)} tensor field(s), fingerprint={summary['fingerprint']}."
def _v2_update_vision_memory(self, global_state, clip_vision_output=None, source_image=None):
memory = global_state.setdefault("vision_memory", {})
previous_image = memory.get("last_image") if isinstance(memory.get("last_image"), dict) else {}
image_metadata, image_status = self._v2_image_metadata(source_image, previous_image)
clip_summary, clip_status = self._v2_clip_vision_summary(clip_vision_output)
current = {
"image": image_metadata,
"clip_vision": clip_summary,
}
if image_metadata:
memory["last_image"] = image_metadata
if clip_summary:
memory["last_clip_vision"] = clip_summary
memory["last_context"] = current
return current, f"{image_status} {clip_status}"
def _v2_gen_context_snapshot(self, lora_stack, sampler_context, vision_context):
"""Build a compact, serializable snapshot of the generation context for this run.
sampler_context is read from state["last_sampler_context"], written there by
FunPack LTXAV Scene Chain Sampler when a refinement_key is connected.
All sampler fields are None when the sampler has no refinement_key wired.
"""
loras = []
if isinstance(lora_stack, dict):
for entry in lora_stack.get("loras", []):
if isinstance(entry, dict) and entry.get("name"):
loras.append({
"name": str(entry["name"]),
"type": str(entry.get("type", "general")),
"weight": round(float(entry.get("weight", 1.0)), 4),
})
sc = sampler_context if isinstance(sampler_context, dict) else {}
image_fp = ""
if isinstance(vision_context, dict):
image_fp = str(vision_context.get("image", {}).get("fingerprint", "") or "")
return {
"loras": loras,
"carry_i2v_guides": sc.get("carry_i2v_guides"),
"frame_overlap": sc.get("frame_overlap"),
"transitions_enabled": sc.get("transitions_enabled"),
"image_fp": image_fp,
}
def _v2_gen_context_diff(self, previous_ctx, current_ctx):
"""Return a list of change dicts describing what differed between two gen context snapshots."""
if not isinstance(previous_ctx, dict) or not isinstance(current_ctx, dict):
return []
changes = []
prev_fp = str(previous_ctx.get("image_fp") or "")
curr_fp = str(current_ctx.get("image_fp") or "")
if prev_fp and curr_fp and prev_fp != curr_fp:
changes.append({"field": "image_fp"})
for flag in ("carry_i2v_guides", "transitions_enabled"):
pv = previous_ctx.get(flag)
cv = current_ctx.get(flag)
if pv is not None and cv is not None and pv != cv:
changes.append({"field": flag, "prev": pv, "curr": cv})
pov = previous_ctx.get("frame_overlap")
cov = current_ctx.get("frame_overlap")
if pov is not None and cov is not None and pov != cov:
changes.append({"field": "frame_overlap", "prev": pov, "curr": cov})
prev_loras = {e["name"]: e for e in (previous_ctx.get("loras") or []) if isinstance(e, dict) and e.get("name")}
curr_loras = {e["name"]: e for e in (current_ctx.get("loras") or []) if isinstance(e, dict) and e.get("name")}
for name, entry in prev_loras.items():
if name not in curr_loras:
changes.append({"field": "lora_removed", "name": name, "type": entry.get("type", "general"), "weight": entry.get("weight", 1.0)})
else:
pw = float(entry.get("weight", 1.0))
cw = float(curr_loras[name].get("weight", 1.0))
if abs(pw - cw) > 0.01:
changes.append({"field": "lora_weight", "name": name, "type": entry.get("type", "general"), "prev": pw, "curr": cw})
for name, entry in curr_loras.items():
if name not in prev_loras:
changes.append({"field": "lora_added", "name": name, "type": entry.get("type", "general"), "weight": entry.get("weight", 1.0)})
return changes
def _v2_gen_context_insight(self, previous_ctx, current_ctx, rating_label, has_previous_run):
"""Generate a human-readable insight connecting context changes to the current rating."""
if not has_previous_run or not isinstance(previous_ctx, dict) or not isinstance(current_ctx, dict):
return ""
diff = self._v2_gen_context_diff(previous_ctx, current_ctx)
if not diff:
return ""
profile = V2_RATING_PROFILES.get(rating_label) or V2_RATING_PROFILES.get(V2_RATING_ALIASES.get(str(rating_label), ""), {})
rating_key = profile.get("key", "")
appearance_wrong = rating_key == "wrong_appearance"
positive = rating_key in {"like", "loved_it", "nailed_it"}
negative = float(profile.get("reward", 0.5)) < 0.3
lines = []
for ch in diff:
field = ch["field"]
if field == "image_fp":
if appearance_wrong:
lines.append("Source image changed — appearance baseline shifted; this rating may reflect the new reference, not a prompt problem.")
elif negative:
lines.append("Source image changed since the last rated run.")
else:
lines.append("Source image changed since the last rated run.")
elif field == "carry_i2v_guides":
if not ch["curr"] and appearance_wrong:
lines.append("carry_i2v_guides is now OFF — appearance drift is likely from missing i2v guides, not the prompt itself.")
elif not ch["curr"] and negative:
lines.append("carry_i2v_guides turned off — character/scene consistency guides are no longer active.")
elif ch["curr"] and positive:
lines.append("carry_i2v_guides turned on — guides may be contributing to this improvement.")
elif ch["curr"] and not ch["prev"]:
lines.append("carry_i2v_guides turned on since the last run.")
else:
lines.append("carry_i2v_guides turned off since the last run.")
elif field == "transitions_enabled":
if ch["curr"] and not ch["prev"]:
lines.append("Transitions turned on since the last run.")
else:
lines.append("Transitions turned off since the last run.")
elif field == "frame_overlap":
lines.append(f"Frame overlap changed: {ch['prev']} → {ch['curr']}.")
elif field == "lora_added":
stem = os.path.splitext(os.path.basename(str(ch["name"])))[0][:40]
ltype = ch.get("type", "general")
wt = ch.get("weight", 1.0)
if appearance_wrong:
lines.append(f"LoRA added: '{stem}' ({ltype}, w={wt:.2f}) — may be changing appearance.")
elif positive:
lines.append(f"LoRA added: '{stem}' ({ltype}, w={wt:.2f}) — may be contributing to this improvement.")
else:
lines.append(f"LoRA added: '{stem}' ({ltype}, w={wt:.2f}).")
elif field == "lora_removed":
stem = os.path.splitext(os.path.basename(str(ch["name"])))[0][:40]
ltype = ch.get("type", "general")
wt = ch.get("weight", 1.0)
if appearance_wrong:
lines.append(f"LoRA removed: '{stem}' ({ltype}, was w={wt:.2f}) — may explain appearance change.")
else:
lines.append(f"LoRA removed: '{stem}' ({ltype}, was w={wt:.2f}).")
elif field == "lora_weight":
stem = os.path.splitext(os.path.basename(str(ch["name"])))[0][:40]
ltype = ch.get("type", "general")
lines.append(f"LoRA '{stem}' ({ltype}) weight changed: {ch['prev']:.2f} → {ch['curr']:.2f}.")
if not lines:
return ""
return "Context changes since last rated run:\n" + "\n".join(f" • {ln}" for ln in lines)
def _v2_update_negative_prompt_memory(self, global_state, previous_run, rating_profile, axis_feedback=None):
memory = global_state.setdefault("negative_prompt_memory", {})
tags = memory.setdefault("tags", {})
if not isinstance(previous_run, dict) or rating_profile.get("skip_learning"):
return "Negative repair memory: no update."
axis_feedback = axis_feedback or self._v2_axis_feedback(rating_profile, global_state.get("last_missing_axes", []))
wrong_axes = set(axis_feedback.get("wrong_axes", []))
missing_axes = set(axis_feedback.get("missing_axes", []))
reward = float(rating_profile.get("reward", 0.0))
should_store = bool(wrong_axes) or rating_profile.get("key") == "awful" or reward < -0.30
if not should_store:
return "Negative repair memory: no poor-rated tags added."
raw_intent_prompt = self._v2_prompt_key(previous_run.get("intent_prompt", ""))
has_explicit_intent = bool(
raw_intent_prompt and
not previous_run.get("intent_prompt_is_vague") and
not self._v2_user_intent_prompt_is_vague(raw_intent_prompt)
)
intent_prompt = raw_intent_prompt if has_explicit_intent else ""
intent_phrases = (
[phrase for phrase in previous_run.get("intent_phrases", []) or [] if isinstance(phrase, dict)]
if has_explicit_intent else
[]
)
family_key, family_slot, _ = self._v2_intent_family_slot(global_state, intent_prompt, create=False)
added = []
skipped_intent = 0
def remember_tag(text, phrase_axes, source, intent_locked=False):
nonlocal skipped_intent
text = self._v2_clean_phrase_text(text)
if not text:
return
if intent_locked or self._v2_negative_text_conflicts_with_request(
text,
current_prompt=previous_run.get("prompt", "") if has_explicit_intent else "",
intent_prompt=intent_prompt,
intent_phrases=intent_phrases,
family_slot=family_slot,
):
skipped_intent += 1
return
if wrong_axes and not (phrase_axes & wrong_axes):
return
if not wrong_axes and missing_axes and not (phrase_axes & missing_axes):
return
entry = tags.setdefault(text, {
"text": text,
"count": 0,
"axes": {},
"family_key": family_key,
"source": source,
"probe_count": 0,
"last_rating": "",
"last_seen_iter": 0,
})
entry["count"] = int(entry.get("count", 0)) + (2 if wrong_axes else 1)
entry["family_key"] = family_key or entry.get("family_key", "")
entry["source"] = source
entry["last_rating"] = rating_profile.get("label", "")
entry["last_seen_iter"] = int(global_state.get("total_iterations", 0)) + 1
axes = entry.setdefault("axes", {})
for axis in self._v2_order_axes((phrase_axes & (wrong_axes or missing_axes)) or phrase_axes):
axes[axis] = int(axes.get(axis, 0)) + 1
added.append(text)
for phrase in previous_run.get("phrases", []) or []:
if not isinstance(phrase, dict):
continue
remember_tag(
phrase.get("text", ""),
self._v2_phrase_axes(phrase),
"enhancer_or_prompt",
intent_locked=has_explicit_intent and bool(phrase.get("intent_locked")),
)
for candidate in previous_run.get("repair_candidates", []) or []:
if not isinstance(candidate, dict):
continue
remember_tag(
candidate.get("text", ""),
set(candidate.get("axes", [])),
"repair_candidate",
intent_locked=False,
)
if not added:
suffix = f" Skipped {skipped_intent} intent-locked tag(s)." if skipped_intent else ""
return f"Negative repair memory: no matching poor-rated tags.{suffix}"
memory["tags"] = dict(sorted(
tags.items(),
key=lambda item: (int(item[1].get("count", 0)), int(item[1].get("last_seen_iter", 0))),
reverse=True,
)[:80])
suffix = f" Skipped {skipped_intent} intent-locked tag(s)." if skipped_intent else ""
return f"Negative repair memory: added {len(added)} poor-rated tag(s): {', '.join(added[:8])}.{suffix}"
def _v2_negative_probe_due(self, text, entry, iter_num):
if not isinstance(entry, dict):
return False
count = int(entry.get("count", 0))
if count <= 1:
return False
digest = int(md5(str(text or "").encode("utf-8")).hexdigest()[:6], 16)
return (int(iter_num or 0) + digest) % 11 == 0
def _v2_repair_negative_prompt(
self,
negative_prompt,
global_state,
axis_feedback=None,
current_prompt="",
intent_prompt="",
intent_phrases=None,
intent_family_slot=None,
):
prompt = self._v2_prompt_key(negative_prompt)
memory = global_state.get("negative_prompt_memory", {})
tags = memory.get("tags", {}) if isinstance(memory, dict) else {}
if not isinstance(tags, dict) or not tags:
return prompt, "Negative repair: no stored poor-rated tags."
wanted_axes = set(axis_feedback.get("missing_axes", [])) | set(axis_feedback.get("wrong_axes", [])) if isinstance(axis_feedback, dict) else set()
ranked = []
withheld_intent = 0
withheld_family = 0
withheld_probe = 0
current_family_key = intent_family_slot.get("family_key", "") if isinstance(intent_family_slot, dict) else ""
for text, entry in tags.items():
if not isinstance(entry, dict):
continue
clean = self._v2_clean_phrase_text(entry.get("text", text))
if not clean or self._v2_prompt_contains_text(prompt, clean):
continue
if self._v2_negative_text_conflicts_with_request(
clean,
current_prompt=current_prompt,
intent_prompt=intent_prompt,
intent_phrases=intent_phrases or [],
family_slot=intent_family_slot,
):
withheld_intent += 1
continue
entry_family = str(entry.get("family_key", ""))
if entry_family and current_family_key and entry_family != current_family_key:
withheld_family += 1
continue
if self._v2_negative_probe_due(clean, entry, int(global_state.get("total_iterations", 0)) + 1):
entry["probe_count"] = int(entry.get("probe_count", 0)) + 1
withheld_probe += 1
continue
axes = set(entry.get("axes", {}).keys()) if isinstance(entry.get("axes"), dict) else set()
axis_bonus = 1.0 if wanted_axes and axes & wanted_axes else 0.0
score = float(entry.get("count", 0)) + axis_bonus + min(0.5, int(entry.get("last_seen_iter", 0)) * 0.001)
ranked.append((score, clean))
ranked.sort(reverse=True)
additions = [text for _, text in ranked[:8]]
if not additions:
withheld = []
if withheld_intent:
withheld.append(f"intent/current {withheld_intent}")
if withheld_family:
withheld.append(f"other-family {withheld_family}")
if withheld_probe:
withheld.append(f"probe {withheld_probe}")
detail = f" Withheld: {', '.join(withheld)}." if withheld else ""
return prompt, f"Negative repair: stored poor-rated tags already present.{detail}"
repaired = f"{prompt}, {', '.join(additions)}" if prompt else ", ".join(additions)
withheld = []
if withheld_intent:
withheld.append(f"intent/current {withheld_intent}")
if withheld_family:
withheld.append(f"other-family {withheld_family}")
if withheld_probe:
withheld.append(f"probe {withheld_probe}")
detail = f" Withheld: {', '.join(withheld)}." if withheld else ""
return repaired, f"Negative repair: added {len(additions)} persistent poor-rated tag(s): {', '.join(additions)}.{detail}"
def _v2_lora_type(self, lora_type):
lora_type = str(lora_type or "general").strip().lower()
return "action" if lora_type == "concept" else lora_type
def _v2_lora_words(self, text):
return {
word.strip().lower()
for word in re.split(r"[\s,;:_\\/\-().\[\]{}]+", text or "")
if self._is_valuable_token(word.strip())
}
def _v2_lora_feedback_axes(self, lora_type):
lora_type = self._v2_lora_type(lora_type)
if lora_type == "action":
return {"action", "details"}
if lora_type == "quality":
return {"quality"}
if lora_type == "style":
return {"quality", "details"}
if lora_type == "character":
return {"action", "details"}
return set(V2_FEEDBACK_AXES)
def _v2_update_lora_suggestions(self, lora_stack, prompt_history, global_state, phrases, rating_profile, axis_feedback=None):
if not isinstance(lora_stack, dict) or not lora_stack.get("loras"):
return "LoRA suggestions: no FunPack LoRA stack connected."
memory = global_state.setdefault("lora_weight_memory", {})
phrase_by_category = {}
for phrase in phrases:
scores = (
phrase.get("effective_category_scores", phrase.get("category_scores", {})) or
{phrase.get("primary", "details"): 1.0}
)
primary, confidence = self._v2_scores_primary(scores)
words = self._v2_lora_words(phrase.get("text", ""))
phrase_by_category.setdefault(primary, set()).update(words)
for category, score in scores.items():
if category != primary and float(score or 0.0) >= max(0.30, confidence * 0.72):
phrase_by_category.setdefault(category, set()).update(words)
suggestions = {}
parts = []
axis_feedback = axis_feedback or self._v2_axis_feedback(
rating_profile,
global_state.get("last_missing_axes", []),
)
missing_axes = set(axis_feedback.get("missing_axes", []))
satisfied_axes = set(axis_feedback.get("satisfied_axes", []))
resolved_axes = set(axis_feedback.get("resolved_axes", []))
reward = float(rating_profile.get("reward", 0.0))
for entry in lora_stack.get("loras", []):
name = entry.get("name", "")
raw_type = entry.get("type", "general")
lora_type = self._v2_lora_type(raw_type)
lora_id = entry.get("id") or self._lora_state_id(name, raw_type)
# Relevance comes from the type the user set on the LoRA and the categories
# present in the prompt — NOT from the file's name. Matching prompt words against
# the filename guessed at what a LoRA does from what someone happened to call it,
# which is a different feature from learning its weight, and a worse one.
relation = 0.20 if lora_type == "general" else 0.0
if lora_type == "action":
relation = max(relation, 0.62 if phrase_by_category.get("action") else 0.0)
elif lora_type == "quality":
relation = max(relation, 0.62 if phrase_by_category.get("quality") else 0.0)
elif lora_type == "style":
relation = max(relation, 0.54 if phrase_by_category.get("style") or phrase_by_category.get("camera") else 0.0)
elif lora_type == "character":
relation = max(relation, 0.58 if phrase_by_category.get("subject") or phrase_by_category.get("appearance") else 0.0)
state = memory.setdefault(lora_id, {"name": name, "type": lora_type, "offset_ratio": 0.0, "iterations": 0})
offset = float(state.get("offset_ratio", 0.0))
step = 0.036 if lora_type == "action" else 0.026
lora_axes = self._v2_lora_feedback_axes(lora_type)
if rating_profile.get("key") == "like":
offset += step * max(0.18, relation) * 0.55
elif rating_profile.get("key") == "nailed_it":
quality_type = lora_type in {"quality", "style"}
offset += step * max(0.18, relation) * (0.44 if quality_type else 0.24)
elif rating_profile.get("key") == "loved_it":
quality_type = lora_type in {"quality", "style"}
offset += step * max(0.18, relation) * (0.55 if quality_type else 0.30)
elif rating_profile.get("key") == "awful":
offset -= step * max(0.22, relation) * 1.20
elif lora_axes & missing_axes:
offset += step * max(0.28, relation) * 1.35
elif lora_axes & resolved_axes:
offset += step * max(0.18, relation) * 0.62
elif lora_axes & satisfied_axes:
offset += step * max(0.12, relation) * 0.18
else:
offset += step * reward * max(0.12, relation)
offset = _clamp(offset, -0.75, 0.55)
state["offset_ratio"] = round(float(offset), 6)
state["iterations"] = int(state.get("iterations", 0)) + 1
state["type"] = lora_type
state["name"] = name
base_model = float(entry.get("base_model_weight", entry.get("model_weight", 1.0)))
model_weight = base_model * (1.0 + offset)
action = (
"mute" if model_weight == 0.0 else
"reduce" if abs(model_weight) < abs(base_model) else
"hold" if abs(model_weight - base_model) < 1e-6 else
"boost"
)
suggestions[lora_id] = {
"name": name,
"type": lora_type,
"model_weight": model_weight,
"base_model_weight": base_model,
"offset_ratio": offset,
"relation": relation,
"action": action,
"rating_label": rating_profile.get("label", ""),
"rating_range": (
"reduce awful"
if rating_profile.get("key") == "awful" else
"boost " + "+".join(self._v2_order_axes(lora_axes & missing_axes))
if lora_axes & missing_axes else
"preserve " + "+".join(self._v2_order_axes(lora_axes & resolved_axes))
if lora_axes & resolved_axes else
rating_profile.get("key", "")
),
}
parts.append(f"{name}[{lora_type}] rel={relation:.2f} next={model_weight:+.3f}")
prompt_history["lora_weight_suggestions"] = suggestions
return "LoRA suggestions: " + " | ".join(parts)
_V2_ADVISOR_MAX_TOKENS = 1600
def refine_v2(self, positive_prompt, clip=None, rating="Missing action", refinement_key="",
reset_session=False, lora_stack=None, im_feeling_lucky=False, user_intent_prompt="",
refinement_key_input="", positive_conditioning=None, clip_vision_output=None,
source_image=None, model=None, mode="Refine", advisor_mode="Off", advisor_thinking=True,
advisor_clip=None, feedback_prompt="", temporal_style="natural",
split_by_transitions=False, split_transition_placement="start", reference_injection=False,
value_guidance=True, latent=None, seed_output_connected=False,
steer_mode="relative", absolute_strength=0.6,
h3_phrase_emphasis=False,
_seed=None, _seed_source="fresh seed", _scene_seeds=None, _velocity_keys=None,
batch_variants=1, guess_mode=False, guess_direction="up", guess_range=1.0,
guess_freeze_seed=True, movie_editor_scene_ratings=None, scene_segments=None,
variables=None, h3_references=None, link_texts=None):
seed = int(_seed) if _seed is not None else random.randint(1, 0xffffffffffffffff)
# Project-scoped `$name` variables, resolved DEAD LAST (after shortcut-expand + split)
# so they can never affect scene-cut detection. Empty/None = no-op.
try:
from .templates import resolve_variables as _resolve_variables
except ImportError:
from templates import resolve_variables as _resolve_variables
_prompt_variables = variables or None
# Exactly what the editor handed to nodes OUTSIDE Studio (server._expanded_link_texts):
# shortcuts rolled and $variables resolved. When a wired conditioning owns the prompt,
# one of these IS the string that was encoded, so the emphasis can be placed on the
# real tokens instead of on Studio's own copy of a prompt it did not encode.
_link_texts = link_texts if isinstance(link_texts, dict) else None
# MiniMax H3 ref2va reference media, in the order the user listed it. No other model
# family has the tokenizer kwarg it feeds, so it is inert when unset or on LTX.
# minimax_h3.resolve_ref_spec explains why the ORDER is load-bearing.
try:
from .minimax_h3 import normalize_ref_spec as _normalize_ref_spec
except ImportError:
from minimax_h3 import normalize_ref_spec as _normalize_ref_spec
# New Studio pass: per-run log suppression starts over, so a failure that also
# happened last run is reported again rather than deduped away forever.
_log.begin_run()
_run_started = time.perf_counter()
self._v2_reset_encode_tally()
_h3_references = _normalize_ref_spec(h3_references) or None
encode_cache = {}
linked_refinement_key = str(refinement_key_input or "").strip()
if linked_refinement_key:
refinement_key = linked_refinement_key
# Pre-shortcut prompt, kept so Batch Training can re-resolve shortcuts with different
# seeds to build N variant conditionings (Studio owns conditioning production).
_raw_positive_prompt = str(positive_prompt or "")
_raw_intent_prompt = str(user_intent_prompt or "")
# revolver_commit only on the real positive prompt: the one draw per run that advances
# the shortcut revolver's cycle. The intent prompt peeks, and does so FIRST — if it
# echoes the positive prompt's triggers, both then resolve to the same chamber (the
# same alignment the shared seed used to give them).
user_intent_prompt, intent_shortcut_status, intent_shortcut_applied = self._v2_apply_global_shortcuts(user_intent_prompt, seed=seed)
positive_prompt, shortcut_status, shortcut_applied = self._v2_apply_global_shortcuts(
positive_prompt, seed=seed, revolver_commit=True)
if intent_shortcut_applied:
shortcut_status = (
f"{shortcut_status}\nIntent {intent_shortcut_status}"
if shortcut_applied else
f"Intent {intent_shortcut_status}"
)
# When a batch is being generated, the rating selected on the Studio node is stale (it
# would apply learning to the PREVIOUS run). Ignore it — the batch is rated afterward in
# the Batch Training panel. '-Just forget it-' is skip_learning, so no learning fires.
if int(batch_variants or 1) > 1:
rating = "-Just forget it-"
rating_profile = normalize_refiner_v2_rating(rating)
rating_label = rating_profile.get("label", str(rating))
execution_mode = self._v2_execution_mode(mode)
learning_mode = execution_mode == "Learning"
advisor_mode = self._v2_advisor_mode(advisor_mode)
advisor_clip = advisor_clip if advisor_clip is not None else clip
prompt_only_mode = execution_mode == "Prompt only"
state, state_status = self._v2_load_state(refinement_key, reset_session=reset_session,
encoder_family=self._encoder_family(clip))
if reset_session:
extra_reset = self._v2_reset_prompt_keys(refinement_key, _raw_positive_prompt, _raw_intent_prompt)
if extra_reset:
state_status = f"{state_status} (also reset keys: {', '.join(extra_reset)})"
global_state = state.setdefault("global", {})
global_state.setdefault("phrase_memory", {})
global_state.setdefault("axis_conditioning_memory", {})
global_state.setdefault("negative_prompt_memory", {})
# Interactive-Guessing dual auto-cap (learned safe ceiling). Applied on NORMAL runs only —
# never during the guessing ladder itself, which deliberately sweeps the spread.
_guess_spread_cap = None
if not guess_mode:
_gsf = float(global_state.get("guess_safe_factor", 0) or 0)
if _gsf > 0:
absolute_strength = min(float(absolute_strength), _gsf) # (a) soft strength pre-limit
_gss = float(global_state.get("guess_safe_spread", 0) or 0)
if _gss > 0:
_guess_spread_cap = _gss # (b) output-spread clamp (the guarantee)
global_state.setdefault("lora_weight_memory", {})
global_state.setdefault("preferred_context_memory", {})
global_state.setdefault("intent_alignment_memory", {})
global_state.setdefault("intent_family_memory", {})
global_state.setdefault("perfect_anchors", {})
global_state.setdefault("appearance_anchor", {})
global_state.setdefault("appearance_drift", {})
global_state.setdefault("variant_evidence", {})
global_state.setdefault("intent_preference_phrases", {})
global_state.setdefault("conditioning_deltas", {})
global_state.setdefault("active_repair_axes", [])
global_state.setdefault("vision_memory", {})
global_state.setdefault("loss_history", [])
global_state.setdefault("intent_expansion_memory", {})
global_state.setdefault("session_source_mean_count", 0)
global_state.setdefault("liked_dir", {})
global_state.setdefault("bad_dir", {})
global_state.setdefault("successful_seed_memory", {})
previous_run = state.get("last_run")
previous_run_refusal = self._v2_run_looks_like_refusal(previous_run)
has_previous_run = isinstance(previous_run, dict) and not previous_run_refusal
if previous_run_refusal:
learning_profile = dict(rating_profile)
learning_profile["skip_learning"] = True
learning_profile["refusal_filtered"] = True
else:
learning_profile = rating_profile if has_previous_run else dict(V2_RATING_PROFILES["Initial discovery"], label="Initial discovery")
previous_rating_label = str(global_state.get("last_rating_label", "Initial discovery"))
previous_missing_axes = (
list(global_state.get("last_missing_axes", []))
if previous_rating_label != "Initial discovery" else
None
)
me_ratings = [
item for item in (movie_editor_scene_ratings or [])
if isinstance(item, dict) and str(item.get("rating", "") or "").strip()
]
use_me_scene_ratings = (
bool(me_ratings)
and has_previous_run
and int((previous_run or {}).get("scene_count") or len((previous_run or {}).get("scene_texts") or []) or 1) > 1
)
if use_me_scene_ratings:
learning_profile = dict(rating_profile)
learning_profile["skip_learning"] = True
axis_feedback = self._v2_axis_feedback(learning_profile, previous_missing_axes)
me_status, aggregate_profile = self._v2_apply_movie_editor_scene_ratings(
global_state,
previous_run,
me_ratings,
int(global_state.get("total_iterations", 0)) + 1,
clip=clip,
encode_cache=encode_cache,
seed_output_connected=bool(seed_output_connected),
refinement_key=refinement_key,
)
memory_status = me_status
seed_memory_status = ""
intent_family_status = ""
intent_learning_status = ""
negative_memory_status = ""
if aggregate_profile and has_previous_run and not aggregate_profile.get("skip_learning"):
self._v2_update_streaks(global_state, aggregate_profile, update_conditioning_strength=not prompt_only_mode)
repair_feedback, repair_persistence_status = self._v2_active_repair_feedback(
global_state,
self._v2_axis_feedback(aggregate_profile or learning_profile, previous_missing_axes),
aggregate_profile or learning_profile,
)
else:
axis_feedback = self._v2_axis_feedback(learning_profile, previous_missing_axes)
memory_status = self._v2_update_phrase_memory(
global_state,
previous_run,
learning_profile,
int(global_state.get("total_iterations", 0)) + 1,
axis_feedback,
)
self._v2_update_concept_delta_memory(global_state, previous_run, learning_profile)
self._v2_update_concept_pair_dirs(global_state, previous_run, learning_profile)
seed_memory_status = self._v2_update_successful_seed_memory(
global_state,
previous_run,
learning_profile,
int(global_state.get("total_iterations", 0)) + 1,
seed_output_connected=bool(seed_output_connected),
)
self._v2_update_path_outcomes(
global_state,
previous_run,
learning_profile,
int(global_state.get("total_iterations", 0)) + 1,
)
intent_family_status, _ = self._v2_update_intent_family_memory(
global_state,
previous_run,
learning_profile,
int(global_state.get("total_iterations", 0)) + 1,
axis_feedback,
)
self._v2_update_appearance_anchor(
global_state,
previous_run,
learning_profile,
int(global_state.get("total_iterations", 0)) + 1,
)
negative_memory_status = self._v2_update_negative_prompt_memory(
global_state,
previous_run,
learning_profile,
axis_feedback,
)
intent_learning_status = self._v2_update_intent_alignment_memory(
global_state,
previous_run,
learning_profile,
int(global_state.get("total_iterations", 0)) + 1,
axis_feedback,
)
memory_status = f"{memory_status}\n{seed_memory_status}\n{intent_family_status}\n{intent_learning_status}"
self._v2_update_conditioning_memory(global_state, previous_run, learning_profile, axis_feedback)
# Always train the value function in the background (cheap: a small MLP, no diffusion
# calls) so the reward asset accumulates regardless of whether guidance is applied.
# value_guidance only controls APPLICATION (ascent below) - a user who runs with it
# off still builds the VF, so enabling guidance later works immediately.
if (has_previous_run and refinement_key and not learning_profile.get("skip_learning")
and self._v2_reward_admissible(learning_profile)):
n = self._v2_train_value_function(
refinement_state_path(refinement_key, "value_fn", prefix="refine_v2", extension="pt"),
(previous_run or {}).get("conditioning"),
float(learning_profile.get("reward", 0.0)),
)
if n is not None:
print(f"[FunPackRefiner] Value function updated — {n} samples")
# output_guidance's value function, trained on the sampler's x0_snapshot from the
# run this rating scores (whole-run granularity — see _v2_train_output_value_function
# for why this sits at the single overall-reward call site, not the per-scene ones).
n_out = self._v2_train_output_value_function(refinement_key, float(learning_profile.get("reward", 0.0)))
if n_out is not None:
print(f"[FunPackRefiner] Output value function updated — {n_out} samples")
# DynaShift negative memory: pair the sampler's pending raw latent with this
# rating — promote into the per-key negative bank when the rating marks the run
# a bad outcome (intrusions: awful / wrong_appearance; or the quality-missing
# family via reward <= threshold — see negative_memory.is_negative_profile),
# discard otherwise. Separate from the value-function block above because
# wrong_appearance is deliberately NOT reward-admissible there (repair
# semantics), yet it is the canonical intrusion signal here.
if has_previous_run and refinement_key and not learning_profile.get("skip_learning"):
try:
try:
from .negative_memory import consume_pending, is_negative_profile
except ImportError:
from negative_memory import consume_pending, is_negative_profile
n_neg = consume_pending(
refinement_key,
is_negative_profile(learning_profile),
rating_key=learning_profile.get("key"),
)
if n_neg is not None:
print(f"[FunPackRefiner] DynaShift negative bank updated — {n_neg} entr"
f"{'y' if n_neg == 1 else 'ies'}")
except Exception as _e:
print(f"[FunPackRefiner] DynaShift negative intake failed: {_e}")
# Absolute store: the same rating also feeds the keyless, prompt-agnostic taste prior.
# Runs even with no refinement_key (Absolute is global), so standalone runs still build it.
# Skipped for Wrong-* repair ratings (skip_value_function): Absolute reads reward as pure
# quality, so a 0.0/low repair reward on a good gen would push it into the global bad_dir.
if has_previous_run:
self._v2_learn_absolute(previous_run, learning_profile)
if has_previous_run and not learning_profile.get("skip_learning"):
self._v2_update_streaks(global_state, learning_profile, update_conditioning_strength=not prompt_only_mode)
repair_feedback, repair_persistence_status = self._v2_active_repair_feedback(
global_state,
axis_feedback,
learning_profile,
)
# Custom shortcut keys (e.g. a 'cs'-tagged cumshot shortcut) ALSO fire on a plain
# single-scene run, not just a rated multi-scene Movie Editor chain — but the
# use_me_scene_ratings branch above was the ONLY place that ever trained them, and
# it requires BOTH a multi-scene run AND explicit per-scene ratings from the editor.
# Most generations are single-scene with one overall rating, so those keys were
# never created at all. Train them here too, ADDITIVELY (the default key above keeps
# learning exactly as it always has — unlike the multi-scene path, this does NOT make
# custom keys exclusive of the default; that exclusivity is the multi-scene path's own
# documented design, not something to risk replicating into this far more common,
# heavily-relied-upon single-scene path without it being separately tested).
if has_previous_run and not learning_profile.get("skip_learning"):
fired_keys = set()
for ks in (previous_run.get("scene_refinement_keys") or []):
fired_keys |= set(ks or [])
fired_keys.discard(str(refinement_key or "").strip())
if fired_keys:
trained_keys = self._v2_learn_scene_into_keys(
sorted(fired_keys), refinement_key, previous_run, learning_profile,
int(global_state.get("total_iterations", 0)) + 1, axis_feedback,
bool(seed_output_connected),
)
if trained_keys:
memory_status += f"\nTrained custom key(s): {', '.join(trained_keys)}"
vision_context, vision_status = self._v2_update_vision_memory(
global_state,
clip_vision_output=clip_vision_output,
source_image=source_image,
)
_sc_path = refinement_state_path(refinement_key, "sampler_ctx", prefix=self.V2_STATE_PREFIX)
_last_sampler_context = {}
if os.path.exists(_sc_path):
try:
with open(_sc_path, "r", encoding="utf-8") as _scf:
_last_sampler_context = json.load(_scf)
except Exception:
pass
current_gen_context = self._v2_gen_context_snapshot(lora_stack, _last_sampler_context, vision_context)
previous_gen_context = previous_run.get("gen_context") if isinstance(previous_run, dict) else None
context_insight = self._v2_gen_context_insight(previous_gen_context, current_gen_context, rating_label, has_previous_run)
analysis_prompt = self._v2_prompt_key(positive_prompt)
intent_prompt = self._v2_prompt_key(user_intent_prompt)
intent_prompt_is_vague = self._v2_user_intent_prompt_is_vague(intent_prompt)
current_prompt_refusal = self._prompt_looks_like_refusal(analysis_prompt)
phrases = [] if current_prompt_refusal else self._v2_classify_phrases(
clip,
self._ordered_prompt_phrases(analysis_prompt),
global_state,
encode_cache=encode_cache,
)
intent_phrases = []
if intent_prompt and not intent_prompt_is_vague and not current_prompt_refusal:
intent_phrases = self._v2_classify_phrases(
clip,
self._ordered_prompt_phrases(intent_prompt),
global_state,
encode_cache=encode_cache,
)
intent_source_prompt = self._v2_intent_source_prompt(analysis_prompt, intent_prompt, intent_prompt_is_vague)
current_family_key, current_family_slot, current_family_similarity = self._v2_intent_family_slot(
global_state,
intent_source_prompt,
create=bool(intent_source_prompt and not current_prompt_refusal),
)
self._v2_mark_intent_locks(phrases, intent_source_prompt, intent_phrases, current_family_slot)
self._v2_mark_intent_locks(intent_phrases, intent_source_prompt, intent_phrases, current_family_slot)
self._v2_mark_semantic_intent_locks(
clip,
phrases,
intent_source_prompt,
intent_phrases,
encode_cache=encode_cache,
)
refusal_status = ""
repair_status = "Prompt repair: none."
repair_candidates = []
intent_alignment_status = "Intent alignment: no explicit original intent."
intent_alignment_adjustments = []
perfect_repair_status = "Perfect repairs: none."
perfect_repair_adjustments = []
advisor_status = "Advisor: off."
model_patch_status = "Model patch: no model connected."
advisor_diagnostic = ""
advisor_applied = False
advisor_suggested = ""
pre_advisor_prompt = ""
feedback_history = ""
advisor_rating_label = rating_label
is_perfect_rating = has_previous_run and learning_profile.get("key") == "like"
prev_intent_key = self._v2_prompt_key(
(previous_run or {}).get("intent_source_prompt", "") or (previous_run or {}).get("intent_prompt", "")
)
intent_drifted = bool(
prev_intent_key and intent_source_prompt and
self._v2_prompt_body_similarity(prev_intent_key, intent_source_prompt) < 0.50
)
image_changed = bool(isinstance(vision_context, dict) and vision_context.get("changed_from_previous", False))
perfect_freeze = False # Disabled: conditioning memory + VF maintain quality direction
# without freezing the prompt, which blocked intentional prompt changes after a Perfect.
if current_prompt_refusal:
prompt_to_encode = analysis_prompt
lucky_status = "Prompt refusal filter: enhancer refusal detected; current prompt will not be stored or learned."
encoded_role = "refusal passthrough"
refusal_status = "Current prompt refused by enhancer; storage skipped."
elif learning_mode:
prompt_to_encode = analysis_prompt
lucky_status = "Learning mode: observing only; Lucky, intent alignment, and prompt repair skipped."
intent_alignment_status = "Intent alignment: skipped in Learning mode."
perfect_repair_status = "Perfect repairs: skipped in Learning mode."
repair_status = "Prompt repair: skipped in Learning mode."
encoded_role = "learning passthrough"
elif perfect_freeze:
frozen = self._v2_prompt_key(previous_run.get("encoded_prompt", "")) or analysis_prompt
prompt_to_encode = frozen
lucky_status = "Perfect freeze: prompt locked to previous perfect output."
intent_alignment_status = "Intent alignment: skipped (perfect freeze)."
perfect_repair_status = "Perfect repairs: skipped (perfect freeze)."
repair_status = "Prompt repair: skipped (perfect freeze)."
encoded_role = "perfect frozen prompt"
elif im_feeling_lucky and clip is not None:
prompt_to_encode, lucky_status = self._v2_compose_lucky_prompt(
analysis_prompt,
phrases,
global_state,
intent_prompt=intent_source_prompt,
intent_phrases=intent_phrases,
clip=clip,
encode_cache=encode_cache,
)
encoded_role = "lucky prompt"
else:
aligned_prompt, intent_alignment_status, intent_alignment_adjustments = self._v2_apply_intent_alignment_memory(
analysis_prompt,
phrases,
intent_prompt,
intent_phrases,
global_state,
)
# Prompt repair removed (Stage 2). The intent-aligned prompt is encoded as-is; the only
# remaining prompt-rewriting path is the advisor (when enabled). Perfect-repair phrase
# injection, missing-axis repair, and the emphasis pass are gone. repair_feedback /
# active_repair_axes is KEPT — it's the "what's missing" signal the advisor still reads.
perfect_repair_status = "Perfect repairs: removed."
perfect_repair_adjustments = []
prompt_to_encode = aligned_prompt
emphasis_status = "Prompt emphasis: removed."
advisor_active = advisor_mode != "Off"
repair_status = "Prompt repair: removed."
if im_feeling_lucky and clip is None:
lucky_status = f"Lucky: unavailable without CLIP | connected CONDITIONING accepted. {emphasis_status}"
else:
lucky_status = f"Lucky: off | trained memory only. {emphasis_status}"
encoded_role = "current prompt"
if not current_prompt_refusal:
feedback_history = self._v2_format_advisor_feedback_history(global_state)
intent_expansions = self._v2_format_intent_expansions(global_state, intent_source_prompt)
pre_advisor_prompt = prompt_to_encode
advisor_rating_label = rating_label if has_previous_run else "No previous output (first run or session reset)"
prompt_to_encode, advisor_status, advisor_diagnostic, advisor_applied, advisor_suggested = self._v2_prompt_advisor(
advisor_clip,
advisor_mode,
prompt_to_encode,
intent_source_prompt,
repair_feedback,
repair_candidates,
previous_run=previous_run,
intent_phrases=intent_phrases,
allow_prompt_change=not learning_mode and advisor_mode in {"Only prompt", "Full"},
seed=seed,
image=source_image,
thinking=advisor_thinking,
encode_cache=encode_cache,
feedback_prompt=feedback_prompt,
feedback_history=feedback_history,
max_length=self._V2_ADVISOR_MAX_TOKENS,
rating_profile=learning_profile,
intent_expansions=intent_expansions,
)
if advisor_applied:
encoded_role = "advisor repaired prompt"
if advisor_diagnostic:
self._v2_record_advisor_diagnostic(global_state, advisor_diagnostic, advisor_rating_label, int(global_state.get("total_iterations", 0)))
if advisor_mode != "Off" and not learning_mode and repair_candidates and not advisor_applied:
additions = [c["text"] for c in repair_candidates if isinstance(c, dict) and str(c.get("text", "")).strip()]
if additions:
base = str(pre_advisor_prompt or "").strip()
prompt_to_encode = f"{base}, {', '.join(additions)}" if base else ", ".join(additions)
repair_status = repair_status.replace("suggestion(s)", "phrase(s)").replace("suggested", "added")
# Resolve $name variables into the base prompt before encoding (single-scene path +
# the modified_positive output). The multi-scene per-scene texts are resolved after the
# split below; this keeps the order shortcuts -> split -> variables intact.
if _prompt_variables:
prompt_to_encode = _resolve_variables(prompt_to_encode, _prompt_variables)[0]
cond, meta, encode_status, conditioning_owner = self._v2_conditioning_source(
clip,
prompt_to_encode,
positive_conditioning,
encode_cache=encode_cache,
reference_image=source_image,
h3_references=_h3_references,
)
# Before any steering runs: the reference's vision rows are Qwen's encoding of the
# picture and must come out of this node exactly as they went in.
meta = self._v2_stash_reference_rows(cond, meta)
fallback_graph = render_refinement_loss_graph(refinement_key, "v2", "clip", 0, 0.0, [])
if not isinstance(cond, torch.Tensor):
status = f"ERROR: V2 could not prepare conditioning | {encode_status}"
training_info = f"Rating: {rating_label}\n{memory_status}\n{lucky_status}\n{encode_status}"
return (
[],
status,
training_info,
fallback_graph,
self._v2_encoded_prompts_output(prompt_to_encode, advisor_diagnostic=advisor_diagnostic, pre_advisor_prompt=pre_advisor_prompt, advisor_suggested=advisor_suggested),
model,
latent,
)
# Load value function for conditioning memory modulation (confidence + gradient boost)
_vf_for_memory = None
_eff_key_early = str(refinement_key_input or refinement_key or "").strip()
if not learning_mode and not prompt_only_mode and _eff_key_early:
try:
try:
from .value_function import OnlineValueFunction as _OVF
except ImportError:
from value_function import OnlineValueFunction as _OVF
_vf_path = refinement_state_path(_eff_key_early, "value_fn", prefix="refine_v2", extension="pt")
if os.path.exists(_vf_path):
import torch as _torch
with _torch.inference_mode(False):
_vf_for_memory = _OVF.load(_vf_path)
if not _vf_for_memory.is_ready():
_vf_for_memory = None
except Exception:
_vf_for_memory = None
if learning_mode:
refined = cond
adaptation_status = "Adaptation: Learning mode; conditioning vectors passed through unchanged."
elif prompt_only_mode:
refined = cond
adaptation_status = "Adaptation: Prompt only mode; conditioning vectors passed through unchanged."
elif str(steer_mode).lower() == "absolute":
# Pure Absolute: skip the per-prompt conditioning memory entirely. Steering happens
# only via the keyless global-taste pull in _v2_finalize_conditioning, so the result
# is moved toward general preferences "despite the prompt". (Both keeps per-prompt
# memory and layers the global pull on top.)
refined = cond
adaptation_status = "Adaptation: Absolute mode; per-prompt memory bypassed, global taste pull applied at output."
else:
_current_final = [t for t in (prompt_to_encode or "").split(",") if len(t.strip()) > 1]
_prev_final = previous_run.get("final_phrase_texts", []) if isinstance(previous_run, dict) else []
_concept_dir, _concept_strength = self._v2_concept_delta_direction(
global_state, _current_final, _prev_final
)
refined, adaptation_status = self._v2_apply_conditioning_memory(
cond,
global_state,
learning_profile,
repair_feedback,
intent_family_slot=current_family_slot,
vf=_vf_for_memory,
concept_delta_dir=_concept_dir,
concept_delta_strength=_concept_strength,
current_final_texts=_current_final,
)
prompt_key = self._v2_prompt_key(analysis_prompt)
prompt_history = None
if current_prompt_refusal:
lora_status = "LoRA suggestions: skipped for prompt-enhancer refusal."
else:
prompt_history = state.setdefault("prompt_histories", {}).setdefault(prompt_key, {
"canonical_prompt": analysis_prompt,
"history": [],
})
lora_status = self._v2_update_lora_suggestions(
lora_stack,
prompt_history,
global_state,
phrases,
learning_profile,
axis_feedback,
)
should_record_iteration = bool(has_previous_run or not current_prompt_refusal)
if should_record_iteration:
global_state["total_iterations"] = int(global_state.get("total_iterations", 0)) + 1
else:
global_state["total_iterations"] = int(global_state.get("total_iterations", 0))
learning_loss = max(0.02, (1.0 - max(-1.0, min(1.0, float(global_state.get("avg_reward_ema", 0.0))))) * 0.5)
loss_history = list(global_state.get("loss_history", []))[-511:]
if should_record_iteration:
loss_history.append({
"total_iteration": int(global_state["total_iterations"]),
"learning_loss": round(float(learning_loss), 6),
"rating": int(rating_profile.get("legacy_score", 6)),
"rating_label": rating_label,
"similarity": 1.0,
"scheduler_mode": "automatic",
"mode": "clip",
})
global_state["loss_history"] = loss_history
phrase_preview = "\n".join(
(
f"- {item['text']} ["
f"{item.get('machine_primary', item['primary'])}->{item['primary']}:"
f"{item['confidence']:.2f}/{item['source']}/"
f"ctx={item.get('context_source', 'none')}/"
f"e{int(item.get('category_evidence_count', 0))}]"
)
for item in phrases[:10]
) or "none"
category_diagnostics = self._v2_category_diagnostics(phrases)
training_guidance = self._v2_training_guidance(
has_previous_run,
learning_profile,
axis_feedback,
phrases,
memory_status,
lora_status,
)
prompt_preview = re.sub(r"\s+", " ", prompt_to_encode).strip()
if len(prompt_preview) > 240:
prompt_preview = prompt_preview[:237].rstrip() + "..."
state_repair_candidates = self._v2_serializable_repair_candidates(repair_candidates)
state_intent_alignment_adjustments = [
{
"text": str(item.get("text", "")),
"source": str(item.get("source", "")),
"action": str(item.get("action", "")),
}
for item in list(intent_alignment_adjustments) + list(perfect_repair_adjustments)
if isinstance(item, dict) and str(item.get("text", "")).strip()
]
if prompt_history is not None:
prompt_history["canonical_prompt"] = analysis_prompt
prompt_history.setdefault("history", [])
prompt_history["history"] = list(prompt_history.get("history", []))[-119:]
prompt_history["history"].append({
"iteration": int(global_state["total_iterations"]),
"rating_label": rating_label if has_previous_run else "Unrated",
"encoded_role": encoded_role,
"prompt": prompt_to_encode,
"phrases": phrases,
"repair_candidates": state_repair_candidates,
"vision_context": vision_context,
"intent_alignment_adjustments": state_intent_alignment_adjustments,
"advisor": {
"mode": advisor_mode,
"status": advisor_status,
"diagnostic": advisor_diagnostic,
"applied": bool(advisor_applied),
},
})
# ONE canonical split (of the RAW prompt) feeds BOTH the per-scene encode texts AND the
# per-scene refinement keys, so generation, attribution, preview and reset can never disagree
# on scene count or boundaries — no guard, no fallback. split_scenes walks the shortcut
# expansion pieces and cuts only at real transition triggers (qcut/cut), giving each scene's
# expanded text + keys; _v2_transition_scene_texts then prepends the anchor exactly as before.
split_scene_texts = []
split_scene_effects = []
scene_refinement_keys = []
current_scene_seeds = []
current_scene_seed_source = str(_seed_source or "fresh seed")
if split_by_transitions:
try:
# Prefer EXPLICIT scene boundaries handed in by the caller (the Movie Editor
# passes its own scene list). Falls back to text-splitting the prompt only for
# raw node-pack use where there is no structured scene list. Either way no
# `scene N` delimiter is ever needed in the prompt.
structural = isinstance(scene_segments, dict) and scene_segments.get("scenes") is not None
if structural:
canonical = split_scenes_from_segments(
scene_segments.get("anchor", ""), scene_segments.get("scenes", []),
placement=split_transition_placement,
postfix=scene_segments.get("postfix", ""))
# The editor's scene list is authoritative on COUNT — one conditioning scene
# per editor scene. Keep every scene, including a text-less i2v scene (it stays
# at its index as an anchor-only conditioning); dropping it would shift anchors.
canon_scenes = list(canonical.get("scenes", []) or [])
else:
canonical = split_scenes(_raw_positive_prompt, placement=split_transition_placement)
canon_scenes = [s for s in (canonical.get("scenes", []) or [])
if (s.get("expanded") or "").strip()]
if len(canon_scenes) >= 1:
anchor_exp = (canonical.get("anchor_expanded", "") or "").strip()
postfix_exp = (canonical.get("postfix_expanded", "") or "").strip()
if structural:
# Fold the anchor (prefix) and postfix (suffix) into each scene WITHOUT
# dropping empties (mirrors _v2_transition_scene_texts but count-preserving
# for the structural path).
split_scene_texts, split_scene_effects = [], []
for s in canon_scenes:
seg = (s.get("expanded") or "").strip()
if anchor_exp and seg:
sep = " " if anchor_exp[-1] in ".!?," else ", "
seg = anchor_exp + sep + seg
else:
seg = anchor_exp or seg
if postfix_exp and seg:
sep = " " if seg[-1] in ".!?," else ", "
seg = seg + sep + postfix_exp
elif postfix_exp:
seg = postfix_exp
split_scene_texts.append(seg)
split_scene_effects.append(s.get("effect"))
else:
segments = [(anchor_exp, None)] + \
[(s.get("expanded", ""), s.get("effect")) for s in canon_scenes]
split_scene_texts, split_scene_effects = self._v2_transition_scene_texts(segments)
# Variables resolve here — AFTER shortcut-expand and the transition split —
# so a variable whose value contains a comma or a trigger word can never
# change the scene count. Per-scene texts are what the chain sampler encodes.
if _prompt_variables and split_scene_texts:
split_scene_texts = [
_resolve_variables(t, _prompt_variables)[0] for t in split_scene_texts
]
scene_refinement_keys = [set(s.get("keys") or set()) for s in canon_scenes]
current_scene_seeds = self._v2_scene_seed_values(seed, len(split_scene_texts), _scene_seeds)
current_scene_seed_source = (
"successful seed memory" if _scene_seeds else "base seed + scene index"
)
except Exception as e:
print(f"[FunPackVideoRefinerV2] Transition scene prep failed: {e}")
if current_prompt_refusal:
state["last_run"] = None
else:
state["last_run"] = {
"prompt": analysis_prompt,
"encoded_prompt": prompt_to_encode,
"conditioning": tensor_to_serializable(refined.detach().cpu()),
"source_conditioning": tensor_to_serializable(cond.detach().cpu()),
"phrases": phrases,
"final_phrase_texts": [t for t in (p.strip().lower() for p in (prompt_to_encode or "").split(",")) if len(t) > 1],
"prev_final_phrase_texts": previous_run.get("final_phrase_texts", []) if isinstance(previous_run, dict) else [],
"intent_prompt": intent_prompt,
"intent_phrases": intent_phrases,
"intent_prompt_is_vague": bool(intent_prompt_is_vague),
"intent_source_prompt": intent_source_prompt,
"intent_family_key": current_family_key,
"intent_family_similarity": round(float(current_family_similarity), 6),
"intent_alignment_adjustments": state_intent_alignment_adjustments,
"advisor": {
"mode": advisor_mode,
"status": advisor_status,
"diagnostic": advisor_diagnostic,
"applied": bool(advisor_applied),
},
"repair_candidates": state_repair_candidates,
"vision_context": vision_context,
"encoded_role": encoded_role,
"auto_injected_prompt_additions": [
item.get("text", "")
for item in state_repair_candidates
if isinstance(item, dict) and str(item.get("text", "")).strip()
],
"rating_label": "Unrated",
"iteration": int(global_state["total_iterations"]),
"seed": int(seed),
"seed_source": str(_seed_source or "fresh seed"),
"seed_output_connected": bool(seed_output_connected),
"scene_count": len(split_scene_texts) if split_scene_texts else 1,
"scene_texts": list(split_scene_texts) if split_scene_texts else [],
"scene_seeds": current_scene_seeds,
# Non-default refinement keys that participated in each scene (sorted lists),
# so the NEXT run's rating pass can train every key that shaped the scene.
"scene_refinement_keys": [sorted(s) for s in scene_refinement_keys],
"gen_context": current_gen_context,
}
# "-Just forget it-" ("Just ignore it") and auto-discarded refusals must leave NO
# trace: skip the feedback-learning sinks too, not only the rating-learning ones above.
if not self._v2_rating_is_discard(learning_profile):
self._v2_update_advisor_feedback_history(global_state, feedback_prompt, advisor_rating_label, int(global_state.get("total_iterations", 0)))
if feedback_prompt and intent_source_prompt and not current_prompt_refusal:
self._v2_update_intent_expansion(global_state, intent_source_prompt, feedback_prompt)
state["global"] = global_state
self._v2_save_state(state, refinement_key)
loss_graph = render_refinement_loss_graph(
refinement_key=refinement_key,
scheduler_mode="v2-auto",
mode="clip",
total_iterations=int(global_state["total_iterations"]),
latest_learning_loss=float(learning_loss),
points=loss_history[-256:],
)
def _active(line):
"""Return line only if it contains meaningful content, not just 'none/off/skipped/idle'."""
if not str(line or "").strip():
return ""
low = line.lower()
if any(p in low for p in (": none.", ": none\n", " none.", "skipped.", ": off.", ": idle.", "skipped in", "disabled.")):
return ""
return line
if previous_run_refusal:
learning_reason = "previous prompt was an enhancer refusal"
elif learning_profile.get("key") == "continue":
learning_reason = "no rating — refining from session memory only"
elif use_me_scene_ratings:
learning_reason = f"trained on {len(me_ratings)} scene rating(s) from previous chain run"
elif learning_profile.get("skip_learning"):
learning_reason = "rating skipped learning"
elif not has_previous_run:
learning_reason = "no previous run yet"
else:
learning_reason = "trained on previous run"
intent_family_line = (
f"Intent family: {current_family_key[:12] if current_family_key else 'none'} "
f"(sim={current_family_similarity:.2f})"
)
learning_flag = "trained" if has_previous_run and not learning_profile.get("skip_learning") else "not applied"
refusal_flag = f" | REFUSAL DETECTED" if refusal_status else ""
seed_status = f"Seed: {int(seed)} ({str(_seed_source or 'fresh seed')})"
if current_scene_seeds:
seed_status += f" | scene seeds: {', '.join(str(item) for item in current_scene_seeds[:8])}"
if len(current_scene_seeds) > 8:
seed_status += f", ... ({len(current_scene_seeds)} total)"
status_lines = [
f"V2 {state_status} | {execution_mode} | {rating_label} | iter {global_state['total_iterations']} | learning {learning_flag}{refusal_flag}",
adaptation_status,
seed_status,
]
for line in (shortcut_status, repair_status, advisor_status, intent_alignment_status, vision_status, lucky_status):
if _active(line):
status_lines.append(line)
if perfect_freeze:
status_lines.append("Perfect freeze active.")
status = "\n".join(status_lines)
training_info = "\n\n".join(filter(None, [
(
"Run\n"
f"Mode: {execution_mode} | Encoded as: {encoded_role}\n"
f"Rating: {rating_label} | Learning: {learning_reason}\n"
f"Seed: {int(seed)} ({str(_seed_source or 'fresh seed')})\n"
f"Prompt: {prompt_preview}"
),
(
"Memory\n"
f"{self._v2_axis_feedback_status(axis_feedback)}\n"
f"{memory_status}"
+ (f"\n{_active(repair_persistence_status)}" if _active(repair_persistence_status) else "")
),
(
"Prompt Analysis\n"
f"{intent_family_line}\n"
f"{category_diagnostics}\n"
f"Phrases:\n{phrase_preview}"
),
(
"Adaptation\n"
f"{adaptation_status}\n\n"
f"Direction memory:\n{self._v2_direction_readout(global_state, repair_feedback)}"
+ ("\n\n" + "\n".join(filter(None, [
_active(intent_alignment_status),
_active(perfect_repair_status),
_active(repair_status),
_active(shortcut_status),
_active(advisor_status),
_active(lucky_status),
_active(vision_status),
model_patch_status if model is not None else "",
refusal_status if refusal_status else "",
])) if any(filter(None, [
_active(intent_alignment_status),
_active(perfect_repair_status),
_active(repair_status),
_active(shortcut_status),
_active(advisor_status),
_active(lucky_status),
_active(vision_status),
model_patch_status if model is not None else "",
refusal_status if refusal_status else "",
])) else "")
),
(
"Context\n"
f"{context_insight}"
) if context_insight else "",
(
"Guidance\n"
f"{training_guidance}"
) if training_guidance and _active(training_guidance) else "",
(
"LoRA\n"
f"{lora_status}"
) if _active(lora_status) else "",
]))
patched_model = model
model_patch_status = "Model patch: no model connected."
if model is not None:
model_strength = self._v2_auto_strength(global_state)
emphasis_phrases = [
c["text"] for c in repair_candidates
if isinstance(c, dict) and str(c.get("text", "")).strip()
]
emphasis_ranges = (
self._v2_find_phrase_token_ranges(clip, cond, emphasis_phrases, encode_cache=encode_cache)
if clip is not None and emphasis_phrases and self._v2_model_reads_attn2(model)
else []
)
patched = self._v2_apply_model_patches(
model, global_state, repair_feedback, learning_profile, emphasis_ranges, model_strength
)
if patched is not None:
patched_model = patched
axis_memory = global_state.get("axis_conditioning_memory", {})
active_dirs = []
liked_c = int(global_state.get("liked_dir", {}).get("direction_count", 0))
if liked_c >= 3:
active_dirs.append(f"liked({liked_c})")
for ax in V2_FEEDBACK_AXES:
ax_slot = axis_memory.get(ax, {}) if isinstance(axis_memory, dict) else {}
pos_c = int(ax_slot.get("positive", {}).get("direction_count", 0))
if pos_c >= 3:
active_dirs.append(f"{ax}({pos_c})")
dirs_str = ", ".join(active_dirs) if active_dirs else "none yet"
phrases_str = (
f": {', '.join(repr(p) for p in emphasis_phrases[:4])}"
if emphasis_phrases else ""
)
model_patch_status = (
f"Model patch (attn2 K/V injection): active\n"
f" Strength: {model_strength:.3f}\n"
f" Active directions injected: {dirs_str}\n"
f" Phrase emphasis: {len(emphasis_ranges)} position range(s){phrases_str}"
)
else:
model_patch_status = "Model patch: connected but no directions ready yet (need 3+ rated runs per slot)."
# --- LTX enhancements (temperature, temporal RoPE, attention anchors) ---
enhancement_status = ""
eff_key = refinement_key or ""
prev_reward = float(learning_profile.get("reward", 0.0)) if isinstance(learning_profile, dict) else 0.0
# Rating-gated commit of the previous gen's staged rescue trajectory: a positive
# rating teaches rescue what to steer toward, Awful what to steer away from,
# everything else is dropped (never lets an unrated/awful run pollute the banks).
if isinstance(learning_profile, dict):
_rk = learning_profile.get("key")
_verdict = "good" if _rk in {"like", "loved_it", "nailed_it"} else ("bad" if _rk == "awful" else "drop")
# Commit to every key the sampler may have captured under: the effective
# refinement key AND each pass's resolved velocity key (which differs when the
# user set a custom velocity_refinement_key). Dedup; skip empties.
_commit_keys = {k for k in ([eff_key] + list(_velocity_keys or [])) if str(k or "").strip()}
if _commit_keys:
try:
try:
from .samplers import commit_staged_velocity
except ImportError:
from samplers import commit_staged_velocity
for _ck in _commit_keys:
commit_staged_velocity(_ck, _verdict)
except Exception as _e:
print(f"[FunPackVideoRefinerV2] velocity commit failed: {_e}")
if patched_model is not None:
try:
try:
from .ltx_enhancements import build_enhancements, bless_attention_maps
except ImportError:
from ltx_enhancements import build_enhancements, bless_attention_maps
# Bless attention maps when the previous run was rated Perfect or Loved it
if eff_key and isinstance(learning_profile, dict) and learning_profile.get("key") in {"like", "loved_it", "nailed_it"}:
bless_attention_maps(eff_key)
try:
from .ltx_enhancements import bless_attn_weights, bless_kv
except ImportError:
from ltx_enhancements import bless_attn_weights, bless_kv
bless_attn_weights(eff_key)
bless_kv(eff_key)
patched_model = build_enhancements(
patched_model,
rating_profile=learning_profile if isinstance(learning_profile, dict) else {},
temporal_style=temporal_style,
refinement_key=eff_key,
reward=prev_reward,
reference_latent=latent if reference_injection else None,
conditioning=refined,
)
enhancement_status = f"\nLTX enhancements: temporal={temporal_style}, reward={prev_reward:+.2f}"
except Exception as e:
print(f"[FunPackVideoRefinerV2] LTX enhancements failed: {e}")
video_latent = latent # pass through unchanged by default; mask overwrites if it fires
try:
try:
from .ltx_enhancements import load_and_apply_creativity_mask
except ImportError:
from ltx_enhancements import load_and_apply_creativity_mask
result = load_and_apply_creativity_mask(
eff_key,
learning_profile if isinstance(learning_profile, dict) else {},
prev_reward,
latent=latent,
)
if result is not None:
video_latent = result
except Exception as e:
print(f"[FunPackVideoRefinerV2] Creativity mask failed: {e}")
output_conditioning = [(refined, meta)]
# A wired CONDITIONING can be a MULTI-ENTRY list — an r2v node emits the reference
# block and the encoded prompt as separate entries. `_v2_extract_conditioning` reads
# entry 0 and Studio rebuilt a single-entry list around it, so everything after the
# first entry was dropped on the floor: the character survived and the prompt did
# not, which is a tensor holding the vision block and nothing to say.
#
# The extra entries pass through UNCHANGED. Steering is fitted to the entry Studio
# measured; applying it blind to a companion entry of a different length and purpose
# would be a guess, and dropping them is the bug being fixed here.
if conditioning_owner == "CONDITIONING-owned" and isinstance(positive_conditioning, list) \
and len(positive_conditioning) > 1:
output_conditioning = output_conditioning + list(positive_conditioning[1:])
_log.note_on_change(
"studio:wired_extra_entries", "FunPackStudio",
f"the wired positive CONDITIONING has {len(positive_conditioning)} entries; "
f"entry 1 is steered and the other "
f"{len(positive_conditioning) - 1} pass through unchanged. They used to be "
f"discarded, which cut everything after the first entry out of the prompt.")
# Batch Training packs N variant entries (tagged 'funpack_batch_variant') onto the FINAL
# conditioning just before each return below — so it wraps whatever output_conditioning
# ends up being: single-scene OR multi-scene (transition split). Works WITH transitions.
_do_batch = int(batch_variants or 1) > 1 and not learning_mode and not prompt_only_mode
# Path planner: route the conditioning variant from per-config outcome history — lock a
# liked region's conditioning, widen the search to flee a disliked one. None with no data.
_conditioning_plan, _conditioning_plan_reason = self._v2_path_conditioning_plan(
global_state.get("path_outcomes", {}),
self._v2_seed_concepts_from_text(prompt_to_encode),
)
if _conditioning_plan_reason:
print(f"[FunPackVideoRefinerV2] {_conditioning_plan_reason}")
# The split RE-ENCODES every scene from text and its entries replace
# `output_conditioning` wholesale. It re-establishes H3's visual conditioning from
# `source_image` / `h3_references` — but a wired CONDITIONING carries a reference this
# node never received: nothing in the editor pipeline sets either of those, so the
# re-encode is textually blind to the picture and the identity is simply gone. Not
# drift — a different person. Studio cannot split a conditioning it did not build, so
# it keeps the one it was given.
if split_by_transitions and conditioning_owner == "CONDITIONING-owned":
_log.note_on_change(
"studio:split_vs_wired", "FunPackStudio",
"the positive CONDITIONING is wired, so the scene split is SKIPPED — "
"splitting means re-encoding each scene from text, and this node cannot "
"reproduce a reference it never received. Every scene uses the wired "
"conditioning. Disconnect it to let Studio split and encode per scene.")
status = status + ("\nTransition split: skipped — the wired positive "
"CONDITIONING owns the prompt") + enhancement_status
elif split_by_transitions:
try:
scene_texts = split_scene_texts
if scene_texts:
scene_conditionings = self._v2_transition_scene_conditionings(
clip, scene_texts, scene_effects=split_scene_effects, encode_cache=encode_cache,
# The per-scene entries replace `output_conditioning` wholesale, so H3's
# visual conditioning has to be re-established here or the anchor image
# and the ref2va references never reach the sampler at all.
reference_image=source_image, h3_references=_h3_references,
)
if scene_conditionings is not None:
output_conditioning = self._v2_apply_scene_seed_metadata(
scene_conditionings,
current_scene_seeds,
current_scene_seed_source,
)
scene_mode = (
"Scene chain mode: multi-entry conditioning for "
"FunPack LTXAV Scene Chain Sampler"
)
else:
scene_mode = "Scene chain mode unavailable: CLIP missing, single conditioning returned"
detected_count = len(scene_texts)
status_suffix = f"\nTransition split: {detected_count} scenes detected\n{scene_mode}"
status = status + status_suffix + enhancement_status
scene_lines = "\n".join(
f" Scene {i + 1}: {t}" for i, t in enumerate(scene_texts)
)
split_encoded_prompts = (
self._v2_encoded_prompts_output(
prompt_to_encode,
advisor_diagnostic=advisor_diagnostic,
pre_advisor_prompt=pre_advisor_prompt,
advisor_suggested=advisor_suggested,
)
+ f"\n\nDetected scenes:\n{scene_lines}"
)
if _do_batch:
output_conditioning = self._v2_pack_batch(
output_conditioning, batch_variants, _raw_positive_prompt, seed, clip,
encode_cache, source_image, prompt_to_encode, global_state, learning_profile,
repair_feedback, current_family_slot, _vf_for_memory, _concept_dir,
_concept_strength, _current_final,
guess_mode=guess_mode, guess_direction=guess_direction, guess_range=guess_range,
guess_freeze_seed=guess_freeze_seed)
return (
self._v2_finalize_conditioning(output_conditioning, refinement_key, value_guidance, steer_mode, absolute_strength, spread_cap=_guess_spread_cap, temporal_style=temporal_style, temporal_fallback_text=prompt_to_encode, scene_refinement_keys=scene_refinement_keys, learning_profile=learning_profile, conditioning_plan=_conditioning_plan, clip=clip, encode_cache=encode_cache, phrase_memory=global_state.get("phrase_memory"), axis_feedback=repair_feedback, h3_phrase_emphasis=h3_phrase_emphasis, auto_strength=self._v2_auto_strength(global_state), variables=_prompt_variables, link_texts=_link_texts),
status,
training_info,
loss_graph,
split_encoded_prompts,
patched_model,
video_latent,
)
except Exception as e:
print(f"[FunPackVideoRefinerV2] Transition split failed: {e}")
if _do_batch:
output_conditioning = self._v2_pack_batch(
output_conditioning, batch_variants, _raw_positive_prompt, seed, clip,
encode_cache, source_image, prompt_to_encode, global_state, learning_profile,
repair_feedback, current_family_slot, _vf_for_memory, _concept_dir,
_concept_strength, _current_final)
_encode_line = self._v2_encode_tally_status(time.perf_counter() - _run_started)
if _encode_line:
print(f"[FunPackStudio] {_encode_line}")
return (
self._v2_finalize_conditioning(output_conditioning, refinement_key, value_guidance, steer_mode, absolute_strength, spread_cap=_guess_spread_cap, temporal_style=temporal_style, temporal_fallback_text=prompt_to_encode, scene_refinement_keys=scene_refinement_keys, learning_profile=learning_profile, conditioning_plan=_conditioning_plan, clip=clip, encode_cache=encode_cache, phrase_memory=global_state.get("phrase_memory"), axis_feedback=repair_feedback, h3_phrase_emphasis=h3_phrase_emphasis, auto_strength=self._v2_auto_strength(global_state), variables=_prompt_variables, link_texts=_link_texts),
status + enhancement_status,
training_info,
loss_graph,
self._v2_encoded_prompts_output(prompt_to_encode, advisor_diagnostic=advisor_diagnostic, pre_advisor_prompt=pre_advisor_prompt, advisor_suggested=advisor_suggested),
patched_model,
video_latent,
)
def _v2_pack_guess_ladder(self, output_conditioning, n, direction, guess_range, seed,
freeze_seed=True):
"""Interactive Guessing: build N variants that scale the conditioning's spread (sigma) on
video channels by a LINEAR ramp. Variant 0 = base (factor 1.0); the last = 1.0 +/-
guess_range (up amplifies, down dampens). Each variant records 'funpack_guess_factor' so
the rating ladder is interpretable and Phase-3 learning can read each rung's strength.
freeze_seed=True (default): every rung shares the same noise, so the ONLY variable is
conditioning strength (cleanest for learning the safe ceiling). False: each rung gets a
distinct seed (different compositions too), like a regular batch."""
n = max(2, int(n))
guess_range = abs(float(guess_range or 1.0))
up = str(direction or "up").lower() != "down"
packed = []
for i in range(n):
t = i / (n - 1)
factor = (1.0 + guess_range * t) if up else (1.0 - min(0.95, guess_range) * t)
for entry in output_conditioning:
cond = entry[0]
meta = entry[1] if len(entry) > 1 and isinstance(entry[1], dict) else {}
m = dict(meta)
m["funpack_batch_variant"] = i
m["funpack_guess_factor"] = round(float(factor), 4)
packed.append((scale_conditioning_spread(cond, factor), m))
# Seed assignment. Frozen: every rung shares one base seed (only spread varies). Unfrozen:
# a distinct seed per rung (composition varies too).
rng = random.Random(int(seed) & 0xffffffffffffffff)
frozen_base = rng.randint(1, 0xffffffffffffffff)
variant_seed = {}
scene_counter = {}
for entry in packed:
m = entry[1]
vi = int(m.get("funpack_batch_variant", 0))
if freeze_seed:
vseed = frozen_base
else:
if vi not in variant_seed:
variant_seed[vi] = rng.randint(1, 0xffffffffffffffff)
vseed = variant_seed[vi]
si = scene_counter.get(vi, 0)
m["funpack_scene_seed"] = (vseed + si) & 0xffffffffffffffff
scene_counter[vi] = si + 1
print(f"[FunPackVideoRefinerV2] Interactive Guessing: packed {n} rungs "
f"(direction={'up' if up else 'down'}, range={guess_range}, "
f"seed={'frozen' if freeze_seed else 'varied'}).")
return packed
def _v2_pack_batch(self, output_conditioning, n, raw_prompt, seed, clip, encode_cache,
source_image, base_prompt, global_state, learning_profile, repair_feedback,
intent_family_slot, vf, concept_dir, concept_strength, current_final_texts,
guess_mode=False, guess_direction="up", guess_range=1.0, guess_freeze_seed=True):
"""Pack N Batch-Training variant entries onto the FINAL conditioning, each tagged
'funpack_batch_variant' so the Chain Sampler samples one chain per variant. Single-scene
output -> re-resolve shortcuts per variant (different option picks). Multi-scene output
(transition split) -> duplicate the whole scene-set per variant (seed-only; the sampler
varies the seed). Groups stay intact because each variant tags ALL its scene entries with
the same index, so the sampler chains a variant's scenes and keeps variants independent."""
n = max(1, int(n or 1))
if n <= 1 or not output_conditioning:
return output_conditioning
# Interactive Guessing: a conditioning-spread ladder (seed frozen), not shortcut/seed variants.
if guess_mode:
return self._v2_pack_guess_ladder(output_conditioning, n, guess_direction, guess_range, seed,
freeze_seed=guess_freeze_seed)
# Build the N variant entries. Single-scene output -> _v2_build_batch_variants (re-resolves
# shortcuts per variant). Multi-scene (transition split) -> duplicate the whole scene-set
# per variant. Each variant tags ALL its scene entries with the same index.
if len(output_conditioning) == 1:
packed = self._v2_build_batch_variants(
raw_prompt, n, seed, clip, encode_cache, source_image,
base_entry=output_conditioning[0], base_prompt=base_prompt,
global_state=global_state, learning_profile=learning_profile,
repair_feedback=repair_feedback, intent_family_slot=intent_family_slot,
vf=vf, concept_dir=concept_dir, concept_strength=concept_strength,
current_final_texts=current_final_texts)
else:
packed = []
for i in range(n):
for entry in output_conditioning:
cond = entry[0]
meta = entry[1] if len(entry) > 1 and isinstance(entry[1], dict) else {}
m = dict(meta)
m["funpack_batch_variant"] = i
packed.append((cond, m))
# CRITICAL: each scene entry carries a 'funpack_scene_seed' the Chain Sampler honors OVER
# its own seed. Variants inherit the base entry's scene seed, so without this every variant
# uses identical noise (the single-scene bug seen in the wild). Assign a fresh, well-spread
# seed PER VARIANT (seeded by the gen seed -> reproducible), with a per-scene offset, so the
# N chains have genuinely different noise. Applies to BOTH single- and multi-scene paths.
rng = random.Random(int(seed) & 0xffffffffffffffff)
variant_seed = {}
scene_idx = {}
for entry in packed:
m = entry[1]
if not isinstance(m, dict):
continue
vi = int(m.get("funpack_batch_variant", 0))
if vi not in variant_seed:
variant_seed[vi] = rng.randint(1, 0xffffffffffffffff)
scene_idx[vi] = 0
m["funpack_scene_seed"] = (variant_seed[vi] + scene_idx[vi]) & 0xffffffffffffffff
scene_idx[vi] += 1
print(f"[FunPackVideoRefinerV2] Batch Training: packed {n} variant(s) over "
f"{len(output_conditioning)} scene(s) = {len(packed)} entries.")
return packed
def _v2_build_batch_variants(self, raw_prompt, n, seed, clip, encode_cache, source_image,
base_entry, base_prompt, global_state, learning_profile,
repair_feedback, intent_family_slot, vf, concept_dir,
concept_strength, current_final_texts):
"""Build N conditioning entries for Batch Training, each tagged 'funpack_batch_variant'.
Variant 0 is the fully-processed base; variants 1..N-1 re-resolve shortcuts with seed+i
(random option per entry) and get the same conditioning-memory treatment. When the prompt
has no shortcut variability, every variant resolves to the same text and reuses the base
cond — so the batch becomes seed-only (the Chain Sampler varies the seed per entry)."""
n = max(1, min(64, int(n or 1)))
def tagged(entry, i, text):
cond_i, meta_i = entry
m = dict(meta_i) if isinstance(meta_i, dict) else {}
m["funpack_batch_variant"] = i
if text:
m["funpack_scene_text"] = str(text)
return (cond_i, m)
try:
try:
from .templates import revolver_enabled as _revolver_enabled
except ImportError:
from templates import revolver_enabled as _revolver_enabled
revolver_on = bool(_revolver_enabled())
except Exception:
revolver_on = False
out = [tagged(base_entry, 0, base_prompt)]
if revolver_on:
# Shortcut revolver: each variant draw below COMMITS and advances the shared
# cycle, so N variants sweep N consecutive chambers (no repeats until the pool
# exhausts). Re-expanding the base here would both burn a chamber and collide
# with variant 1's draw — skip seeding the dedupe cache with it.
base_expansion = str(raw_prompt or "")
cache = {}
else:
try:
base_expansion = self._v2_apply_global_shortcuts(str(raw_prompt or ""), seed=seed)[0]
except Exception:
base_expansion = str(raw_prompt or "")
cache = {base_expansion: base_entry}
for i in range(1, n):
try:
exp_i = self._v2_apply_global_shortcuts(str(raw_prompt or ""), seed=seed + i,
revolver_commit=True)[0]
except Exception:
exp_i = base_expansion
entry = cache.get(exp_i)
if entry is None:
cond_i, meta_i, _, _ = self._v2_conditioning_source(
clip, exp_i, None, encode_cache=encode_cache, reference_image=source_image,
h3_references=_h3_references)
if not isinstance(cond_i, torch.Tensor):
entry = base_entry # encode failed -> fall back to the base conditioning
else:
try:
refined_i, _ = self._v2_apply_conditioning_memory(
cond_i, global_state, learning_profile, repair_feedback,
intent_family_slot=intent_family_slot, vf=vf,
concept_delta_dir=concept_dir, concept_delta_strength=concept_strength,
current_final_texts=current_final_texts)
except Exception:
refined_i = cond_i
entry = (refined_i, meta_i)
cache[exp_i] = entry
out.append(tagged(entry, i, exp_i))
print(f"[FunPackVideoRefinerV2] Batch Training: packed {len(out)} variant conditioning(s) "
f"({len(cache)} distinct prompt expansion(s)).")
return out
def _v2_ascend_conditioning(self, conditioning_list, refinement_key, apply=True, conditioning_plan=None):
"""Gradient-ascend each conditioning tensor toward higher predicted reward.
No-op if disabled (apply=False), or if the value function is missing or not yet
ready (< MIN_SAMPLES ratings). The VF still trains in the background when this is
disabled — only the application of guidance is skipped.
conditioning_plan (from _v2_path_conditioning_plan) routes the stochastic search:
"lock" skips it (hold a liked region's conditioning), "explore" widens it (push a
disliked region elsewhere), otherwise the standard search runs."""
plan = conditioning_plan if isinstance(conditioning_plan, str) else None
if not apply or not conditioning_list or not refinement_key:
return conditioning_list
try:
try:
from .value_function import OnlineValueFunction
except ImportError:
from value_function import OnlineValueFunction
vf_path = refinement_state_path(refinement_key, "value_fn", prefix="refine_v2", extension="pt")
import torch as _torch
with _torch.inference_mode(False):
vf = OnlineValueFunction.load_or_create(vf_path)
if vf is None or not vf.is_ready():
return conditioning_list
ascended = []
for entry in conditioning_list:
tensor, extra = entry[0], entry[1]
# Gradient ascent first (exploit known gradient direction)
c = vf.ascend(tensor)
# Monte Carlo search around ascended point. The path planner routes its breadth:
# lock = hold the liked conditioning (no search), explore = wider search.
if plan == "lock":
pass
elif plan == "explore":
c = vf.search(c, n=32, eps_scale=2.0)
else:
c = vf.search(c)
c = protect_audio_channels(c, tensor) # keep audio on the original conditioning
ascended.append((c, extra))
print(f"[FunPackRefiner] Conditioning ascent+search applied "
f"({vf.n_trained} samples, {len(ascended)} scene(s), plan={plan or 'normal'})")
return ascended
except Exception as e:
print(f"[FunPackRefiner] Conditioning ascent failed: {e}")
return conditioning_list
def _v2_apply_scene_refinement_keys(self, conditioning_list, scene_refinement_keys,
default_key, value_guidance=True, learning_profile=None,
conditioning_plan=None):
"""Relative-mode output steering with per-scene multi-key support.
For each scene entry, look up the non-default keys whose shortcuts fired in it:
- 0 keys -> steer with the project default key (today's _v2_ascend_conditioning).
- >=1 key -> steer a clone with EACH key's learned state (per-prompt conditioning
memory + that key's value-function ascend), then AVERAGE the per-key
results into one merged conditioning (one key == mean of one == itself).
This REPLACES the default key for that scene; every key that appeared in
the scene shaped its final conditioning.
With no scene keys at all this is exactly _v2_ascend_conditioning(default_key)."""
if not conditioning_list:
return conditioning_list
has_keys = any(scene_refinement_keys or [])
if not has_keys:
return self._v2_ascend_conditioning(conditioning_list, default_key, apply=value_guidance,
conditioning_plan=conditioning_plan)
profile = learning_profile or dict(V2_RATING_PROFILES["Initial discovery"], label="Initial discovery")
key_state_cache = {}
def _key_global(key):
if key not in key_state_cache:
try:
state, _ = self._v2_load_state(key)
key_state_cache[key] = state.get("global", {}) if isinstance(state, dict) else {}
except Exception as error:
print(f"[FunPackStudio] Scene key '{key}' state load failed: {error}")
key_state_cache[key] = {}
return key_state_cache[key]
out = []
merged_scenes = []
for entry in conditioning_list:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[0], torch.Tensor)):
out.append(entry)
continue
cond, meta = entry[0], entry[1]
scene_index = meta.get("funpack_scene_index") if isinstance(meta, dict) else None
keys = set()
if isinstance(scene_index, int) and 0 <= scene_index < len(scene_refinement_keys or []):
keys = set(scene_refinement_keys[scene_index] or set())
if not keys:
# default-key path for this scene (per-entry so it matches no-key behaviour)
out.append(self._v2_ascend_conditioning([entry], default_key, apply=value_guidance,
conditioning_plan=conditioning_plan)[0])
continue
# Each key contributes only its learned DIRECTIONS here (per-prompt
# conditioning memory). The value-function ascend+search runs ONCE on the
# merged result below — not per key — because per-key search injected
# independent noise and a poorly-fit fresh key's VF could ascend toward a
# bad optimum, which the averaging then baked in (a body-horror path).
steered = []
for key in sorted(keys):
try:
c, _ = self._v2_apply_conditioning_memory(cond.clone(), _key_global(key), profile)
steered.append(c if isinstance(c, torch.Tensor) else cond.clone())
except Exception as error:
print(f"[FunPackStudio] Scene key '{key}' steering failed: {error}")
if not steered:
out.append(entry)
continue
if len(steered) == 1:
merged = steered[0]
else:
merged = torch.stack(steered, dim=0).mean(dim=0)
# Averaging per-key tensors that point different ways shrinks the
# per-token magnitude (and collapses it toward zero when keys
# disagree), leaving the merge far from the scene encode at the wrong
# magnitude, which free-runs into warping on simple prompts. Re-impose
# the single-key discipline: cap the delta vs the raw encode, then
# renormalize to its per-token norm (same tail as
# _v2_apply_conditioning_memory_impl). Capping BEFORE renorm is what
# keeps a near-zero mean from being amplified into noise.
delta = merged - cond
original_norm = cond.norm(dim=-1, keepdim=True).clamp_min(1e-8)
delta_norm = delta.norm(dim=-1, keepdim=True).clamp_min(1e-8)
scale = torch.minimum(torch.ones_like(delta_norm), (original_norm * 0.08) / delta_norm)
merged = cond + delta * scale
merged = torch.clamp(merged, min=-60.0, max=60.0)
merged = merged / merged.norm(dim=-1, keepdim=True).clamp_min(1e-8) * original_norm
# Single VF ascend+search on the merged conditioning, using the always-on
# project default key (the most-trained, most stable VF) rather than any
# one scene key.
try:
merged = self._v2_ascend_conditioning([(merged, meta)], default_key, apply=value_guidance,
conditioning_plan=conditioning_plan)[0][0]
except Exception as error:
print(f"[FunPackStudio] Merged scene ascend failed: {error}")
merged = protect_audio_channels(merged, cond)
out.append((merged, meta))
merged_scenes.append((scene_index, sorted(keys)))
if merged_scenes:
desc = "; ".join(f"scene {i + 1 if isinstance(i, int) else '?'}={'+'.join(ks)}"
for i, ks in merged_scenes)
print(f"[FunPackStudio] Scene refinement-key merge applied ({desc}).")
return out
# ====================== ABSOLUTE STEERING (prompt-agnostic) ======================
# Relative steering (everything above) keys every learned direction to one prompt's
# state file. Absolute steering accumulates the SAME pooled, channel-space directions
# into a single keyless file so they describe a global "taste" that applies to any
# prompt. It is the learned, automated analogue of FunPackConditioningAdjust: instead
# of a user typing a phrase + strength, the direction is the running average of what
# was rated liked (vs. the global source mean), applied as a broadcast shift.
def _v2_absolute_state_path(self):
return refinement_state_path(FUNPACK_ABSOLUTE_KEY, "clip", prefix=self.V2_STATE_PREFIX)
def _v2_absolute_vf_path(self):
return refinement_state_path(FUNPACK_ABSOLUTE_KEY, "value_fn", prefix=self.V2_STATE_PREFIX, extension="pt")
def _v2_load_absolute_global(self):
"""Load the keyless Absolute store and return (full_state, global_dict)."""
path = self._v2_absolute_state_path()
state = None
if os.path.exists(path):
try:
with open(path, "r", encoding="utf-8") as f:
state = json.load(f)
except (json.JSONDecodeError, OSError, ValueError):
state = None
if not isinstance(state, dict):
state = {"version": 2, "global": {}}
global_state = state.setdefault("global", {})
global_state.setdefault("axis_conditioning_memory", {})
return state, global_state
def _v2_save_absolute_global(self, state):
path = self._v2_absolute_state_path()
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(state, f)
except OSError as e:
print(f"[FunPackRefiner] Absolute store save failed: {e}")
def _v2_reward_admissible(self, profile):
"""Should this rating's reward reach the value function(s)? Wrong-* labels are
skip_value_function=True by default (a "good gen, wrong axis" mismatch carries no
quality signal on its own — see the comment on V2_RATING_PROFILES). But the loved
modifier ("Wrong action|loved" etc.) is the user explicitly overriding that: they're
saying the QUALITY was still great despite the mismatch, which is a genuine positive
quality signal, not noise. normalize_refiner_v2_rating already reflects this in the
reward NUMBER (max(reward, 0.85) when loved_modifier is set) — this just controls
whether that (now-correct) number is allowed to reach training at all."""
return not profile.get("skip_value_function") or bool(profile.get("loved_modifier"))
def _v2_train_value_function(self, vf_path, payload, reward):
"""Add one (conditioning, reward) sample to the value function at vf_path. Shared by
the per-key (relative) and keyless (absolute) reward assets. Returns n_trained or None."""
if not isinstance(payload, dict):
return None
try:
import torch as _torch
try:
from .value_function import OnlineValueFunction
except ImportError:
from value_function import OnlineValueFunction
with _torch.inference_mode(False), _torch.enable_grad():
cond_tensor = serializable_to_tensor(payload).clone()
vf = OnlineValueFunction.load_or_create(vf_path, hidden_dim=cond_tensor.shape[-1])
if vf is not None:
vf.train_on(cond_tensor, float(reward))
vf.save(vf_path)
return vf.n_trained
except Exception as e:
print(f"[FunPackRefiner] Value function training failed: {e}")
return None
def _v2_train_output_value_function(self, refinement_key, reward):
"""Sibling of _v2_train_value_function for output_guidance: pairs the same reward with
the x0_hat snapshot the Chain Sampler wrote at the END of the run this rating scores
(refinement_key.x0_snapshot.pt — see FunPackLTXAVSceneChainSampler._save_output_value_snapshot),
not with any conditioning payload. No-op if the sampler never wrote a snapshot (e.g. no
refinement_key_input was wired on the sampler, or no generation has completed yet)."""
if not refinement_key:
return None
try:
import os as _os
import torch as _torch
try:
from .value_function import LatentValueFunction
except ImportError:
from value_function import LatentValueFunction
snap_path = refinement_state_path(refinement_key, "x0_snapshot", prefix="refine_v2", extension="pt")
if not _os.path.exists(snap_path):
return None
vf_path = refinement_state_path(refinement_key, "value_fn_x0", prefix="refine_v2", extension="pt")
with _torch.inference_mode(False), _torch.enable_grad():
snapshot = _torch.load(snap_path, map_location="cpu", weights_only=False)
vf = LatentValueFunction.load_or_create(vf_path, hidden_dim=LatentValueFunction.DEFAULT_HIDDEN_DIM)
if vf is not None:
vf.train_on(snapshot, float(reward))
vf.save(vf_path)
return vf.n_trained
except Exception as e:
print(f"[FunPackRefiner] Output value function training failed: {e}")
return None
def _v2_learn_absolute(self, previous_run, learning_profile):
"""Fold the just-rated generation into the prompt-agnostic Absolute store.
Rating semantics differ from Relative here. Relative asks "does this match the
prompt?" and attributes credit per missing/satisfied axis. Absolute asks "do I like
these details IN GENERAL, regardless of the prompt?" — so we deliberately IGNORE the
prompt-specific axis attribution and steer purely on the overall reward:
* a high rating ("Perfect" = I love this and want more of this in general) pushes
the global liked direction toward this generation;
* a low rating ("missing details I like in general") pushes the disliked direction.
Pooled across every prompt, this converges on a single global-taste prior. Also trains
the keyless value function. Runs independent of refinement_key — Absolute is keyless."""
if not isinstance(previous_run, dict) or learning_profile.get("skip_learning"):
return
# Wrong-* repair ratings carry no quality signal — Absolute is purely reward-driven
# ("do I like this in general"), so feeding a repair reward here would store a good gen as
# global bad-taste and train the keyless VF on a mislabel. Repair belongs to the relative
# per-key direction/category memory, not the global taste prior. Exception: loved_modifier
# ("Wrong action|loved") is the user overriding that assumption — see _v2_reward_admissible.
if not self._v2_reward_admissible(learning_profile):
return
payload = previous_run.get("conditioning")
if not isinstance(payload, dict):
return
reward = float(learning_profile.get("reward", 0.0))
try:
state, global_state = self._v2_load_absolute_global()
self._v2_update_session_mean(global_state, previous_run.get("source_conditioning"))
session_mean = global_state.get("session_source_mean")
if reward >= 0.3:
# "more of this, generally"
self._v2_store_direction(global_state.setdefault("liked_dir", {}), payload, session_mean)
elif reward <= 0.0:
# "this lacks the details I like, generally"
self._v2_store_direction(global_state.setdefault("bad_dir", {}), payload, session_mean)
# the mild-positive band (0, 0.3) is too ambiguous to move either pole
global_state["total_iterations"] = int(global_state.get("total_iterations", 0)) + 1
self._v2_save_absolute_global(state)
except Exception as e:
print(f"[FunPackRefiner] Absolute learning failed: {e}")
n = self._v2_train_value_function(self._v2_absolute_vf_path(), payload, reward)
if n is not None:
print(f"[FunPackRefiner] Absolute value function updated — {n} samples")
def _v2_apply_absolute(self, conditioning_list, strength, value_blend=True):
"""Pull every conditioning entry toward the global learned taste direction, regardless
of prompt. Two layered engines: (1) the pooled liked direction (minus a global bad-dir
repel), applied as a calibrated broadcast shift via _v2_apply_direction; (2) the keyless
value-function ascent on top for the nonlinear component. No-op until each engine has
enough data (liked/bad need direction_count>=3; VF needs MIN_SAMPLES)."""
if not conditioning_list or strength <= 0:
return conditioning_list
try:
_, global_state = self._v2_load_absolute_global()
except Exception as _e:
_log.failed("FunPackStudio", "Absolute steer store", _e,
"Absolute steer mode is INERT for this run")
return conditioning_list
liked = global_state.get("liked_dir", {})
bad = global_state.get("bad_dir", {})
liked_ready = int(liked.get("direction_count", 0)) >= 3
bad_ready = int(bad.get("direction_count", 0)) >= 3
gvf = None
if value_blend:
try:
import torch as _torch
try:
from .value_function import OnlineValueFunction
except ImportError:
from value_function import OnlineValueFunction
vf_path = self._v2_absolute_vf_path()
if os.path.exists(vf_path):
with _torch.inference_mode(False):
candidate = OnlineValueFunction.load(vf_path)
gvf = candidate if candidate.is_ready() else None
except Exception:
gvf = None
if not (liked_ready or bad_ready or gvf is not None):
return conditioning_list
out = []
applied = False
for entry in conditioning_list:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[0], torch.Tensor)):
out.append(entry)
continue
mixed, extra = entry[0], entry[1]
if liked_ready:
mixed = self._v2_apply_direction(mixed, liked, strength)
if bad_ready:
mixed = self._v2_apply_direction(mixed, bad, strength * 0.5, negate=True)
if gvf is not None:
try:
mixed = gvf.ascend(mixed)
except Exception:
pass
mixed = protect_audio_channels(mixed, entry[0]) # audio rides the original conditioning
out.append((mixed, extra))
applied = True
if applied:
# Report the audio split every run (the one-time dim log is easy to miss). If
# vsplit == D the conditioning has no recognised audio half → audio NOT protected.
try:
D = int(out[0][0].shape[-1]) if isinstance(out[0][0], torch.Tensor) else None
except Exception:
D = None
vsplit = ltxav_video_channels(D) if D is not None else None
if D is None:
audio_note = "audio split: n/a"
elif vsplit < D:
audio_note = f"audio PROTECTED (video[:{vsplit}] of {D})"
else:
audio_note = f"audio NOT split (dim={D} unrecognised — steering all channels)"
print(
f"[FunPackRefiner] Absolute steer applied (strength={strength}, "
f"liked={liked_ready}, bad={bad_ready}, value_fn={'on' if gvf is not None else 'off'}; {audio_note})"
)
return out
def _v2_apply_pulse_temporal(self, conditioning_list, temporal_style):
"""temporal_style="pulse": tag each scene entry so the Scene Chain Sampler installs
a segmented ease-down temporal wrapper per scene."""
if str(temporal_style or "natural").strip().lower() != "pulse":
return conditioning_list
out = []
for entry in conditioning_list or []:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict)):
out.append(entry)
continue
cond, meta = entry[0], dict(entry[1])
meta["funpack_temporal_mode"] = "pulse"
meta["funpack_temporal_label"] = "pulse"
out.append([cond, meta] if isinstance(entry, list) else (cond, meta))
return out
def _v2_apply_rapid_temporal(self, conditioning_list, temporal_style):
"""temporal_style in (rapid_start, rapid_end, rapid_start_end): tag each scene
entry so the Scene Chain Sampler installs a rapid-edge ease temporal wrapper
per scene (mirrors _v2_apply_pulse_temporal)."""
style = str(temporal_style or "natural").strip().lower()
if style not in ("rapid_start", "rapid_end", "rapid_start_end"):
return conditioning_list
out = []
for entry in conditioning_list or []:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict)):
out.append(entry)
continue
cond, meta = entry[0], dict(entry[1])
meta["funpack_temporal_mode"] = style
meta["funpack_temporal_label"] = style
out.append([cond, meta] if isinstance(entry, list) else (cond, meta))
return out
def _v2_apply_auto_temporal(self, conditioning_list, temporal_style, fallback_text=""):
"""temporal_style="auto": bake a per-scene frame_rate multiplier into each
conditioning entry's metadata so the Scene Chain Sampler can apply per-scene motion
energy (the chain sampler reads funpack_temporal_*). The heuristic director classifies
each scene's own prompt; entries without scene text fall back to the run prompt.
No-op for any other style (concrete styles use the global build_enhancements path)."""
if str(temporal_style or "natural").strip().lower() != "auto":
return conditioning_list
try:
try:
from .ltx_enhancements import classify_temporal_intent
except ImportError:
from ltx_enhancements import classify_temporal_intent
except Exception as _e:
_log.failed("FunPackStudio", "temporal style classifier", _e,
"every scene renders at the natural temporal style")
return conditioning_list
out = []
for entry in conditioning_list or []:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict)):
out.append(entry)
continue
cond, meta = entry[0], dict(entry[1])
text = str(meta.get("funpack_scene_text") or "").strip() or str(fallback_text or "")
intent = classify_temporal_intent(text)
meta["funpack_temporal_mult"] = intent["mult"]
meta["funpack_temporal_label"] = intent["label"]
if intent["loop"]:
meta["funpack_temporal_loop"] = True
# Preserve the original container type (ComfyUI conditioning entries are lists
# and some downstream code mutates them in place).
out.append([cond, meta] if isinstance(entry, list) else (cond, meta))
return out
def _v2_bounded_attention_split_encode(self, text, clip, encode_cache=None):
"""EXPERIMENTAL, Structured-Diffusion-Guidance-style: split a scene prompt into two
halves by sentence count (subject 1's description vs subject 2's) and encode each half
SEPARATELY through the real text encoder, then concatenate the two resulting conditioning
tensors end to end. Unlike tokenizing each half just to COUNT tokens (the original,
approximate version of this), this gives an EXACT boundary — span 1's tokens and span 2's
tokens are never part of the same tokenization run, so there's no BPE/SentencePiece merge
ambiguity at the seam, and no possibility of cross-span leakage even before any masking
is applied. Returns (combined_cond, n1) or (None, None) if the prompt can't be split
(single sentence) or either half fails to encode."""
text = str(text or "").strip()
if not text or clip is None:
return None, None
sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()]
if len(sentences) < 2:
return None, None
mid = (len(sentences) + 1) // 2
span1 = " ".join(sentences[:mid])
span2 = " ".join(sentences[mid:])
cond1, _, _ = self._v2_encode_prompt(clip, span1, encode_cache=encode_cache,
purpose="bounded attention")
cond2, _, _ = self._v2_encode_prompt(clip, span2, encode_cache=encode_cache,
purpose="bounded attention")
if not isinstance(cond1, torch.Tensor) or not isinstance(cond2, torch.Tensor):
return None, None
try:
cond2 = cond2.to(device=cond1.device, dtype=cond1.dtype)
combined = torch.cat([cond1, cond2], dim=1)
except Exception as _e:
_log.failed("FunPackStudio", "Bounded Attention split encode", _e,
"the prompt is encoded as one span — subject bleed is not masked")
return None, None
return combined, int(cond1.shape[1])
def _v2_apply_bounded_attention(self, conditioning_list, clip, fallback_text="", encode_cache=None):
"""Always-on, cheap text-only step: for scenes with 2+ sentences, replace the scene's
conditioning with the exact-boundary concatenated encoding above and tag the split point
(funpack_bound_split_tokens) for the EXPERIMENTAL Bounded Attention sampler toggle to
read. No behavior change unless that sampler toggle is enabled — the original single-
sentence-or-fewer encoding (and metadata) passes through untouched either way."""
out = []
for entry in conditioning_list or []:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict)):
out.append(entry)
continue
cond, meta = entry[0], dict(entry[1])
if meta.get("funpack_conditioning_owner") == "wired":
# This entry came in on a wire. The split below RE-ENCODES the text and
# replaces the tensor with the result — which has no reference image in it,
# because this node was never given one. On a reference run that swaps the
# whole conditioning for a text-only encode.
_log.note_on_change(
"studio:ba_vs_wired", "FunPackStudio",
"Bounded Attention's exact-boundary split is SKIPPED for the wired "
"positive CONDITIONING — it works by re-encoding the prompt, and this "
"node cannot put back a reference it never received.")
out.append(entry)
continue
text = str(meta.get("funpack_scene_text") or "").strip() or str(fallback_text or "")
combined, n1 = self._v2_bounded_attention_split_encode(text, clip, encode_cache=encode_cache)
if combined is not None:
cond = combined
meta["funpack_bound_split_tokens"] = n1
out.append([cond, meta] if isinstance(entry, list) else (cond, meta))
return out
def _v2_apply_h3_token_weights(self, conditioning_list, clip, phrase_memory=None,
axis_feedback=None, fallback_text="", enabled=False,
auto_strength=None, variables=None, link_texts=None):
"""Tag each scene with the token spans the RATING says deserve more attention.
Studio has decomposed and scored every phrase, word and n-gram since 2.0
(_v2_concept_units_for_run, _v2_update_phrase_memory). Until now the only consumer
was the attn2 K/V patch, which H3 skips outright — one packed self-attention stream,
no cross-attention — so on H3 that learning ran every generation and landed nowhere.
This writes it as (start, end, weight) token spans for the sampler to turn into an
attention-logit bias. Text-only and cheap: phrases are located with the tokenizer's
offset mapping, not by re-encoding each one through a 32B text encoder.
OFF by default. It changes what the model sees on every rated run, and an
unvalidated feature that does that should be something you turn on to try, not
something you discover. Also gated on having anything to say: an H3 CLIP, a phrase
memory, and a rating that named a missing axis.
"""
if not enabled:
return conditioning_list
if clip is None or not isinstance(phrase_memory, dict) or not phrase_memory:
return conditioning_list
try:
try: # matches how this file reaches its siblings
from . import minimax_h3 as _h3
from . import h3_token_weights as _tw
except ImportError:
import minimax_h3 as _h3
import h3_token_weights as _tw
if not _h3.is_h3_clip(clip):
return conditioning_list
tokenizer = _tw.h3_tokenizer(clip)
if tokenizer is None:
return conditioning_list
feedback = axis_feedback or {}
units = [{"text": text, "kind": entry.get("kind"),
"effective_category_scores": entry.get("effective_category_scores")}
for text, entry in phrase_memory.items() if isinstance(entry, dict)]
weighted = _tw.weights_from_memory(
units, phrase_memory,
missing_axes=feedback.get("missing_axes", ()),
wrong_axes=feedback.get("wrong_axes", ()),
# Studio's own evidence ramp decides how hard, not this module. One Awful is
# one data point and must not boost a phrase to the ceiling.
strength=_tw.strength_from_auto(auto_strength))
if not weighted:
return conditioning_list
except Exception as _e: # noqa: BLE001
_log.failed("FunPackStudio", "H3 token weighting", _e,
"the rating's per-phrase emphasis is NOT being applied")
return conditioning_list
def _resolve_phrase_variables(pairs):
"""Phrase memory stores the RAW prompt text, so a phrase can still carry `$style`
while the encoded prompt has it resolved — and then it matches nothing.
Variables are a plain substitution, so both sides can be put in the same form.
Shortcuts are NOT expanded here. A `/trigger` rolls per seed, and peeking would
produce a different roll from the one that was encoded — a phrase carrying an
unrolled shortcut failing to match is the honest outcome."""
if not variables:
return list(pairs)
try:
try:
from .templates import resolve_variables as _rv
except ImportError:
from templates import resolve_variables as _rv
except Exception: # noqa: BLE001
return list(pairs)
done = []
for phrase, weight in pairs:
try:
done.append((_rv(str(phrase), variables)[0], weight))
except Exception: # noqa: BLE001
done.append((phrase, weight))
return done
# Resolved ONCE, up front: the matching below needs it, and so does the status line.
# Reporting the raw text made it look like `$style` was reaching the encoder.
weighted = _resolve_phrase_variables(weighted)
out = []
applied = 0
landed: dict = {}
for entry in conditioning_list or []:
if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict)):
out.append(entry)
continue
cond, meta = entry[0], dict(entry[1])
# funpack_encode_text is the string that was actually ENCODED — shortcuts rolled
# and $variables resolved. funpack_scene_text is the RAW one, and tokenizing that
# measures a different string from the one inside the conditioning: every span
# lands on the wrong words, or the placement guard rejects the run outright.
# Candidates for "the string this tensor was encoded from", best first. On a
# wired conditioning that string came from the editor's expansion for linked
# nodes, not from Studio's own copy — which is why this does not simply trust
# one. `choose_encoded_text` CHECKS each against the tensor's own text-tag run
# and takes the one that fits, so the placement is verified rather than assumed.
candidates = [meta.get("funpack_encode_text"), meta.get("funpack_scene_text"),
fallback_text]
if isinstance(link_texts, dict):
candidates += [link_texts.get(k) for k in ("full_prompt", "prompt")]
try:
from . import h3_token_weights as _tw
except ImportError:
import h3_token_weights as _tw
# Capped AFTER ranking by weight, so the eight that survive are the eight the
# rating cared most about; ordered longest-first only for application, so a
# phrase's bias lands before one of its own words adds to it.
cond_len = int(cond.shape[1]) if hasattr(cond, "shape") and cond.dim() >= 2 else 0
text, _n, verified_base = _tw.choose_encoded_text(
tokenizer, candidates, meta.get("minimax_token_tags"), cond_len)
if text is None:
# No candidate matches the text the tensor holds, so nothing here knows where
# the prompt sits. Biasing by arithmetic would land on the wrong words.
text = str(next((c for c in candidates if str(c or "").strip()), ""))
verified_base = None
# Every learned phrase that is IN this prompt, with no cap. Phrase memory
# accumulates everything the session has rated, including phrases from prompts
# since edited away — those match nothing and are dropped here rather than
# counted, which is also why they used to appear in the status line.
#
# There is no limit because there is no cost to lift: locating a phrase is a
# substring find and a scan of the tokenizer's offset map. The 8 this inherited
# belongs to `_v2_find_phrase_token_ranges`, which finds a phrase by RE-ENCODING
# it through the text encoder — eight of those is eight Qwen3-VL-32B forward
# passes, and capping them is the whole point there.
lowered = text.lower()
present = [pair for pair in weighted if pair[0] and pair[0] in lowered]
chosen = _tw.order_for_application(present)
spans, prompt_tokens = _tw.locate(tokenizer, text, chosen)
if spans and prompt_tokens:
entry_meta = {"spans": spans, "prompt_tokens": int(prompt_tokens)}
# Where the prompt SITS is read from the modality tags when they are here,
# rather than assumed to be the tail of the conditioning.
base = verified_base if verified_base is not None else _tw.prompt_base(
meta.get("minimax_token_tags"), cond_len, int(prompt_tokens))
if base is None and meta.get("funpack_conditioning_owner") == "wired":
# Only reachable when NO candidate matched the tensor's text run.
# The fallback places the prompt by arithmetic — cond_len minus the
# tokens measured HERE. That only holds when Studio encoded the text. On
# a wired conditioning another node did, from a string this one never
# saw, so the arithmetic would bias a window of the wrong words with no
# sign of it. The tags are the only trustworthy placement, and without
# them the honest result is no emphasis.
_log.note_on_change(
"h3:emphasis_placement", "FunPackStudio",
"phrase emphasis is SKIPPED for the wired positive CONDITIONING: "
"none of the prompt texts this node holds tokenizes to the number of "
"text positions the tensor actually has, so the string it was encoded "
"from is not among them and the bias cannot be placed.")
out.append([cond, meta] if isinstance(entry, list) else (cond, meta))
continue
if base is not None:
entry_meta["base"] = int(base)
meta["funpack_h3_token_weights"] = entry_meta
applied += 1
for _t, _w in chosen:
if _t in lowered:
landed[_t] = _w
out.append([cond, meta] if isinstance(entry, list) else (cond, meta))
if applied:
# Names what was APPLIED. Reporting the candidate list meant phrases that are no
# longer in the prompt were named as weighted when they had matched nothing.
top = sorted(landed.items(), key=lambda p: -p[1])[:3]
_log.note_on_change(
"h3:token_weights", "FunPackStudio",
f"rating-derived phrase emphasis: {len(landed)} of {len(weighted)} learned "
f"phrase(s) are in this prompt and were weighted across {applied} scene(s)"
+ (" (e.g. " + ", ".join(f"{t!r} x{w:.2f}" for t, w in top) + ")" if top else "")
+ " — applied as an attention bias on H3's packed stream, where the attn2 "
"K/V emphasis has nothing to hook.")
return out
def _v2_finalize_conditioning(self, conditioning_list, refinement_key, value_guidance,
steer_mode, absolute_strength, spread_cap=None,
temporal_style="natural", temporal_fallback_text="",
scene_refinement_keys=None, learning_profile=None,
conditioning_plan=None, clip=None, encode_cache=None,
phrase_memory=None, axis_feedback=None,
h3_phrase_emphasis=False, auto_strength=None,
variables=None, link_texts=None):
"""Single output hook for both steering modes. Relative = per-key VF ascend (current
behaviour). Absolute = global taste pull. Both = layer them. Finally, if Interactive
Guessing has learned a safe-spread ceiling, clamp the output conditioning's video-channel
spread to it (the backstop half of the dual auto-cap).
scene_refinement_keys (when any scene has non-default keys) REPLACES the project-key
relative steering for those scenes with the merged per-key steering (see
_v2_apply_scene_refinement_keys); scenes without bound keys keep the project key."""
mode = str(steer_mode or "relative").lower()
def _lengths(entries):
got = []
for e in entries or []:
try:
got.append(int(e[0].shape[1]))
except Exception: # noqa: BLE001
got.append(None)
return got
def _step(name, entries, fn):
"""Run one stage and name it if it changed how many positions a conditioning has.
No stage here is allowed to. One did — Bounded Attention, which rebuilds the
tensor by re-encoding — and the only downstream symptom was a tag-count mismatch
three functions away that read like housekeeping.
"""
before = _lengths(entries)
result = fn(entries)
after = _lengths(result)
if before != after:
_log.note_on_change(
f"studio:len:{name}", "FunPackStudio",
f"'{name}' changed the conditioning length: {before} -> {after}. Part of "
f"the prompt is no longer in the tensor the sampler receives. This is a "
f"bug in that step, not a setting.")
return result
out = _step("pulse temporal", conditioning_list,
lambda c: self._v2_apply_pulse_temporal(c, temporal_style))
out = _step("rapid temporal", out, lambda c: self._v2_apply_rapid_temporal(c, temporal_style))
out = _step("auto temporal", out,
lambda c: self._v2_apply_auto_temporal(c, temporal_style,
fallback_text=temporal_fallback_text))
out = _step("bounded attention", out,
lambda c: self._v2_apply_bounded_attention(
c, clip, fallback_text=temporal_fallback_text, encode_cache=encode_cache))
out = self._v2_apply_h3_token_weights(out, clip, phrase_memory=phrase_memory,
axis_feedback=axis_feedback,
fallback_text=temporal_fallback_text,
enabled=h3_phrase_emphasis,
auto_strength=auto_strength,
variables=variables,
link_texts=link_texts)
if mode in ("relative", "both"):
out = _step("relative steering", out, lambda c: self._v2_apply_scene_refinement_keys(
c, scene_refinement_keys, refinement_key,
value_guidance=value_guidance, learning_profile=learning_profile,
conditioning_plan=conditioning_plan,
))
if mode in ("absolute", "both"):
out = _step("absolute steering", out,
lambda c: self._v2_apply_absolute(c, float(absolute_strength)))
if spread_cap and spread_cap > 0:
capped = []
clamped_any = False
for entry in out:
if isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[0], torch.Tensor):
sp = conditioning_spread(entry[0])
if sp and sp > spread_cap:
entry = (scale_conditioning_spread(entry[0], spread_cap / sp), entry[1])
clamped_any = True
capped.append(entry)
out = capped
if clamped_any:
print(f"[FunPackRefiner] Interactive-Guessing cap: output conditioning spread "
f"clamped to learned-safe {spread_cap:.3f}.")
# LAST, after every manipulation above: a reference image's encoded rows go back
# exactly as Qwen produced them.
out = self._v2_restore_reference_rows(
out, clip=clip,
candidates=[temporal_fallback_text]
+ [(link_texts or {}).get(k) for k in ("full_prompt", "prompt")])
# Nothing in this node is allowed to change how many positions a conditioning has.
# If one does, the prompt has been cut and the only visible symptom downstream is a
# tag-count mismatch, which reads like housekeeping.
for before, after in zip(conditioning_list or [], out):
try:
a = int(before[0].shape[1])
b = int(after[0].shape[1])
except Exception: # noqa: BLE001
continue
if a != b:
_log.note_on_change(
"studio:length_changed", "FunPackStudio",
f"the conditioning changed length inside Studio: {a} -> {b} positions. "
f"Part of the prompt is no longer in the tensor the sampler receives. "
f"This is a bug, not a setting.")
break
return out
class FunPackSaveRefinementLatent:
CATEGORY = "FunPack/Refinement"
RETURN_TYPES = ("LATENT", "STRING")
RETURN_NAMES = ("latent", "status")
OUTPUT_NODE = True
FUNCTION = "save_latent"
DESCRIPTION = (
"Saves a video latent under a refinement key so Studio and Refiner V2 can compute "
"a spatial creativity mask on the next run. Connect your KSampler latent output here. "
"Audio and combined AV latents are skipped automatically - video only."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"latent": ("LATENT",),
"refinement_key": ("STRING", {"default": "my_style_v1", "multiline": False}),
}
}
@classmethod
def IS_CHANGED(cls, **kwargs):
return float("nan")
def save_latent(self, latent, refinement_key):
samples = latent_samples(latent)
if samples is None:
return (clone_latent(latent), "No latent samples tensor found.")
try:
try:
from .ltx_enhancements import save_creativity_latent
except ImportError:
from ltx_enhancements import save_creativity_latent
save_creativity_latent(latent, refinement_key)
except Exception as e:
print(f"[FunPackSaveRefinementLatent] Save failed: {e}")
return (clone_latent(latent), f"Save failed: {e}")
return (clone_latent(latent), f"Saved video latent for key '{refinement_key}' shape={tuple(samples.shape)}")
class FunPackPromptCombiner:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"main_prompt": ("STRING", {
"multiline": True,
"default": "masterpiece, best quality, detailed background"
}),
"delimiter": ("STRING", {
"multiline": False,
"default": ",",
}),
},
"optional": {
"prompt1": ("STRING", {"default": "", "multiline": True}),
"prompt2": ("STRING", {"default": "", "multiline": True}),
"prompt3": ("STRING", {"default": "", "multiline": True}),
"prompt4": ("STRING", {"default": "", "multiline": True}),
"prompt5": ("STRING", {"default": "", "multiline": True}),
}
}
RETURN_TYPES = ("STRING", "STRING", "STRING", "STRING", "STRING", "STRING")
RETURN_NAMES = ("out1", "out2", "out3", "out4", "out5", "random")
FUNCTION = "combine"
CATEGORY = "FunPack"
OUTPUT_NODE = False
def combine(self, main_prompt, delimiter=",",
prompt1="", prompt2="", prompt3="", prompt4="", prompt5=""):
main = main_prompt.strip()
def merge(base, delim, addon):
addon = addon.strip()
if not addon:
return base
if not base:
return addon
return f"{base}{delim}{addon}"
results = []
for p in (prompt1, prompt2, prompt3, prompt4, prompt5):
combined = merge(main, delimiter, p)
results.append(combined)
random_choice = random.choice(results)
return (*results, random_choice)
class FunPackLorebookEnhancer:
"""
Injects context from SillyTavern-style lorebook JSON files.
Always appends activated entries to the END of the prompt.
Supports multiple lorebooks, constants, selective filtering, probability, etc.
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"prompt": ("STRING", {
"multiline": True,
"default": "The brave explorer enters an ancient forest temple at twilight, discovering glowing runes on the walls."
}),
},
"optional": {
"lorebook_1": ("STRING", {"default": "", "multiline": False}),
"lorebook_2": ("STRING", {"default": "", "multiline": False}),
"lorebook_3": ("STRING", {"default": "", "multiline": False}),
"lorebook_4": ("STRING", {"default": "", "multiline": False}),
"entry_delimiter": ("STRING", {"default": "", "multiline": True}),
"context_history": ("STRING", {"multiline": True, "default": ""}),
"scan_depth": ("INT", {
"default": 4,
"min": 1,
"max": 12,
"step": 1
}),
}
}
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("enhanced_prompt", "injected_content")
FUNCTION = "enhance"
CATEGORY = "FunPack"
OUTPUT_NODE = True
def _match_keys(self, keys, text):
if not keys:
return False
if isinstance(keys, str):
keys = [k.strip() for k in keys.split(",") if k.strip()]
text = text.lower()
for key in keys:
key = key.strip()
if not key:
continue
if key.startswith("/") and key.endswith("/"):
try:
pattern = re.compile(key[1:-1], re.IGNORECASE)
if pattern.search(text):
return True
except:
continue
elif key.lower() in text:
return True
return False
def _match_secondary(self, secs, text, logic):
if not secs:
return True
if isinstance(secs, str):
secs = [s.strip() for s in secs.split(",") if s.strip()]
matches = [self._match_keys([s], text) for s in secs]
if logic == 0: return any(matches)
elif logic == 1: return all(matches)
elif logic == 2: return not any(matches)
elif logic == 3: return not all(matches)
return True
def _process_lorebook(self, path, scan_text, activated):
if not path or not os.path.exists(path):
print(f"[Lorebook Enhancer] File not found: {path}")
return activated
try:
with open(path, 'r', encoding='utf-8') as f:
lorebook = json.load(f)
entries = lorebook.get("entries", [])
if isinstance(entries, dict):
entries = list(entries.values())
for entry in entries:
if not entry.get("enabled", True):
continue
is_constant = entry.get("constant", False)
is_constant = entry.get("constant", False)
if not is_constant:
keys = entry.get("keys", entry.get("key", [])) # Fixed: use "keys" (plural) first
if not self._match_keys(keys, scan_text):
continue
if entry.get("selective", False):
sec_keys = entry.get("keysecondary", []) or entry.get("secondary_keys", [])
logic = entry.get("selectiveLogic", 0)
if not self._match_secondary(sec_keys, scan_text, logic):
continue
if entry.get("selective", False):
sec_keys = entry.get("keysecondary", []) or entry.get("secondary_keys", [])
logic = entry.get("selectiveLogic", 0)
if not self._match_secondary(sec_keys, scan_text, logic):
continue
prob = entry.get("extensions", {}).get("probability", 100)
if random.randint(1, 100) > prob:
continue
activated.append(entry)
except json.JSONDecodeError as e:
print(f"[Lorebook Enhancer] JSON decode error in {path}: {str(e)}")
except Exception as e:
print(f"[Lorebook Enhancer] Failed to process {path}: {type(e).__name__}: {str(e)}")
return activated
def enhance(self, prompt,
lorebook_1="", lorebook_2="", lorebook_3="", lorebook_4="",
context_history="", scan_depth=4, entry_delimiter=""):
full_text = (context_history + "\n" + prompt).lower()
lines = full_text.splitlines()
scan_text = "\n".join(lines[-scan_depth:]) if scan_depth > 0 else full_text
activated = []
for path in [lorebook_1, lorebook_2, lorebook_3, lorebook_4]:
if path.strip():
activated = self._process_lorebook(path.strip(), scan_text, activated)
if not activated:
return (prompt, "No lorebook entries were triggered.")
activated.sort(key=lambda e: e.get("insertion_order", e.get("order", 0)))
injected = []
enhanced = prompt.strip() # clean up any trailing space
for entry in activated:
content = entry.get("content", "").strip()
if not content:
continue
prefixed_content = f"{entry_delimiter}{content}" if entry_delimiter else content
# Always append to the end - this is the requested behavior
if enhanced:
enhanced += "\n" + prefixed_content
else:
enhanced = prefixed_content
source = entry.get("comment") or entry.get("name") or f"uid:{entry.get('uid','?')}" or "unnamed"
injected.append(f"[{source}] {prefixed_content}")
injected_text = "\n\n".join(injected) if injected else "No content injected"
return (enhanced, injected_text)
class _FunPackAdvisorLLMWrapper:
"""Wraps a HuggingFace CausalLM to expose the tokenize/generate/decode
interface expected by FunPack Refiner V2's advisor."""
def __init__(self, model, tokenizer, device):
self._model = model
self._tokenizer = tokenizer
self._device = device
def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True):
return self._tokenizer.apply_chat_template(
messages, tokenize=tokenize, add_generation_prompt=add_generation_prompt
)
def tokenize(self, text, system_prompt=None, skip_template=False, **kwargs):
thinking = bool(kwargs.get("thinking", True))
if skip_template:
# Text already has role tokens baked in (e.g. from TextGenerateLTX2Prompt).
result = self._tokenizer.encode(
str(text), return_tensors="pt", add_special_tokens=False
)
input_ids = (result.input_ids if hasattr(result, "input_ids") else result).to(self._device)
else:
if system_prompt:
messages = [
{"role": "system", "content": str(system_prompt)},
{"role": "user", "content": str(text)},
]
else:
messages = [{"role": "user", "content": str(text)}]
try:
result = self._tokenizer.apply_chat_template(
messages, tokenize=True, add_generation_prompt=True, return_tensors="pt",
enable_thinking=thinking,
)
except TypeError:
result = self._tokenizer.apply_chat_template(
messages, tokenize=True, add_generation_prompt=True, return_tensors="pt"
)
# apply_chat_template may return a BatchEncoding or a plain tensor depending
# on the transformers version and tokenizer. Extract input_ids from either.
input_ids = result.input_ids if hasattr(result, "input_ids") else result
input_ids = input_ids.to(self._device)
return {"input_ids": input_ids, "prompt_length": int(input_ids.shape[1]), "thinking": thinking}
def generate(self, tokens, do_sample=True, max_length=800, temperature=0.7,
top_k=50, top_p=0.92, min_p=0.0, repetition_penalty=1.3,
no_repeat_ngram_size=5, presence_penalty=0.0, seed=None, **kwargs):
input_ids = tokens["input_ids"]
prompt_length = tokens.get("prompt_length", int(input_ids.shape[1]))
thinking = tokens.get("thinking", False)
if seed:
torch.manual_seed(int(seed))
# When thinking mode is active, add extra budget for the reasoning chain so the
# actual response is not crowded out by thinking tokens.
effective_max = int(max_length) + 2048 if thinking else int(max_length)
# Explicit attention mask prevents the "pad token == eos token" warning and
# stops the model from attending to padding positions, which causes it to echo
# the input, drop spaces, or otherwise produce garbled output.
attention_mask = torch.ones_like(input_ids)
required = dict(
do_sample=do_sample,
max_new_tokens=effective_max,
temperature=float(temperature),
top_k=int(top_k),
top_p=float(top_p),
repetition_penalty=float(repetition_penalty),
pad_token_id=self._tokenizer.pad_token_id or self._tokenizer.eos_token_id,
attention_mask=attention_mask,
)
# Optional params not supported by all models - try progressively stripping
optional = {}
if no_repeat_ngram_size:
optional["no_repeat_ngram_size"] = int(no_repeat_ngram_size)
if min_p > 0:
optional["min_p"] = float(min_p)
if presence_penalty != 0:
optional["presence_penalty"] = float(presence_penalty)
strips = [[], ["presence_penalty"], ["presence_penalty", "min_p"], list(optional.keys())]
output = None
for strip in strips:
gen_kwargs = {**required, **{k: v for k, v in optional.items() if k not in strip}}
try:
with torch.no_grad():
output = self._model.generate(input_ids, **gen_kwargs)
break
except TypeError:
continue
if output is None:
with torch.no_grad():
output = self._model.generate(input_ids, **required)
return {"output_ids": output, "prompt_length": prompt_length}
def decode(self, generated, skip_special_tokens=True):
output_ids = generated["output_ids"]
prompt_length = generated.get("prompt_length", 0)
new_tokens = output_ids[0][prompt_length:]
# Decode with special tokens preserved so thinking delimiters are visible
# for stripping. Models like Qwen3 encode <think>/</ think> as special token
# IDs - with skip_special_tokens=True they vanish silently and the raw
# reasoning content bleeds into the output as if it were the response.
text = self._tokenizer.decode(new_tokens, skip_special_tokens=False)
# Strip complete thinking blocks first, then any truncated one (no closing tag).
text = re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
text = re.sub(r"<think>[\s\S]*", "", text, flags=re.IGNORECASE)
# Strip remaining special tokens: <|im_end|>, <|endoftext|>, etc.
text = re.sub(r"<\|[^|]*\|>", "", text)
return text.strip()
_FUNPACK_ADVISOR_LLM_CACHE = {}
class FunPackAdvisorLLM:
CATEGORY = "FunPack/Refinement"
RETURN_TYPES = ("CLIP",)
RETURN_NAMES = ("advisor_clip",)
FUNCTION = "load_advisor"
DESCRIPTION = "Loads any HuggingFace CausalLM for use as FunPack Refiner V2 advisor. Connect the output to advisor_clip."
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model_path": ("STRING", {
"default": "huihui-ai/Huihui-Qwen3-8B-abliterated-v2",
"multiline": False,
"tooltip": "HuggingFace repo ID (e.g. huihui-ai/Huihui-Qwen3-8B-abliterated-v2) or absolute local path to a model directory.",
}),
"dtype": (["bfloat16", "float16", "float32"], {
"default": "bfloat16",
"tooltip": "Model precision. bfloat16 recommended for CUDA; float32 for CPU.",
}),
},
}
def load_advisor(self, model_path, dtype):
model_path = str(model_path).strip()
cache_key = (model_path, dtype)
if cache_key in _FUNPACK_ADVISOR_LLM_CACHE:
print(f"[FunPackAdvisorLLM] Using cached model: {model_path}")
return (_FUNPACK_ADVISOR_LLM_CACHE[cache_key],)
device = "cuda" if torch.cuda.is_available() else "cpu"
dtype_map = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}
torch_dtype = dtype_map.get(dtype, torch.bfloat16)
print(f"[FunPackAdvisorLLM] Loading {model_path} as {dtype} on {device} ...")
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
model_path, trust_remote_code=True, torch_dtype=torch_dtype,
).eval().to(device).requires_grad_(False)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
tokenizer.pad_token_id = tokenizer.eos_token_id
wrapper = _FunPackAdvisorLLMWrapper(model, tokenizer, device)
_FUNPACK_ADVISOR_LLM_CACHE[cache_key] = wrapper
print(f"[FunPackAdvisorLLM] Ready.")
return (wrapper,)
class FunPackPromptEnhancer:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"user_prompt": ("STRING", {"multiline": True, "default": "A photo of a [subject] in a [setting]. [action]."}),
"system_prompt": ("STRING", {
"multiline": True,
"default": "<You are a creative AI assistant tasked with describing videos.\n\nDescribe the video by detailing the following aspects:\n1. The main content and theme of the video.\n2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.\n4. background environment, light, style and atmosphere.\n5. camera angles, movements, and transitions used in the video:"
}),
"model_path_type": (["Local Safetensors", "HuggingFace Pretrained"],),
"model_path": ("STRING", {"multiline": False, "default": "mlabonne/NeuralLlama-3-8B-Instruct-abliterated"}),
"llm_safetensors_file": (folder_paths.get_filename_list('clip'),),
"top_p": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.05, "default": 0.75}),
"top_k": ("INT", {"min": 0, "max": 1000, "step": 1, "default": 40}),
"temperature": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.01, "default": 0.6}),
"max_new_tokens": ("INT", {"min": 64, "max": 4096, "step": 64, "default": 512}),
"repetition_penalty": ("FLOAT", {"min": 0.0, "max": 3.0, "step": 0.01, "default": 1.0}),
}
}
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("enhanced_prompt",)
FUNCTION = "enhance_prompt"
CATEGORY = "FunPack"
def enhance_prompt(self, user_prompt, system_prompt, model_path_type, model_path, llm_safetensors_file, top_p, top_k, temperature, max_new_tokens, repetition_penalty):
llm_model = None
llm_tokenizer = None
llm_model_device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"[FunPackPromptEnhancer] Starting prompt enhancement...")
try:
if model_path_type == "HuggingFace Pretrained":
print(f"[FunPackPromptEnhancer] Loading LLM from HuggingFace pretrained: {model_path}")
llm_tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
llm_model = AutoModelForCausalLM.from_pretrained(model_path, ignore_mismatched_sizes=True, trust_remote_code=True)
elif model_path_type == "Local Safetensors":
print(f"[FunPackPromptEnhancer] Loading LLM from local safetensors file: {llm_safetensors_file}")
full_safetensors_path = folder_paths.get_full_path('clip', llm_safetensors_file)
llm_tokenizer = AutoTokenizer.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", trust_remote_code=True)
config = AutoConfig.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", trust_remote_code=True)
model_base = AutoModelForCausalLM.from_config(config, trust_remote_code=True)
state_dict = load_file(full_safetensors_path, device="cpu")
model_base.load_state_dict(state_dict, strict=False)
llm_model = model_base
llm_model = llm_model.eval().to(torch.bfloat16 if llm_model_device == "cuda" else torch.float32).to(llm_model_device).requires_grad_(False)
print(f"[FunPackPromptEnhancer] LLM model loaded successfully to {llm_model_device}!")
# Model detection to apply correct chat template
llm_tokenizer.chat_template = """{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' %}{% if loop.first %}{{ '<|begin_of_text|>' + content }}{% else %}{{ content }}{% endif %}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' if add_generation_prompt else '' }}"""
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
]
if llm_tokenizer.pad_token_id is None:
llm_tokenizer.pad_token = llm_tokenizer.eos_token
llm_tokenizer.pad_token_id = llm_tokenizer.eos_token_id
llm_tokens = llm_tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
return_tensors="pt",
tokenize=True
).to(llm_model_device)
print("[FunPackPromptEnhancer] Generating enhanced prompt...")
with torch.no_grad():
generated_ids = llm_model.generate(
**llm_tokens,
do_sample=True,
top_p=top_p,
top_k=top_k,
temperature=temperature,
max_new_tokens=max_new_tokens,
repetition_penalty=repetition_penalty,
pad_token_id=llm_tokenizer.pad_token_id
)
output_text = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True)
print(f"[FunPackPromptEnhancer] Enhanced prompt generated: {output_text}")
return (output_text,)
except Exception as e:
print(f"[FunPackPromptEnhancer] Error during prompt enhancement: {e}")
raise
finally:
if llm_model is not None:
del llm_model
llm_model = None
if llm_tokenizer is not None:
del llm_tokenizer
llm_tokenizer = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
gc.collect()
print("[FunPackPromptEnhancer] LLM model and tokenizer unloaded and memory cleared.")
class FunPackStoryWriter:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"user_prompt": ("STRING", {"multiline": True, "default": "A photo of a [subject] in a [setting]. [action]."}),
"prompt1": ("STRING", {"multiline": False, "default": ""}),
"prompt2": ("STRING", {"multiline": False, "default": ""}),
"prompt3": ("STRING", {"multiline": False, "default": ""}),
"prompt4": ("STRING", {"multiline": False, "default": ""}),
"prompt5": ("STRING", {"multiline": False, "default": ""}),
"story_system_prompt": ("STRING", {
"multiline": True,
"default": ""
}),
"sequence_system_prompt": ("STRING", {
"multiline": True,
"default": ""
}),
"model_path_type": (["Local Safetensors", "HuggingFace Pretrained"],),
"model_path": ("STRING", {"multiline": False, "default": "mlabonne/NeuralLlama-3-8B-Instruct-abliterated"}),
"llm_safetensors_file": (folder_paths.get_filename_list('clip'),),
"prompt_count": ("INT", {"min": 1, "max": 5, "step": 1, "default": 3}),
"top_p": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.05, "default": 0.75}),
"top_k": ("INT", {"min": 0, "max": 1000, "step": 1, "default": 40}),
"min_p": ("FLOAT", {"min": 0.0, "max": 1.0, "step": 0.01, "default": 0.1}),
"temperature": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.01, "default": 0.6}),
"max_new_tokens": ("INT", {"min": 64, "max": 4096, "step": 64, "default": 512}),
"repetition_penalty": ("FLOAT", {"min": 0.0, "max": 3.0, "step": 0.01, "default": 1.0}),
"mode": (["Sequences from story", "Sequences from user prompt"],),
"vision_input": ("STRING", {"multiline": True, "default": "Put outputs of your VL model here to make the Story Writer aware of the starting image."}),
"sanity_check": ("BOOLEAN", {"default": True, "label": "Enable Sanity Check"}),
"sanity_check_system_prompt": ("STRING", {
"multiline": True,
"default": "Analyze the given sequence and perform a correction, if the sequence does not match the given requirements:\n1. The sequence is related to given user's prompt.\n2. The sequence contains only physically possible actions.\n3. The sequence contains information about characters, their appearances, positioning, actions, camera angle, focus and zoom.\n4. The sequence is fully describing the requested action.\n\nOutput ONLY corrected sequence, or return it unchanged if it matches the requirements. No additional text except for sequence is allowed."
}),
"disable_continuity": ("BOOLEAN", {"default": False, "label": "Enable/disable continuity - if enabled, does not provide the history of previously generated sequences when generating new one."}),
"provide_current_id": ("BOOLEAN", {"default": True, "label": "If true, provides current sequence ID to the model even if continuity is disabled."}),
}
}
RETURN_TYPES = ("STRING","STRING","STRING","STRING","STRING",)
RETURN_NAMES = ("prompt1","prompt2","prompt3","prompt4","prompt5",)
FUNCTION = "write_story"
CATEGORY = "FunPack"
def write_story(self, user_prompt, prompt1, prompt2, prompt3, prompt4, prompt5, story_system_prompt, sequence_system_prompt, model_path_type, model_path, llm_safetensors_file, prompt_count, top_p, top_k, min_p, temperature, max_new_tokens, repetition_penalty, mode, vision_input, sanity_check, sanity_check_system_prompt, disable_continuity, provide_current_id):
llm_model = None
llm_tokenizer = None
llm_model_device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"[FunPackPromptEnhancer] Making initial story...")
try:
if model_path_type == "HuggingFace Pretrained":
print(f"[FunPackStoryWriter] Loading LLM from HuggingFace pretrained: {model_path}")
llm_tokenizer = AutoTokenizer.from_pretrained(model_path, ignore_mismatched_sizes=True, trust_remote_code=True)
llm_model = AutoModelForCausalLM.from_pretrained(model_path, ignore_mismatched_sizes=True, trust_remote_code=True)
elif model_path_type == "Local Safetensors":
print(f"[FunPackStoryWriter] Loading LLM from local safetensors file: {llm_safetensors_file}")
full_safetensors_path = folder_paths.get_full_path('clip', llm_safetensors_file)
llm_tokenizer = AutoTokenizer.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", ignore_mismatched_sizes=True, trust_remote_code=True)
config = AutoConfig.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", trust_remote_code=True)
model_base = AutoModelForCausalLM.from_config(config, trust_remote_code=True)
state_dict = load_file(full_safetensors_path, device="cpu")
model_base.load_state_dict(state_dict, strict=False)
llm_model = model_base
llm_model = llm_model.eval().to(torch.bfloat16 if llm_model_device == "cuda" else torch.float32).to(llm_model_device).requires_grad_(False)
print(f"[FunPackStoryWriter] LLM model loaded successfully to {llm_model_device}!")
# Applying correct chat template
llm_tokenizer.chat_template = """{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' %}{% if loop.first %}{{ '<|begin_of_text|>' + content }}{% else %}{{ content }}{% endif %}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' if add_generation_prompt else '' }}"""
# Inside write_story method, after model loading and template/pad fix
outputs = [""] * 5
prompts = [""] * 5
prompts[0] = prompt1
prompts[1] = prompt2
prompts[2] = prompt3
prompts[3] = prompt4
prompts[4] = prompt5
recommended_loras = None
# ── Initialize messages ONCE, depending on mode ────────────────────────────
if mode == "Sequences from story":
messages = [
{"role": "system", "content": story_system_prompt},
{"role": "user", "content": user_prompt}
]
# Generate hidden story
llm_tokens = llm_tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt", tokenize=True
).to(llm_model_device)
print("[FunPackStoryWriter] Generating hidden story...")
with torch.no_grad():
generated_ids = llm_model.generate(
**llm_tokens,
do_sample=True,
top_p=top_p,
top_k=top_k,
min_p=min_p,
temperature=temperature,
max_new_tokens=max_new_tokens,
repetition_penalty=repetition_penalty,
pad_token_id=llm_tokenizer.pad_token_id,
eos_token_id=llm_tokenizer.eos_token_id,
)
story = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True).strip()
print(f"[FunPackStoryWriter] Hidden story: {story[:150]}...")
messages.append({"role": "assistant", "content": story})
else: # "Sequences from user prompt"
messages = [
{"role": "system", "content": sequence_system_prompt},
{"role": "user", "content": user_prompt}
]
# ── Now generate sequences — only add new instruction + append output ─────
for seq_idx in range(prompt_count):
# Add **only** the fresh sequence instruction each time
if disable_continuity == True and provide_current_id == False:
messages = [
{"role": "system", "content": sequence_system_prompt},
{"role": "user", "content": user_prompt}
]
if prompts[seq_idx]:
messages.append({"role": "user", "content": f"""User's request for current scene only: {prompts[seq_idx]}"""})
elif disable_continuity == True and provide_current_id == True:
messages = [
{"role": "system", "content": sequence_system_prompt},
{"role": "user", "content": f"""Current request ID: {seq_idx+1}\nUser's instruction for all requests: {user_prompt}"""}
]
if prompts[seq_idx]:
messages.append({"role": "user", "content": f"""User's request for current scene only: {prompts[seq_idx]}"""})
else:
messages = [
{"role": "system", "content": sequence_system_prompt},
{"role": "user", "content": f"""Total amount of requests in this batch: {prompt_count}\nCurrently generating request ID {seq_idx+1} out of {prompt_count}\nRequests left in queue: {prompt_count - seq_idx - 1}\nUser's instruction for all requests: {user_prompt}"""},
{"role": "assistant", "content": f"""History:{chr(10).join([f"ID {i}: {text}" for i, text in enumerate(outputs[:seq_idx])]) if seq_idx > 0 else "No history available."}"""}
]
if prompts[seq_idx]:
messages.append({"role": "user", "content": f"""User's request for current scene only: {prompts[seq_idx]}"""})
if vision_input is not None:
messages.append({"role": "user", "content": f"""Reference image description (this is the starting image in the video batch): {vision_input}"""})
llm_tokens = llm_tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt", tokenize=True
).to(llm_model_device)
print(f"[FunPackStoryWriter] Generating ID {seq_idx + 1}/{prompt_count}...")
with torch.no_grad():
generated_ids = llm_model.generate(
**llm_tokens,
do_sample=True,
top_p=top_p,
top_k=top_k,
min_p=min_p,
temperature=temperature,
max_new_tokens=max_new_tokens,
repetition_penalty=repetition_penalty,
pad_token_id=llm_tokenizer.pad_token_id,
eos_token_id=llm_tokenizer.eos_token_id,
)
seq_text = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True).strip()
# Append generated sequence — this is what chains everything
messages.append({"role": "assistant", "content": seq_text})
if sanity_check == False:
print(f"[FunPackStoryWriter] ID {seq_idx + 1} (sanity check skipped): {seq_text}...")
# Performing sanity check - comparing sequence text to user's prompt according to rules in sanity check system prompt.
if sanity_check == True:
if mode == "Sequences from story" and disable_continuity == False:
sanity_messages = [
{"role": "system", "content": sanity_check_system_prompt},
{"role": "user", "content": f"""Original story: {story}\n
Rules: {story_system_prompt}\n
User's instruction: {user_prompt}\n
Previous response: {outputs[seq_idx] if seq_idx > 0 else "No history available."}\n
Response to validate and correct if rules were broken: {seq_text}"""}
]
elif mode == "Sequences from user prompt" and disable_continuity == False:
sanity_messages = [
{"role": "system", "content": sanity_check_system_prompt},
{"role": "user", "content": f"""User's instruction: {user_prompt}\n
Rules: {sequence_system_prompt}\n
Previous response: {outputs[seq_idx] if seq_idx > 0 else "No history available."}\n
Response to validate and correct if rules were broken: {seq_text}"""}
]
else:
sanity_messages = [
{"role": "system", "content": sanity_check_system_prompt},
{"role": "user", "content": f"""User's instruction: {user_prompt}
Rules: {sequence_system_prompt}
Response to validate and correct if rules were broken: {seq_text}"""}
]
llm_tokens = llm_tokenizer.apply_chat_template(
sanity_messages, add_generation_prompt=True, return_tensors="pt", tokenize=True
).to(llm_model_device)
print(f"[FunPackStoryWriter] Performing sanity check on ID {seq_idx + 1}/{prompt_count}...")
with torch.no_grad():
generated_ids = llm_model.generate(
**llm_tokens,
do_sample=False,
top_p=top_p,
top_k=top_k,
min_p=min_p,
temperature=temperature,
max_new_tokens=1024,
repetition_penalty=1.05,
pad_token_id=llm_tokenizer.pad_token_id,
eos_token_id=llm_tokenizer.eos_token_id,
)
seq_text = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True).strip()
print(f"[FunPackStoryWriter] ID {seq_idx + 1} (sanity check performed): {seq_text}...")
outputs[seq_idx] = seq_text
return tuple(outputs)
except Exception as e:
print(f"[FunPackStoryWriter] Error during prompt enhancement: {e}")
raise
finally:
if llm_model is not None:
del llm_model
llm_model = None
if llm_tokenizer is not None:
del llm_tokenizer
llm_tokenizer = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
gc.collect()
print("[FunPackStoryWriter] LLM model and tokenizer unloaded and memory cleared.")
class FunPackConditioningAdjust:
CATEGORY = "FunPack/Refinement"
RETURN_TYPES = ("CONDITIONING", "STRING")
RETURN_NAMES = ("conditioning", "status")
FUNCTION = "adjust"
DESCRIPTION = (
"Manually adjust conditioning by blending specific phrase directions at user-set strengths. "
"Insert between Refiner V2 and the sampler. Use the popup editor to manage phrase/strength pairs."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"conditioning": ("CONDITIONING",),
"clip": ("CLIP",),
"adjustments": ("STRING", {
"default": "[]",
"multiline": False,
"tooltip": "JSON list managed by the node editor. Do not edit manually.",
}),
},
"optional": {
"refinement_key_input": ("STRING", {
"forceInput": True,
"tooltip": "Optional refinement key from FunPack Refinement Key Loader. When connected, the popup shows learned phrases from that session as a clickable bank.",
}),
},
}
def _cond_mean_vector(self, cond_tensor):
if not isinstance(cond_tensor, torch.Tensor) or cond_tensor.dim() < 2:
return None
vec = cond_tensor.detach().float()
if vec.dim() == 3:
return vec.mean(dim=(0, 1))
return vec.mean(dim=0)
def _encode_mean_vector(self, clip, phrase):
try:
tokens = clip.tokenize(phrase)
encoded = clip.encode_from_tokens_scheduled(tokens)
if not encoded:
return None
cond_tensor, _ = encoded[0]
return self._cond_mean_vector(cond_tensor)
except Exception:
return None
def adjust(self, conditioning, clip, adjustments, refinement_key_input=""):
import json as _json
try:
items = _json.loads(str(adjustments or "[]"))
except Exception:
items = []
active = [
item for item in (items if isinstance(items, list) else [])
if isinstance(item, dict)
and str(item.get("phrase", "") or "").strip()
and abs(float(item.get("strength", 0.0) or 0.0)) > 1e-6
]
if not active:
return (conditioning, "Conditioning adjust: no active adjustments.")
if not conditioning or not isinstance(conditioning, list):
return (conditioning, "Conditioning adjust: invalid conditioning input.")
log = []
new_conditioning = []
for cond_item in conditioning:
if not (isinstance(cond_item, (list, tuple)) and len(cond_item) == 2):
new_conditioning.append(cond_item)
continue
cond_tensor, meta = cond_item
if not isinstance(cond_tensor, torch.Tensor):
new_conditioning.append(cond_item)
continue
base_vec = self._cond_mean_vector(cond_tensor)
if base_vec is None:
new_conditioning.append(cond_item)
continue
# Accumulate direction adjustments then apply once as a uniform shift
# to all sequence positions, keeping the original tensor shape intact.
total_delta = torch.zeros_like(base_vec)
for item in active:
phrase = str(item.get("phrase", "")).strip()
strength = float(item.get("strength", 0.0))
phrase_vec = self._encode_mean_vector(clip, phrase)
if phrase_vec is None:
log.append(f"'{phrase}': encode failed")
continue
phrase_vec = phrase_vec.to(base_vec.device)
delta = phrase_vec - base_vec
delta_norm = delta.norm()
if delta_norm < 1e-8:
log.append(f"'{phrase}': no direction")
continue
direction = delta / delta_norm
magnitude = base_vec.norm()
total_delta = total_delta + strength * magnitude * direction
log.append(f"'{phrase}' {strength:+.2f}")
if total_delta.norm() < 1e-8:
new_conditioning.append(cond_item)
continue
# Broadcast the uniform shift across the sequence dimension
shift = total_delta.to(cond_tensor.dtype)
while shift.dim() < cond_tensor.dim():
shift = shift.unsqueeze(0)
new_tensor = protect_audio_channels(cond_tensor + shift, cond_tensor)
new_conditioning.append((new_tensor, meta))
status = "Conditioning adjust: " + (", ".join(log) if log else "nothing applied")
return (new_conditioning, status)
# A pass's schedule dropdown holds ComfyUI's scheduler names plus this one extra entry,
# which means "don't compute anything, run the sigmas typed in the Sigmas field".
# Both frontends offer it under the same string — keep them in sync if it ever changes.
KSAMPLER_USER_SIGMAS = "use_user_sigmas"
class FunPackStudio:
"""Single node combining Refinement Key, Refiner V2, Scene Builder,
Advisor LLM, LoRA management, and Conditioning Adjust under one UI."""
CATEGORY = "FunPack"
RETURN_TYPES = ("MODEL", "CONDITIONING", "CONDITIONING", "INT", "SAMPLER", "SIGMAS", "SAMPLER", "SIGMAS", "IMAGE", "STRING", "STRING", "STRING", "LATENT")
RETURN_NAMES = ("model", "modified_positive", "negative", "seed", "high_pass_sampler", "high_pass_sigmas", "low_pass_sampler", "low_pass_sigmas", "loss_graph", "status", "training_info", "encoded_prompts", "video_latent")
FUNCTION = "run"
DESCRIPTION = (
"FunPack Studio - all FunPack refinement tools in one node. "
"Rating widget stays on the node face; everything else lives in the popup editor."
)
_lora_loader = None
_apply_weights_node = None
@classmethod
def _get_lora_loader(cls):
if cls._lora_loader is None:
try:
try:
from .model_management import FunPackLoraLoader
except ImportError:
from model_management import FunPackLoraLoader
cls._lora_loader = FunPackLoraLoader()
except Exception:
cls._lora_loader = None
return cls._lora_loader
@classmethod
def _get_apply_weights_node(cls):
if cls._apply_weights_node is None:
try:
try:
from .model_management import FunPackApplyLoraWeights
except ImportError:
from model_management import FunPackApplyLoraWeights
cls._apply_weights_node = FunPackApplyLoraWeights()
except Exception:
cls._apply_weights_node = None
return cls._apply_weights_node
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"rating": (V2_RATING_LABELS, {"default": "Missing action", "label": "Rating"}),
"studio_settings": ("STRING", {"default": "{}", "multiline": False,
"tooltip": "JSON managed by the Studio popup editor. Do not edit manually."}),
"adjustments": ("STRING", {"default": "[]", "multiline": False,
"tooltip": "Conditioning adjustment phrase list. Managed by the Studio popup."}),
},
"optional": {
"model": ("MODEL",),
"clip": ("CLIP",),
"advisor_clip": ("CLIP", {"tooltip": "Pre-loaded text generator for advisor. Overrides the LLM configured inside Studio."}),
"positive_conditioning": ("CONDITIONING",),
"negative_conditioning": ("CONDITIONING", {"tooltip": "Negative conditioning passed through unchanged to the output."}),
"clip_vision_output": ("CLIP_VISION_OUTPUT",),
"source_image": ("IMAGE",),
"lora_stack": ("FUNPACK_LORA_STACK", {"tooltip": "External LoRA stack. Bypasses Studio's internal LoRA management entirely when connected."}),
"positive_prompt": ("STRING", {"multiline": True, "default": "", "forceInput": True,
"tooltip": "Positive prompt. Ignored when Scene Builder mode is not Pass-through."}),
"negative_prompt": ("STRING", {"multiline": True, "default": "", "forceInput": True,
"tooltip": "Negative prompt text. Encoded via CLIP and output as negative conditioning. Skipped when negative_conditioning is connected."}),
"user_intent_prompt": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
"feedback_prompt": ("STRING", {"multiline": True, "default": "", "forceInput": True,
"tooltip": "Feedback about the previous output. Overrides the feedback set inside Studio."}),
"refinement_key_input": ("STRING", {"forceInput": True}),
"latent": ("LATENT", {
"tooltip": "Optional video latent for creativity masking. Takes priority over any saved latent for this key. Connect your i2v or previous KSampler output here.",
}),
},
"hidden": {
"prompt": "PROMPT",
"unique_id": "UNIQUE_ID",
},
}
@classmethod
def IS_CHANGED(cls, **kwargs):
# Shortcut revolver advances persistent per-shortcut state on every run, so a
# fixed-seed re-run must not be served from ComfyUI's cache while it's enabled —
# the next Generate is supposed to draw the next replacement. Normal input-hash
# caching applies when the revolver is off.
try:
try:
from .templates import revolver_enabled
except ImportError:
from templates import revolver_enabled
if revolver_enabled():
return float("nan")
except Exception:
pass
return ""
@staticmethod
def _is_output_connected(prompt, unique_id, output_index):
return FunPackVideoRefiner()._is_output_connected(prompt, unique_id, output_index)
def run(self, rating, studio_settings, adjustments,
model=None, clip=None, advisor_clip=None,
positive_conditioning=None, negative_conditioning=None,
clip_vision_output=None, source_image=None,
lora_stack=None, positive_prompt=None, negative_prompt=None,
user_intent_prompt=None, feedback_prompt=None, refinement_key_input="",
latent=None, prompt=None, unique_id=None):
try:
settings = json.loads(str(studio_settings or "{}"))
except Exception:
settings = {}
active_prompt = str(positive_prompt or "").strip()
# --- Negative conditioning ---
# Priority: pre-encoded input > encode negative_prompt via CLIP > popup default > empty
out_negative = []
if negative_conditioning is not None:
out_negative = negative_conditioning
else:
rf_neg = str(settings.get("refiner", {}).get("negative_prompt", "") or "")
effective_negative = str(negative_prompt or "").strip() or rf_neg
if effective_negative and clip is not None:
try:
# Named, because on H3 this is a Qwen3-VL-32B forward pass whose output the
# sampler will never read: CFG is fixed at 1.0, so the negative branch is
# not evaluated. It still has to be encoded (the output is a real port, and
# negative_erase reads it), but it is a full text-encoder pass and it has to
# be visible when accounting for what a run loaded and when.
# WHICH of the two supplied it. They are different boxes in different
# places, and a stored one silently outranking an empty wired one reads
# as the field on screen having no effect.
source = ("the wired negative_prompt input"
if str(negative_prompt or "").strip()
else "Studio's SAVED setting (the wired negative_prompt input "
"is empty)")
print(f"[FunPackStudio] encoding the negative prompt with the wired CLIP "
f"({len(effective_negative)} chars) from {source}")
neg_encoded = clip.encode_from_tokens_scheduled(clip.tokenize(effective_negative))
if neg_encoded:
out_negative = neg_encoded
except Exception as e:
print(f"[FunPackStudio] Negative encode failed: {e}")
# --- Overrides: when True, popup value wins over connected input ---
ov = settings.get("overrides", {}) if isinstance(settings.get("overrides"), dict) else {}
# negative_prompt override
rf_neg_popup = str(settings.get("refiner", {}).get("negative_prompt", "") or "")
if ov.get("negative_prompt"):
negative_prompt = rf_neg_popup
# --- Refiner settings ---
rf = settings.get("refiner", {}) if isinstance(settings.get("refiner"), dict) else {}
mode = str(rf.get("mode", "Refine") or "Refine")
advisor_mode = str(rf.get("advisor_mode", "Off") or "Off")
advisor_thinking = bool(rf.get("advisor_thinking", True))
im_feeling_lucky = bool(rf.get("im_feeling_lucky", False))
reset_session = bool(rf.get("reset_session", False))
temporal_style = str(rf.get("temporal_style", "natural") or "natural").strip().lower()
split_by_transitions = bool(rf.get("split_by_transitions", False))
split_transition_placement = str(rf.get("split_transition_placement", "start") or "start").strip().lower()
if split_transition_placement not in ("start", "end", "silent"):
split_transition_placement = "start"
reference_injection = bool(rf.get("reference_injection", False))
vision_conditioning = bool(rf.get("vision_conditioning", True))
value_guidance = bool(rf.get("value_guidance", True))
# feedback_prompt: popup wins if override is on, else external wins
popup_feedback = str(rf.get("feedback_prompt", "") or "")
if ov.get("feedback_prompt"):
feedback_prompt = popup_feedback
else:
feedback_prompt = str(feedback_prompt) if feedback_prompt is not None else popup_feedback
# user_intent_prompt: popup wins if override is on, else external wins
popup_intent = str(rf.get("user_intent_prompt_override", "") or "")
if ov.get("user_intent_prompt"):
effective_intent = popup_intent
else:
effective_intent = str(user_intent_prompt or "") or popup_intent
# --- Refinement key: popup wins if override is on, else external input wins ---
popup_key = str(settings.get("refinement_key", "") or "").strip()
if ov.get("refinement_key"):
key = popup_key
refinement_key_input = "" # suppress external so refine_v2 uses widget key
else:
key = popup_key # refine_v2 uses refinement_key_input when non-empty
# --- Seed memory ---
refiner = FunPackVideoRefinerV2()
seed_output_connected = self._is_output_connected(prompt, unique_id, 3)
fresh_seed = random.randint(1, 0xffffffffffffffff)
seed, seed_source, scene_seed_memory = refiner._v2_choose_successful_seed(
refinement_key_input if str(refinement_key_input or "").strip() and not ov.get("refinement_key") else key,
active_prompt,
fresh_seed,
seed_output_connected=seed_output_connected,
reset_session=reset_session,
)
# --- Advisor LLM: external input takes precedence over popup-configured model ---
if advisor_clip is None:
llm_cfg = settings.get("advisor_llm", {})
if isinstance(llm_cfg, dict) and llm_cfg.get("enabled") and llm_cfg.get("model_path"):
try:
(advisor_clip,) = FunPackAdvisorLLM().load_advisor(
str(llm_cfg["model_path"]),
str(llm_cfg.get("dtype", "bfloat16")),
)
except Exception as e:
print(f"[FunPackStudio] Advisor LLM load failed: {e}")
# --- LoRA: ApplyWeights (session suggestions) then Load ---
# 1. Build lora_list from studio settings or use provided external stack directly.
# 2. FunPackApplyLoraWeights reads session weight suggestions for current prompt.
# 3. FunPackLoraLoader applies the stack to model/clip.
# 4. refine_v2 then applies attn2 direction patch on top of the LoRA-patched model.
active_lora_stack = lora_stack # external stack bypasses steps 1-2
studio_loras = settings.get("loras", [])
lora_cfg = settings.get("loras_config", {}) if isinstance(settings.get("loras_config"), dict) else {}
lora_mode = str(lora_cfg.get("mode", "ltx2") or "ltx2")
per_block = bool(lora_cfg.get("per_block", False))
if not active_lora_stack and studio_loras and model is not None:
valid_loras = [
e for e in (studio_loras if isinstance(studio_loras, list) else [])
if isinstance(e, dict) and str(e.get("name", "") or "").strip() not in ("", "None")
]
if valid_loras:
# Build lora_list in the format FunPackApplyLoraWeights expects
clip_weights = {str(e.get("name", "")): float(e.get("clip_weight", 1.0) or 1.0)
for e in valid_loras}
lora_list_json = json.dumps([
{
"name": str(e.get("name", "")),
"type": str(e.get("type", "general") or "general"),
"strength": float(e.get("model_weight", 1.0) or 1.0),
"on": True,
}
for e in valid_loras
])
apply_node = self._get_apply_weights_node()
if apply_node is not None:
try:
effective_key = key or "studio"
active_lora_stack, lora_apply_status = apply_node.apply_lora_weights(
active_prompt, effective_key, lora_mode,
per_block=per_block,
lora_list=lora_list_json,
)
# Inject clip_weight values (ApplyLoraWeights only handles model weights)
for entry in active_lora_stack.get("loras", []):
entry["clip_weight"] = clip_weights.get(str(entry.get("name", "")), 1.0)
print(f"[FunPackStudio] {lora_apply_status}")
except Exception as e:
print(f"[FunPackStudio] Apply LoRA weights failed: {e}")
loader = self._get_lora_loader()
if loader is not None and active_lora_stack is not None:
try:
model, clip, active_lora_stack, lora_load_status = loader.load_loras(
model, active_lora_stack, clip=clip
)
print(f"[FunPackStudio] {lora_load_status}")
except Exception as e:
print(f"[FunPackStudio] LoRA load failed: {e}")
# Velocity/rescue commit keys: each sampler pass captures trajectories under its
# RESOLVED velocity key (the custom velocity_refinement_key field, else the effective
# refinement key). The refiner must commit the rating to those SAME keys — otherwise a
# custom velocity key captures every gen but never commits (silent no-learn). Resolve
# them with the same logic as _build_one_sampler and hand them to the refiner.
_eff_rkey = (refinement_key_input
if str(refinement_key_input or "").strip() and not ov.get("refinement_key")
else key)
_vel_commit_keys = []
for _pass in ("high", "low"):
_hc = (settings.get("samplers", {}).get(_pass, {}) or {}).get("hybrid", {})
_vk = str((_hc or {}).get("velocity_refinement_key", "") or "").strip()
if not _vk or _vk == "default":
_vk = str(_eff_rkey or "").strip() or "default"
_vel_commit_keys.append(_vk)
_me_scene_ratings = rf.get("movie_editor_scene_ratings")
if not isinstance(_me_scene_ratings, list):
_me_scene_ratings = None
# --- Refiner V2 ---
cond, status, training_info, loss_graph, encoded_prompts, out_model, video_latent = refiner.refine_v2(
active_prompt,
clip=clip,
rating=rating,
refinement_key=key,
_velocity_keys=_vel_commit_keys,
batch_variants=int(rf.get("batch_variants", 1) or 1),
guess_mode=bool(rf.get("guess_mode", False)),
guess_direction=str(rf.get("guess_direction", "up") or "up"),
guess_range=float(rf.get("guess_range", 1.0) or 1.0),
guess_freeze_seed=bool(rf.get("guess_freeze_seed", True)),
reset_session=reset_session,
lora_stack=active_lora_stack,
im_feeling_lucky=im_feeling_lucky,
user_intent_prompt=effective_intent,
refinement_key_input=refinement_key_input,
positive_conditioning=positive_conditioning,
h3_phrase_emphasis=bool(rf.get("h3_phrase_emphasis", False)),
clip_vision_output=clip_vision_output,
source_image=source_image if vision_conditioning else None,
model=model,
mode=mode,
advisor_mode=advisor_mode,
advisor_thinking=advisor_thinking,
advisor_clip=advisor_clip,
feedback_prompt=feedback_prompt,
temporal_style=temporal_style,
split_by_transitions=split_by_transitions,
split_transition_placement=split_transition_placement,
reference_injection=reference_injection,
value_guidance=value_guidance,
steer_mode=str(rf.get("steer_mode", "relative") or "relative"),
absolute_strength=float(rf.get("absolute_strength", 0.6) or 0.6),
latent=latent,
seed_output_connected=seed_output_connected,
_seed=seed,
_seed_source=seed_source,
_scene_seeds=scene_seed_memory,
movie_editor_scene_ratings=_me_scene_ratings,
scene_segments=(rf.get("scenes") if isinstance(rf.get("scenes"), dict) else None),
variables=(rf.get("variables") if isinstance(rf.get("variables"), (list, dict)) else None),
link_texts=(rf.get("link_texts") if isinstance(rf.get("link_texts"), dict) else None),
h3_references=rf.get("h3_references"),
)
# --- Conditioning Adjust ---
adj_raw = str(adjustments or "[]").strip()
if adj_raw not in ("[]", "", "null") and clip is not None:
try:
adj_node = FunPackConditioningAdjust()
cond, adj_status = adj_node.adjust(cond, clip, adj_raw)
status = f"{status}\n{adj_status}"
except Exception as e:
status = f"{status}\nConditioning adjust failed: {e}"
# --- Samplers ---
# Pooled prompt signature so rescue can steer toward good trajectories
# captured for similar prompts (prompt-aware), not a prompt-blind average.
prompt_sig = None
try:
if isinstance(cond, list) and cond and isinstance(cond[0][0], torch.Tensor):
ct = cond[0][0].detach().float()
prompt_sig = ct.mean(dim=(0, 1)) if ct.dim() == 3 else ct.mean(dim=0)
except Exception as e:
print(f"[FunPackStudio] prompt signature failed: {e}")
prompt_sig = None
# Effective refinement key (same resolution as seed memory / refine_v2),
# used to default the velocity/rescue memory namespace so it follows the
# wired refinement key instead of a separate field the user must keep in sync.
effective_refinement_key = (
refinement_key_input
if str(refinement_key_input or "").strip() and not ov.get("refinement_key")
else key
)
high_sampler, high_sigmas, low_sampler, low_sigmas = self._build_samplers(
settings.get("samplers", {}), prompt_sig=prompt_sig,
refinement_key=effective_refinement_key,
# Needed for the KSampler pass's steps/schedule: calculate_sigmas reads the
# model's model_sampling, so the schedule follows whatever model is loaded.
model=out_model,
)
# Give the negative prompt a job at CFG 1. H3 never evaluates the negative branch, so
# the text you wrote there is otherwise dead weight; this takes its direction out of
# the positive conditioning instead. Last, so it operates on the conditioning the DiT
# will actually receive rather than on something later steering would move again.
if bool(rf.get("negative_erase", False)):
try:
try:
from . import negative_erase as _neg
except ImportError:
import negative_erase as _neg
cond, _neg_note = _neg.apply(
cond, out_negative,
float(rf.get("negative_erase_strength", 0.5) or 0.0),
mode=str(rf.get("negative_erase_mode", "project") or "project"),
renorm=bool(rf.get("negative_erase_renorm", True)),
)
if _neg_note:
status = f"{status}\n{_neg_note}"
print(f"[FunPackStudio] {_neg_note}")
except Exception as _e: # noqa: BLE001
status = f"{status}\nnegative_erase failed: {_e}"
print(f"[FunPackStudio] negative_erase failed ({_e}) — conditioning unchanged")
cond = _h3_reconcile_token_tags(cond, "positive")
out_negative = _h3_reconcile_token_tags(out_negative, "negative")
return (out_model, cond, out_negative, seed, high_sampler, high_sigmas, low_sampler, low_sigmas, loss_graph, status, training_info, encoded_prompts, video_latent)
@staticmethod
def _parse_sigmas(text):
try:
vals = [float(v.strip()) for v in str(text or "").replace(";", ",").split(",") if v.strip()]
if vals:
return torch.tensor(vals, dtype=torch.float32)
except Exception:
pass
return None
@classmethod
def _pass_schedule(cls, cfg, model, typed_sigmas, sampler_type):
"""The pass's SIGMAS: either the typed field or a computed schedule.
Every sampler type takes its schedule from the outside — the FunPack samplers walk
whatever sigma list they are handed exactly like a stock KSampler does — so steps +
schedule belong to the PASS, not to the KSampler branch that first grew them. The
dropdown is the switch between the two sources, deliberately: KSAMPLER_USER_SIGMAS
runs the typed Sigmas field, anything else COMPUTES (the way ComfyUI's BasicScheduler
does) and the typed list is ignored for this pass — so a hand-written schedule can be
parked in the field and switched back to without retyping it.
`ksampler_scheduler` / `ksampler_steps` are the pre-split key names, still read here
so configs saved by an older UI keep their schedule.
"""
scheduler = str(cfg.get("scheduler", cfg.get("ksampler_scheduler", KSAMPLER_USER_SIGMAS))
or KSAMPLER_USER_SIGMAS)
if scheduler == KSAMPLER_USER_SIGMAS:
return typed_sigmas
steps = int(cfg.get("steps", cfg.get("ksampler_steps", 0)) or 0)
if model is None or steps <= 0:
print(f"[FunPackStudio] {sampler_type}: schedule '{scheduler}' needs steps > 0 "
f"and a model — falling back to the typed sigmas field.")
return typed_sigmas
try:
import comfy.samplers as _cs
out = _cs.calculate_sigmas(
model.get_model_object("model_sampling"), scheduler, steps
).cpu()
except Exception as e:
# Keep the sampler: a bad scheduler name should cost the schedule, not the pass.
print(f"[FunPackStudio] {sampler_type}: schedule build failed "
f"({scheduler}, {steps} steps): {e}")
return typed_sigmas
if typed_sigmas is not None:
print(f"[FunPackStudio] {sampler_type}: schedule '{scheduler}' ({steps} steps) in "
f"use — the typed sigmas field is ignored until the schedule is set back to "
f"'{KSAMPLER_USER_SIGMAS}'.")
return out
@classmethod
def _build_one_sampler(cls, cfg, prompt_sig=None, refinement_key="", model=None):
sampler_type = str(cfg.get("type", "Hybrid Euler 2S") or "Hybrid Euler 2S")
sigmas_raw = cls._parse_sigmas(cfg.get("sigmas", ""))
# steps + schedule are pass-level and apply to every sampler type; the FunPack
# samplers then prepare (motion pulses, quality split) whatever list this produces.
sigmas_raw = cls._pass_schedule(cfg, model, sigmas_raw, sampler_type)
try:
if sampler_type == "Distilled Flow":
try:
from .samplers import FunPackDistilledFlowSampler
except ImportError:
from samplers import FunPackDistilledFlowSampler
dc = cfg.get("distilled", {}) if isinstance(cfg.get("distilled"), dict) else {}
# Velocity/rescue memory namespace: follow the wired refinement key when the
# field is blank or the "default" placeholder, so capture/rescue share a bucket
# with the Hybrid sampler instead of needing a separate field (matches hybrid).
dvkey = str(dc.get("velocity_refinement_key", "") or "").strip()
if not dvkey or dvkey == "default":
dvkey = str(refinement_key or "").strip() or "default"
node = FunPackDistilledFlowSampler()
sampler, out_sigmas = node.get_sampler(
order=int(dc.get("order", 2)),
final_correction_steps=int(dc.get("final_correction_steps", 1)),
ab2_ramp=bool(dc.get("ab2_ramp", False)),
normalize_strength=float(dc.get("normalize_strength", 0.0)),
normalize_start_sigma=float(dc.get("normalize_start_sigma", 0.9)),
s_noise=float(dc.get("s_noise", 0.0)),
velocity_bias_mode=str(dc.get("velocity_bias_mode", "off")),
velocity_bias_strength=float(dc.get("velocity_bias_strength", 0.0)),
velocity_bias_source=str(dc.get("velocity_bias_source", "mean")),
velocity_refinement_key=dvkey,
rescue_mode=bool(dc.get("rescue_mode", False)),
rescue_threshold=float(dc.get("rescue_threshold", 0.15)),
rescue_strength=float(dc.get("rescue_strength", 0.2)),
rescue_prompt_sig=prompt_sig,
alg_enabled=bool(dc.get("alg_enabled", False)),
alg_strength=float(dc.get("alg_strength", 2.5)),
alg_sigma_threshold=float(dc.get("alg_sigma_threshold", 0.94)),
mg_enabled=bool(dc.get("mg_enabled", False)),
mg_strength=float(dc.get("mg_strength", 0.5)),
mg_decay=float(dc.get("mg_decay", 0.5)),
mg_sigma_threshold=float(dc.get("mg_sigma_threshold", 0.975)),
quality_sharpness=float(dc.get("quality_sharpness", 0.0)),
sigmas=sigmas_raw,
)
elif sampler_type == "KSampler":
import comfy.samplers as _cs
sampler_name = str(cfg.get("ksampler_name", "euler") or "euler")
# A stock KSampler carries no schedule of its own — the SAMPLER object is
# just the step function — so the pass's steps + schedule (built above, and
# shared with the FunPack samplers) is the only thing that can supply one.
sampler = _cs.sampler_object(sampler_name)
# Quality sharpness for a loop we do not own. The unsharp needs only the
# current x0 prediction, the previous one and the step's sigma, so it lifts
# out of the sampler through a denoiser proxy — the same trick ALG uses.
# Wrapping keeps the sampler you chose; it does not replace it.
_sharp = float(cfg.get("ksampler_sharpness", 0.0) or 0.0)
if _sharp > 0.0:
try:
from .samplers import _sharpen_wrap_sampler
except ImportError:
from samplers import _sharpen_wrap_sampler
_wrapped = _sharpen_wrap_sampler(
sampler, _sharp,
float(cfg.get("ksampler_sharpen_start_pct", 0.35) or 0.0))
if _wrapped is not None:
sampler = _wrapped
else:
print(f"[FunPackStudio] ksampler_sharpness is set, but "
f"'{sampler_name}' exposes no sampler_function to wrap — "
f"sampling continues without the sharpening.")
out_sigmas = sigmas_raw
else: # Hybrid Euler 2S (default)
try:
from .samplers import FunPackHybridEuler2SSampler
except ImportError:
from samplers import FunPackHybridEuler2SSampler
hc = cfg.get("hybrid", {}) if isinstance(cfg.get("hybrid"), dict) else {}
# Velocity/rescue memory namespace: if left blank or at the "default"
# placeholder, follow the wired refinement key so capture and rescue
# share a bucket without the user maintaining a separate field.
vkey = str(hc.get("velocity_refinement_key", "") or "").strip()
if not vkey or vkey == "default":
vkey = str(refinement_key or "").strip() or "default"
node = FunPackHybridEuler2SSampler()
sampler, out_sigmas = node.get_sampler(
eta=float(hc.get("eta", 1.0)),
eta_final=float(hc.get("eta_final", 1.0)),
normalize_strength=float(hc.get("normalize_strength", 0.0)),
normalize_start_sigma=float(hc.get("normalize_start_sigma", 0.9)),
s_noise=float(hc.get("s_noise", 1.0)),
high_quality_pct=float(hc.get("high_quality_pct", 0.35)),
correction_blend=float(hc.get("correction_blend", 1.0)),
quality_sharpness=float(hc.get("quality_sharpness", 0.0)),
motion_pulse_mode=str(hc.get("motion_pulse_mode", "off")),
motion_pulse_start_pct=float(hc.get("motion_pulse_start_pct", 0.30)),
motion_pulse_count=int(hc.get("motion_pulse_count", 2)),
motion_pulse_spacing_pct=float(hc.get("motion_pulse_spacing_pct", 0.22)),
motion_pulse_strength=float(hc.get("motion_pulse_strength", 0.85)),
velocity_bias_mode=str(hc.get("velocity_bias_mode", "off")),
velocity_bias_strength=float(hc.get("velocity_bias_strength", 0.0)),
velocity_bias_source=str(hc.get("velocity_bias_source", "mean")),
velocity_refinement_key=vkey,
rescue_mode=bool(hc.get("rescue_mode", False)),
rescue_threshold=float(hc.get("rescue_threshold", 0.15)),
rescue_strength=float(hc.get("rescue_strength", 0.2)),
rescue_prompt_sig=prompt_sig,
sigmas=sigmas_raw,
)
if out_sigmas is None:
out_sigmas = sigmas_raw
return sampler, out_sigmas
except Exception as e:
print(f"[FunPackStudio] Sampler build failed ({sampler_type}): {e}")
return None, sigmas_raw
@classmethod
def _build_samplers(cls, samplers_cfg, prompt_sig=None, refinement_key="", model=None):
if not isinstance(samplers_cfg, dict):
samplers_cfg = {}
high_cfg = samplers_cfg.get("high", {}) if isinstance(samplers_cfg.get("high"), dict) else {}
low_cfg = samplers_cfg.get("low", {}) if isinstance(samplers_cfg.get("low"), dict) else {}
high_sampler, high_sigmas = cls._build_one_sampler(high_cfg, prompt_sig=prompt_sig, refinement_key=refinement_key, model=model)
# The low pass's SAMPLER drives the chain sampler's second pass. Off by default it
# mirrors the high pass's algorithm — the wire has always existed, and turning it into
# a live second sampler for every project that already uses second_pass would change
# what those runs do without anyone asking. `own_sampler` is the explicit opt-in; the
# low pass's SIGMAS is unaffected either way, because that was always pass 2's own.
if not low_cfg.get("own_sampler"):
low_cfg = dict(high_cfg, **{k: v for k, v in low_cfg.items()
if k in ("sigmas", "steps", "scheduler",
"ksampler_steps", "ksampler_scheduler")})
low_sampler, low_sigmas = cls._build_one_sampler(low_cfg, prompt_sig=prompt_sig, refinement_key=refinement_key, model=model)
return high_sampler, high_sigmas, low_sampler, low_sigmas
Reference in New Issue View Git Blame Copy Permalink