import base64 import gc import glob import json import math import os import random import re import time import weakref from datetime import datetime, timezone from hashlib import md5 from typing import Optional import numpy as np import torch try: from . import funpack_log as _log except ImportError: # flat import when ComfyUI loads the pack as a top-level module import funpack_log as _log import torch.nn.functional as F from PIL import Image, ImageDraw, ImageFont from safetensors.torch import load_file from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer import folder_paths V2_SEED_REUSE_CHANCE = 0.20 V2_SEED_MEMORY_MAX_CONCEPTS = 128 V2_SEED_MEMORY_MAX_PER_CONCEPT = 24 # Path-outcome memory (path-planner Phase 1). A "path" = a per-scene generation config # (concepts + family + guidance dims) EXCLUDING the seed; the seed is the random draw # WITHIN an arm. Every rating — not just successful ones — is recorded as a labeled # outcome on the arm it came from, so a later planner can route away from arms that keep # producing disliked results. Counts decay so an arm's recent behavior dominates. V2_PATH_OUTCOME_MAX_ARMS = 256 V2_PATH_OUTCOME_MAX_SEEDS_PER_ARM = 16 V2_PATH_OUTCOME_DECAY = 0.9 LORA_REFINER_TYPE_PROFILES = { "general": {"step": 0.025, "max_offset": 0.20, "min_offset": -0.35, "bad_max_offset": 0.45, "bad_min_offset": -1.35, "culprit_bias": 0.28}, "action": {"step": 0.046, "max_offset": 0.35, "min_offset": -0.45, "bad_max_offset": 0.75, "bad_min_offset": -2.10, "culprit_bias": 0.16}, "style": {"step": 0.032, "max_offset": 0.28, "min_offset": -0.38, "bad_max_offset": 0.58, "bad_min_offset": -1.55, "culprit_bias": 0.20}, "quality": {"step": 0.022, "max_offset": 0.18, "min_offset": -0.30, "bad_max_offset": 0.38, "bad_min_offset": -1.20, "culprit_bias": 0.18}, "character": {"step": 0.024, "max_offset": 0.20, "min_offset": -0.32, "bad_max_offset": 0.42, "bad_min_offset": -1.30, "culprit_bias": 0.10}, } RATING_LABELS = [ "-Just forget it-", "Perfect", "Missing details", "Missing concept", "Missing quality", "Missing details + concept", "Missing details + quality", "Missing concept + quality", "Awful", ] RATING_PROFILES = { "-Just forget it-": { "key": "forget", "level": 0, "legacy_score": 0, "legacy_range": "ignored", "reward": 0.0, "quality_signal": 0.0, "concept_signal": 0.0, "detail_signal": 0.0, "missing_axes": [], "prompt_emphasis": 0.0, "skip_learning": True, }, "Initial discovery": { "key": "discover", "level": 5, "legacy_score": 6, "legacy_range": "discovery", "reward": 0.0, "quality_signal": 0.0, "concept_signal": 0.0, "detail_signal": 0.0, "missing_axes": [], "prompt_emphasis": 0.0, }, "Perfect": { "key": "like", "level": 8, "legacy_score": 10, "legacy_range": "keep", "reward": 1.0, "quality_signal": 1.0, "concept_signal": 1.0, "detail_signal": 1.0, "missing_axes": [], "prompt_emphasis": 0.45, }, "Missing details": { "key": "missing_details", "level": 7, "legacy_score": 8, "legacy_range": "boost details", "reward": 0.45, "quality_signal": 0.85, "concept_signal": 0.70, "detail_signal": -1.0, "missing_axes": ["details"], "prompt_emphasis": 0.65, }, "Missing concept": { "key": "missing_concept", "level": 6, "legacy_score": 6, "legacy_range": "boost concept", "reward": 0.10, "quality_signal": 0.85, "concept_signal": -1.0, "detail_signal": 0.20, "missing_axes": ["concept"], "prompt_emphasis": 0.95, }, "Missing quality": { "key": "missing_quality", "level": 5, "legacy_score": 4, "legacy_range": "boost quality", "reward": -0.25, "quality_signal": -1.0, "concept_signal": 0.65, "detail_signal": 0.65, "missing_axes": ["quality"], "prompt_emphasis": 0.55, }, "Missing details + concept": { "key": "missing_details_concept", "level": 4, "legacy_score": 5, "legacy_range": "boost details+concept", "reward": -0.05, "quality_signal": 0.75, "concept_signal": -1.0, "detail_signal": -1.0, "missing_axes": ["details", "concept"], "prompt_emphasis": 1.05, }, "Missing details + quality": { "key": "missing_details_quality", "level": 3, "legacy_score": 3, "legacy_range": "boost details+quality", "reward": -0.35, "quality_signal": -1.0, "concept_signal": 0.50, "detail_signal": -1.0, "missing_axes": ["details", "quality"], "prompt_emphasis": 0.82, }, "Missing concept + quality": { "key": "missing_concept_quality", "level": 2, "legacy_score": 2, "legacy_range": "boost concept+quality", "reward": -0.55, "quality_signal": -1.0, "concept_signal": -1.0, "detail_signal": 0.15, "missing_axes": ["concept", "quality"], "prompt_emphasis": 1.02, }, "Awful": { "key": "awful", "level": 1, "legacy_score": 2, "legacy_range": "boost all", "reward": -0.85, "quality_signal": -1.0, "concept_signal": -1.0, "detail_signal": -1.0, "missing_axes": ["details", "concept", "quality"], "prompt_emphasis": 1.20, "rollback_on_failure": True, }, } RATING_ALIASES = { "I like it": "Perfect", "I don't like it": "Awful", "Missing details + concepts": "Missing details + concept", "Missing concept + details": "Missing details + concept", "Missing quality + details": "Missing details + quality", "Missing quality + concept": "Missing concept + quality", "Missing everything": "Awful", } CATEGORY_FEEDBACK_MAP = { 1: "general", 2: "concept", 3: "style", 4: "quality", 5: "character", 6: "details", } def _clamp(value, low, high): return max(low, min(high, value)) def normalize_refiner_rating(value): if isinstance(value, str): cleaned = value.strip() cleaned = RATING_ALIASES.get(cleaned, cleaned) if cleaned in RATING_PROFILES: return dict(RATING_PROFILES[cleaned], label=cleaned) try: value = int(float(cleaned)) except (TypeError, ValueError): return dict(RATING_PROFILES["Missing concept"], label="Missing concept") try: legacy_score = int(value) except (TypeError, ValueError): legacy_score = RATING_PROFILES["Missing concept"]["legacy_score"] legacy_score = int(_clamp(legacy_score, 1, 10)) if legacy_score >= 9: label = "Perfect" elif legacy_score >= 7: label = "Missing details" elif legacy_score >= 5: label = "Missing concept" elif legacy_score >= 3: label = "Missing quality" else: label = "Awful" profile = dict(RATING_PROFILES[label], label=label) profile["legacy_score"] = legacy_score return profile PROTECTED_PHRASE_RE = re.compile(r'"([^"]+)"|“([^”]+)”|\\([^\\]+)\\') def tensor_to_serializable(t: torch.Tensor) -> dict: if not isinstance(t, torch.Tensor): raise TypeError(f"Expected torch.Tensor, got {type(t)}") arr = t.detach().cpu().numpy() return { "data": base64.b64encode(arr.tobytes()).decode("utf-8"), "shape": list(arr.shape), "dtype": str(arr.dtype) } # A tensor inside the refinement key is one of three things: # {"data","shape","dtype"} an inline base64 blob — the original format, still read # {"__tensor__": Tensor} loaded from the sidecar, in memory only # {"__tensor_ref__": id} what an externalised tensor looks like ON DISK # The blob form is what makes a key slow: it is base64 inside a JSON that is fully parsed and # fully rewritten every run, and one conditioning delta is several megabytes of it. TENSOR_SIDECAR_SUFFIX = ".tensors.pt" def serializable_to_tensor(d: dict) -> torch.Tensor: held = d.get("__tensor__") if isinstance(d, dict) else None if isinstance(held, torch.Tensor): tensor = held.to(dtype=torch.float32) return tensor.cuda() if torch.cuda.is_available() else tensor arr = np.frombuffer(base64.b64decode(d["data"]), dtype=d["dtype"]).reshape(d["shape"]).copy() tensor = torch.from_numpy(arr).to(dtype=torch.float32) if torch.cuda.is_available(): tensor = tensor.cuda() return tensor def _is_tensor_blob(value): return isinstance(value, dict) and "data" in value and "shape" in value and "dtype" in value def externalize_tensors(value, store, path="state"): """Replace every tensor in a state tree with a reference, collecting them into `store`. Ids are the tensor's PATH in the tree ("state.global.liked_dir.direction"), so the JSON stays readable and diffable and a reader can see what a reference points at. Returns the tree with references in place of blobs; the caller writes `store` beside the JSON. """ if isinstance(value, dict): held = value.get("__tensor__") if isinstance(held, torch.Tensor): store[path] = held.detach().cpu() return {"__tensor_ref__": path, "shape": list(held.shape), "dtype": str(held.dtype)} if _is_tensor_blob(value): arr = np.frombuffer(base64.b64decode(value["data"]), dtype=value["dtype"]).reshape(value["shape"]).copy() store[path] = torch.from_numpy(arr) return {"__tensor_ref__": path, "shape": list(value["shape"]), "dtype": str(value["dtype"])} return {k: externalize_tensors(v, store, f"{path}.{k}") for k, v in value.items()} if isinstance(value, list): return [externalize_tensors(v, store, f"{path}[{i}]") for i, v in enumerate(value)] return value def internalize_tensors(value, store): """Put the sidecar's tensors back behind their references, in memory. A reference whose tensor is missing from the sidecar is left as-is: consumers read tensors through serializable_to_tensor, which returns nothing for it, and a missing direction is a direction not applied rather than a broken run. """ if isinstance(value, dict): ref = value.get("__tensor_ref__") if isinstance(ref, str): held = store.get(ref) return {"__tensor__": held} if isinstance(held, torch.Tensor) else value return {k: internalize_tensors(v, store) for k, v in value.items()} if isinstance(value, list): return [internalize_tensors(v, store) for v in value] return value def _safe_float(value, fallback=0.0): try: return float(value) except (TypeError, ValueError): return fallback def _safe_int(value, fallback=0): try: return int(value) except (TypeError, ValueError): return fallback def _to_image_tensor(image): arr = np.asarray(image).astype(np.float32) / 255.0 return torch.from_numpy(arr).unsqueeze(0) def render_refinement_loss_graph(refinement_key, scheduler_mode, mode, total_iterations, latest_learning_loss, points, width=960, height=540): image = Image.new("RGBA", (width, height), (0, 0, 0, 255)) draw = ImageDraw.Draw(image) font = ImageFont.load_default() bg = (24, 28, 34) panel = (30, 36, 44) grid = (64, 76, 92) axis = (122, 137, 156) text = (235, 239, 244) subtext = (168, 180, 194) line = (74, 201, 255) fill = (74, 201, 255, 72) point_color = (255, 181, 71) image.paste(bg, (0, 0, width, height)) draw.rounded_rectangle((16, 16, width - 16, height - 16), radius=18, fill=panel, outline=(49, 58, 70), width=1) draw.text((32, 28), "FunPack Refinement Loss", fill=text, font=font) draw.text((32, 50), f"Embedding: {refinement_key}", fill=text, font=font) draw.text((32, 68), f"Scheduler: {scheduler_mode.upper()} Mode: {mode.upper()} Iterations: {total_iterations} Latest loss: {latest_learning_loss:.4f}", fill=subtext, font=font) if not points: draw.text((32, 112), "No loss history is available yet.", fill=(255, 160, 122), font=font) return _to_image_tensor(image.convert("RGB")) left = 70 top = 92 right = width - 28 bottom = height - 58 graph_width = max(1, right - left) graph_height = max(1, bottom - top) draw.line((left, top, left, bottom), fill=axis, width=1) draw.line((left, bottom, right, bottom), fill=axis, width=1) y_values = [_safe_float(point.get("learning_loss"), 0.0) for point in points] x_values = [_safe_int(point.get("total_iteration"), index + 1) for index, point in enumerate(points)] y_min = min(y_values) y_max = max(y_values) if abs(y_max - y_min) < 1e-9: pad = 0.25 if y_max == 0 else abs(y_max) * 0.15 y_min -= pad y_max += pad else: pad = max(0.02, (y_max - y_min) * 0.12) y_min -= pad y_max += pad x_min = min(x_values) x_max = max(x_values) if x_min == x_max: x_max = x_min + 1 for i in range(5): y = top + (graph_height * i / 4.0) draw.line((left, y, right, y), fill=grid, width=1) y_label = y_max - ((y_max - y_min) * i / 4.0) draw.text((18, y - 6), f"{y_label:.3f}", fill=subtext, font=font) for i in range(5): x = left + (graph_width * i / 4.0) draw.line((x, top, x, bottom), fill=grid, width=1) x_label = round(x_min + ((x_max - x_min) * i / 4.0)) draw.text((x - 10, bottom + 10), str(x_label), fill=subtext, font=font) coords = [] for x_value, y_value in zip(x_values, y_values): norm_x = (x_value - x_min) / max(1e-9, (x_max - x_min)) norm_y = (y_value - y_min) / max(1e-9, (y_max - y_min)) px = left + norm_x * graph_width py = bottom - norm_y * graph_height coords.append((px, py)) if len(coords) == 1: px, py = coords[0] draw.ellipse((px - 4, py - 4, px + 4, py + 4), fill=point_color) else: polygon = [(coords[0][0], bottom)] + coords + [(coords[-1][0], bottom)] draw.polygon(polygon, fill=fill) draw.line(coords, fill=line, width=3) px, py = coords[-1] draw.ellipse((px - 5, py - 5, px + 5, py + 5), fill=point_color, outline=(255, 243, 214)) return _to_image_tensor(image.convert("RGB")) def update_refinement_sampler_context(refinement_key, sampler_context): """Write sampler settings to a separate lightweight file (no conditioning data, safe to call during sampling).""" if not refinement_key or not isinstance(sampler_context, dict): return path = refinement_state_path(refinement_key, "sampler_ctx", prefix="refine_v2") os.makedirs(os.path.dirname(path), exist_ok=True) try: with open(path, "w", encoding="utf-8") as f: json.dump(sampler_context, f) except Exception as e: print(f"[FunPack] update_refinement_sampler_context: failed to save context: {e}") # Fixed, prompt-agnostic key for the Absolute steering store. Unlike every other refinement # key (which hashes a specific prompt), this one collects liked/disliked directions across # ALL prompts into a single "global taste" file. Relative mode = per-prompt; Absolute = this. FUNPACK_ABSOLUTE_KEY = "__funpack_absolute_global__" # --- Audio-safe conditioning steering ------------------------------------------------------- # LTXAV conditions video and audio from two separate text contexts that the model carves out of # ONE conditioning tensor by splitting its channel dim: comfy/ldm/lightricks/av_model.py # `_prepare_context` does `torch.split(context, [v_context_dim, a_context_dim], -1)` — leading # channels drive video (self.attn2), trailing channels drive audio (self.audio_attn2). So any # conditioning steering that shifts the WHOLE tensor also corrupts the audio's text cross- # attention (observed live as degraded audio). Confining edits to the video channel-slice gives # "original conditioning for audio, modified for video" with no model patching. # LTXv2 fully-processed context = caption_channels*2 = 7680 → video = 3840 # LTXv2.3 fully-processed context = cross_attn_dim + audio_cross_attn_dim = 4096+2048 = 6144 → video = 4096 # Anything else (single-stream LTXV, unrecognised dim) returns the full width, so callers no-op. # # The table is only a FALLBACK. From LTX 2.5 on, comfy reads the two projection widths off the # text encoder's state dict (comfy/text_encoders/lt.py `sd_detect` -> video_projection_dim / # audio_projection_dim) instead of assuming them, so a checkpoint is free to ship a pair this # table has never seen. Guessing wrong here is silent and costly: an unrecognised width makes # ltxav_video_channels return the FULL width, every steering edit then moves the audio text # context too, and the only symptom is degraded audio. So whenever a live model is on hand we # read its real widths and register them — see register_ltxav_split_from_model below. _FUNPACK_AV_SPLIT_TABLE = {7680: 3840, 6144: 4096} _FUNPACK_AV_SPLIT_LEARNED = {} _FUNPACK_AV_LOGGED = set() def register_ltxav_split_from_model(model): """Read the real video/audio text-context widths off a live LTX model and register the resulting concat split, so ltxav_video_channels stops depending on the static table. `cross_attention_dim` conditions video (self.attn2), `audio_cross_attention_dim` conditions audio (self.audio_attn2), and the conditioning tensor is the two concatenated in that order — the same split `_prepare_context` performs. Silent no-op for single-stream / non-LTX models, which have no audio width to find.""" try: dm = model.model.diffusion_model except Exception: return None try: v = int(getattr(dm, "cross_attention_dim", 0) or 0) a = int(getattr(dm, "audio_cross_attention_dim", 0) or 0) except Exception: return None if v <= 0 or a <= 0: return None # single-stream (video-only) LTXV: nothing to protect total = v + a if _FUNPACK_AV_SPLIT_LEARNED.get(total) == v: return v _FUNPACK_AV_SPLIT_LEARNED[total] = v _FUNPACK_AV_LOGGED.discard(total) # re-log, now that the split is measured rather than guessed if _FUNPACK_AV_SPLIT_TABLE.get(total, total) != v: print(f"[FunPack AV] model reports video={v} + audio={a} = {total}; this does NOT match " f"the built-in table — using the model's own widths (the table would have " f"{'split at ' + str(_FUNPACK_AV_SPLIT_TABLE[total]) if total in _FUNPACK_AV_SPLIT_TABLE else 'left audio unprotected'}).") return v def ltxav_video_channels(dim): """Leading channel count that conditions VIDEO for an LTXAV concat-context of width `dim`. Returns `dim` (no audio half) for single-stream / unrecognised layouts.""" dim = int(dim) v = _FUNPACK_AV_SPLIT_LEARNED.get(dim) or _FUNPACK_AV_SPLIT_TABLE.get(dim, dim) if dim not in _FUNPACK_AV_LOGGED: _FUNPACK_AV_LOGGED.add(dim) if v < dim: src = "measured off the model" if dim in _FUNPACK_AV_SPLIT_LEARNED else "known layout" print(f"[FunPack AV] conditioning dim={dim} ({src}): steering confined to video [:{v}], audio [{v}:] protected") else: print(f"[FunPack AV] conditioning dim={dim}: single-stream (no audio half); steering applies to all channels") return v def protect_audio_channels(steered, original): """Restore the audio channel-slice of a steered conditioning tensor from `original`, so the steering only moves the video stream. No-op when shapes mismatch or there is no audio half.""" try: if not isinstance(steered, torch.Tensor) or not isinstance(original, torch.Tensor): return steered if tuple(steered.shape) != tuple(original.shape): return steered v = ltxav_video_channels(steered.shape[-1]) if v >= steered.shape[-1]: return steered out = steered.clone() out[..., v:] = original[..., v:].to(device=out.device, dtype=out.dtype) return out except Exception as _e: _log.failed("FunPackStudio", "audio channel protection", _e, "the conditioning edit reached the AUDIO channels too") return steered def scale_conditioning_spread(cond, factor): """Scale the conditioning's per-token spread (its 'sigma') on VIDEO channels only by `factor`, mean-preserving per token; audio channels are left untouched (LTXAV channel split). factor>1 amplifies the prompt's expression (toward overbake), <1 dampens it. Used by Interactive Guessing to build a strength ladder, and by the learned safe-spread cap.""" try: if not isinstance(cond, torch.Tensor) or abs(float(factor) - 1.0) < 1e-6: return cond v = ltxav_video_channels(cond.shape[-1]) out = cond.clone() vid = out[..., :v] m = vid.mean(dim=-1, keepdim=True) out[..., :v] = m + (vid - m) * float(factor) return out except Exception as _e: _log.failed("FunPackStudio", "conditioning spread scaling", _e, "the conditioning keeps its original spread") return cond def conditioning_spread(cond): """Robust per-token video-channel spread (mean std over tokens) of a conditioning tensor — the scalar Interactive Guessing's ladder and cap reason about. None if not measurable.""" try: if not isinstance(cond, torch.Tensor): return None v = ltxav_video_channels(cond.shape[-1]) vid = cond[..., :v].float() m = vid.mean(dim=-1, keepdim=True) return float(((vid - m) ** 2).mean().clamp_min(1e-12).sqrt().item()) except Exception: return None def _readable_refinement_key(refinement_key): """A human-readable, filesystem-safe rendering of a refinement key, or None when the key can't be represented without ambiguity (then callers fall back to the legacy hash so two distinct keys never collide on one file). Keys in real use are short names ("default", "123"); the hash was only ever defensive.""" key = str(refinement_key or "").strip() if not key or len(key) > 120 or key in {".", ".."}: return None # Conservative whitelist: anything outside it (slashes, colons, newlines, # unicode) would change meaning or break a path, so hash those instead. if not re.fullmatch(r"[A-Za-z0-9 _.\-]+", key): return None return key def refinement_state_path(refinement_key, mode, prefix="refine", extension="json"): """Path to a per-key state file. The clip state — the actual "refinement key" users load, share and drop into the folder — is stored readably as ``.json``; sidecars (value function, sampler context, latents) get a ``..`` suffix so they group with it. Unsafe/over-long keys (which can't be a filename) keep a ``_.ext`` hash.""" base_dir = os.path.dirname(os.path.abspath(__file__)) refinements_dir = os.path.join(base_dir, "refinements") mode_l = (mode or "ltx2").lower() safe = _readable_refinement_key(refinement_key) if safe is None: digest = md5(f"{mode_l}::{refinement_key}".encode("utf-8")).hexdigest() return os.path.join(refinements_dir, f"{prefix}_{digest}.{extension}") if mode_l == "clip": return os.path.join(refinements_dir, f"{safe}.{extension}") return os.path.join(refinements_dir, f"{safe}.{mode_l}.{extension}") def clone_latent(latent): """Shallow-clone a LATENT dict, deep-copying its tensor values. A NestedTensor (LTXAV / H3 packed audio+video) is NOT a torch.Tensor, so it is carried over BY REFERENCE and the "clone" aliases the caller's samples. That is safe here, and an audit should not re-open it: `latent_samples` returns None for a nested value, so `latent_is_plain_video_tensor` is False and `_refine_latent` rejects the latent outright (`_raise_wrong_latent`) before reaching this function. Every other caller returns the clone unmodified — the two writes in `_refine_latent` REPLACE the "samples" key rather than mutating it in place, so nothing can reach through the alias. The sampler's own `_clone_latent` unbinds and clones each stream, because it works on latents that ARE nested. """ if not isinstance(latent, dict): return None cloned = {} for key, value in latent.items(): cloned[key] = value.detach().clone() if isinstance(value, torch.Tensor) else value return cloned def latent_samples(latent): if not isinstance(latent, dict): return None samples = latent.get("samples") return samples if isinstance(samples, torch.Tensor) else None def latent_sample_type_name(latent): if not isinstance(latent, dict): return "missing" samples = latent.get("samples") if samples is None: return "missing samples" shape = tuple(samples.shape) if isinstance(samples, torch.Tensor) else "unknown" return f"{type(samples).__module__}.{type(samples).__name__}, shape={shape}" def latent_is_plain_video_tensor(latent): samples = latent_samples(latent) if samples is None: return False if samples.dim() == 5: return True if samples.dim() == 4 and latent.get("type") != "audio": return True return False def cpu_tensor_bundle(latent): if not isinstance(latent, dict): return {} bundle = {} for key, value in latent.items(): if isinstance(value, torch.Tensor): bundle[key] = value.detach().cpu().clone() return bundle def latent_from_tensor_bundle(bundle): if not isinstance(bundle, dict): return None latent = {} for key, value in bundle.items(): if key == "_meta": continue latent[key] = value.detach().clone() if isinstance(value, torch.Tensor) else value return latent if latent_samples(latent) is not None else None _GENERIC_SCENE_LABEL_PATTERN = r"\bscene\s+(?:[-+]?\d+|minus\s+[a-z][\w-]*)\b" def split_prompt_by_transitions(prompt, placement="start"): """Module-level extraction of FunPackStudio._v2_split_prompt_by_transitions. Returns [(text, visual_effect), ...] with >= 1 entry, same semantics as the method. visual_effect is the effect that fires BEFORE this segment (None for segment 0). """ try: from .templates import load_custom_transition_triggers except ImportError: from templates import load_custom_transition_triggers text = str(prompt or "").strip() if not text: return [(text, None)] custom_map = load_custom_transition_triggers() custom_triggers = list(custom_map.keys()) if custom_triggers: def _custom_trigger_pat(t): end = r"\b" if re.search(r"\w$", t) else r"(?=\s|$)" return r"\b" + re.escape(t) + end custom_parts = "|".join( _custom_trigger_pat(t) for t in sorted(custom_triggers, key=len, reverse=True) ) split_pattern = re.compile( r"(?:" + custom_parts + r"|" + _GENERIC_SCENE_LABEL_PATTERN + r")", re.IGNORECASE, ) else: split_pattern = re.compile(_GENERIC_SCENE_LABEL_PATTERN, re.IGNORECASE) capturing_pattern = re.compile(r"(" + split_pattern.pattern + r")", re.IGNORECASE) parts = capturing_pattern.split(text) segments = [] effects = [] first_segment = parts[0].strip().strip(",;.:").strip() if not first_segment: segments.append("") effects.append(None) current = first_segment current_effect = None i = 1 while i < len(parts): trans_word = parts[i].strip() following = parts[i + 1] if i + 1 < len(parts) else "" following = following.strip().lstrip(",;.:").strip() entry = custom_map.get(trans_word.lower()) or {} trans_placement = entry.get("placement") or placement if isinstance(entry, dict) else placement trans_visual_effect = entry.get("visual_effect", "none") if isinstance(entry, dict) else "none" if trans_visual_effect == "none": trans_visual_effect = None if trans_placement == "end": segments.append((current + " " + trans_word).strip().strip(",;.:").strip() if current else trans_word) effects.append(current_effect) current = following current_effect = trans_visual_effect elif trans_placement == "silent": if current: segments.append(current) effects.append(current_effect) current = following current_effect = trans_visual_effect else: if current: segments.append(current) effects.append(current_effect) current = re.sub(r"\b(a|an|the),\s*", r"\1 ", (trans_word + " " + following).strip().strip(",;.:").strip(), flags=re.IGNORECASE).strip() current_effect = trans_visual_effect i += 2 if current: segments.append(current) effects.append(current_effect) def _real_content(seg): s = split_pattern.sub("", seg).strip().strip(",;.: ") return re.sub(r"^(a|an|the)\s*$", "", s, flags=re.IGNORECASE).strip() i = 0 while i < len(segments): if segments[i] == "": i += 1 continue if not _real_content(segments[i]) and i + 1 < len(segments): connector = " " if re.search(r"\b(a|an|the)\s*$", segments[i], re.IGNORECASE) else ", " segments[i + 1] = (segments[i] + connector + segments[i + 1]).strip() segments.pop(i) effects.pop(i) else: i += 1 if len(segments) > 1 and not _real_content(segments[-1]): segments.pop() effects.pop() if len(segments) > 1: return list(zip(segments, effects)) return [(text, None)] _RUN_VIEWS = [] # the views this run added, for the rating that follows it def focus_scene_raw(scene_segments, variables=None): """Per scene, {shot number: the shot as the person typed it}: shortcut triggers still unexpanded ($variables resolved). {} for a scene whose `[Shot N]` markers only appear once its shortcuts are expanded. What the Reactive focus review prints above each shot.""" try: from .templates import resolve_variables as _resolve except ImportError: from templates import resolve_variables as _resolve try: from . import shot_camera as _sc except ImportError: import shot_camera as _sc anchor = (scene_segments.get("anchor") or "").strip() postfix = (scene_segments.get("postfix") or "").strip() out = [] for seg in scene_segments.get("scenes", []) or []: text = " ".join(p for p in (anchor, (seg or "").strip(), postfix) if p) text = _resolve(text, variables)[0] if variables else text marks = list(_sc.SHOT.finditer(text)) out.append({int(m.group(1)): " ".join(text[m.end():(marks[i + 1].start() if i + 1 < len(marks) else len(text))].split()) for i, m in enumerate(marks)}) return out def focus_scene_texts(scene_segments, variables=None, placement="start"): """Each scene's prompt as the camera-move step will see it: shortcuts expanded, the anchor and postfix folded in, $variables resolved (before cuts, views and moves, whose random draws belong to a run). What the Reactive focus review shows, so a choice is about real text.""" canonical = split_scenes_from_segments( scene_segments.get("anchor", ""), scene_segments.get("scenes", []), placement=placement, postfix=scene_segments.get("postfix", "")) anchor = (canonical.get("anchor_expanded", "") or "").strip() postfix = (canonical.get("postfix_expanded", "") or "").strip() try: from .templates import resolve_variables as _resolve_variables except ImportError: from templates import resolve_variables as _resolve_variables texts = [] for s in canonical.get("scenes", []) or []: seg = (s.get("expanded") or "").strip() if anchor and seg: seg = anchor + (" " if anchor[-1] in ".!?," else ", ") + seg else: seg = anchor or seg if postfix and seg: seg = seg + (" " if seg[-1] in ".!?," else ", ") + postfix elif postfix: seg = postfix texts.append(_resolve_variables(seg, variables)[0] if variables else seg) return texts def _shortcut_texts(min_len=15): """Every enabled shortcut's replacement text, longest first: how shot cuts tell where one of the user's shortcuts ends and the next begins in an already-expanded prompt.""" try: try: from .templates import load_shortcut_db, _shortcut_replacements except ImportError: from templates import load_shortcut_db, _shortcut_replacements out = set() for sc in (load_shortcut_db().get("shortcuts", {}) or {}).values(): if isinstance(sc, dict) and sc.get("enabled", True): for r in _shortcut_replacements(sc.get("replacements", sc.get("replacement", []))): r = str(r or "").strip() if len(r) >= min_len: out.add(r) return sorted(out, key=len, reverse=True) except Exception: # noqa: BLE001 return [] def _expand_with_map(text): """Expand shortcuts EXACTLY like apply_prompt_shortcuts, but record a position map so boundaries found in the expanded text can be projected back onto the original. Returns (expanded, pieces) where each piece is {exp_start, exp_end, orig_start, orig_end, is_shortcut}. Literal pieces map 1:1; shortcut pieces map a (possibly different-length) replacement back to the original trigger span. """ try: from .templates import (load_shortcut_db, shortcut_list, shortcut_key, _shortcut_replacements, _shortcut_trigger_pattern, normalize_refinement_key, load_revolver, revolver_next_replacement) except ImportError: from templates import (load_shortcut_db, shortcut_list, shortcut_key, _shortcut_replacements, _shortcut_trigger_pattern, normalize_refinement_key, load_revolver, revolver_next_replacement) import random from hashlib import md5 whole = [{"exp_start": 0, "exp_end": len(text), "orig_start": 0, "orig_end": len(text), "is_shortcut": False}] try: data = load_shortcut_db() except Exception as _e: _log.failed("FunPackShortcuts", "shortcut library load", _e, "the prompt is used VERBATIM — no shortcut triggers were expanded") return text, whole candidates = [] for db_key, shortcut in (data.get("shortcuts", {}) or {}).items(): if not isinstance(shortcut, dict) or not bool(shortcut.get("enabled", True)): continue reps = _shortcut_replacements(shortcut.get("replacements", shortcut.get("replacement", []))) if not reps: continue sc_key = normalize_refinement_key(shortcut.get("refinement_key", "")) sc_name = str(shortcut.get("name", "") or "") for trig in shortcut_list(shortcut.get("triggers", [])): pat = _shortcut_trigger_pattern(trig) if pat: candidates.append((trig, pat, reps, sc_key, sc_name, str(db_key))) if not candidates: return text, whole candidates.sort(key=lambda it: len(shortcut_key(it[0])), reverse=True) combined = "|".join(f"(?P{pat})" for i, (_, pat, _, _, _, _) in enumerate(candidates)) if not combined: return text, whole rng = random.Random(int(md5(text.encode("utf-8")).hexdigest()[:12], 16)) # Shortcut revolver: peek the same stored cycle generation will draw from (never saved # here) so split boundaries are found in the text generation will actually see. try: _revolver = load_revolver() except Exception: _revolver = {"enabled": False, "random": False, "state": {}} _rev_on = bool(_revolver.get("enabled")) _rev_state = _revolver.get("state") if isinstance(_revolver.get("state"), dict) else {} _rev_random = bool(_revolver.get("random")) try: pattern = re.compile(combined, re.IGNORECASE | re.UNICODE) except re.error as _e: _log.failed("FunPackShortcuts", "shortcut trigger pattern", _e, "the prompt is used VERBATIM — no shortcut triggers were expanded") return text, whole pieces, parts, exp_pos, last = [], [], 0, 0 for m in pattern.finditer(text): if m.start() > last: lit = text[last:m.start()] pieces.append({"exp_start": exp_pos, "exp_end": exp_pos + len(lit), "orig_start": last, "orig_end": m.start(), "is_shortcut": False}) parts.append(lit); exp_pos += len(lit) rep = None pc_key, pc_name = "", "" for i, (_, _, reps, sc_key, sc_name, db_key) in enumerate(candidates): if m.group(f"t{i}") is not None: if _rev_on and len(reps) > 1: rep = revolver_next_replacement(_rev_state, db_key, reps, _rev_random) else: rep = rng.choice(reps) pc_key, pc_name = sc_key, sc_name; break if rep is None: rep = m.group(0) pieces.append({"exp_start": exp_pos, "exp_end": exp_pos + len(rep), "orig_start": m.start(), "orig_end": m.end(), "is_shortcut": True, "refinement_key": pc_key, "name": pc_name}) parts.append(rep); exp_pos += len(rep) last = m.end() if last < len(text): lit = text[last:] pieces.append({"exp_start": exp_pos, "exp_end": exp_pos + len(lit), "orig_start": last, "orig_end": len(text), "is_shortcut": False}) parts.append(lit) return "".join(parts), (pieces or whole) def _project_offset(pieces, exp_off, side): """Map an expanded-text offset back to the original. `side` is "before"/"after": when the offset lands inside an expanded shortcut, the WHOLE original trigger token moves to one side (before -> token leads the next chunk; after -> token trails).""" for pc in pieces: inside = pc["exp_start"] <= exp_off < pc["exp_end"] at_end = exp_off == pc["exp_end"] if inside or (at_end and pc is pieces[-1]): if pc["is_shortcut"]: return pc["orig_start"] if side == "before" else pc["orig_end"] return pc["orig_start"] + (exp_off - pc["exp_start"]) return pieces[-1]["orig_end"] if pieces else exp_off def split_timeline_verbatim(prompt): """Lossless split of the master prompt into anchor + scenes for the editor. Correct order (all internal): expand shortcuts -> detect transitions + placements -> split as Studio does (text before the first transition is the anchor) -> then COMPACT the boundaries back onto the ORIGINAL text so each field shows verbatim chunks. No word is expanded or dropped in the result: anchor + scene texts reproduce the prompt. """ text = str(prompt or "") stripped = text.strip() if not stripped: return {"anchor": "", "scenes": [], "transitions": []} def _single(): return {"anchor": "", "scenes": [{"index": 0, "text": stripped}], "transitions": []} try: from .templates import load_custom_transition_triggers except ImportError: from templates import load_custom_transition_triggers custom_map = load_custom_transition_triggers() expanded, pieces = _expand_with_map(text) triggers = list(custom_map.keys()) if triggers: def _trig_pat(t): end = r"\b" if re.search(r"\w$", t) else r"(?=\s|$)" return r"\b" + re.escape(t) + end parts = "|".join(_trig_pat(t) for t in sorted(triggers, key=len, reverse=True)) split_pattern = re.compile(r"(?:" + parts + r"|" + _GENERIC_SCENE_LABEL_PATTERN + r")", re.IGNORECASE) else: split_pattern = re.compile(_GENERIC_SCENE_LABEL_PATTERN, re.IGNORECASE) def _effect(group): info = custom_map.get(re.sub(r"\s+", " ", group.strip().lower())) or {} eff = info.get("visual_effect") or None return (None if eff == "none" else eff), info.get("placement") cuts = [] # (orig_offset, effect) # (1) Scan the EXPANDED text: catches shortcut-driven splits, i.e. a shortcut whose # trigger word is NOT a transition but whose EXPANSION resolves to one. for m in split_pattern.finditer(expanded): effect, placement = _effect(m.group(0)) # 'start' -> cut before the trigger (leads new scene); otherwise after it. if placement == "start": orig = _project_offset(pieces, m.start(), "before") else: orig = _project_offset(pieces, m.end(), "after") cuts.append((orig, effect)) # (2) Scan the ORIGINAL text too: a transition trigger that is ALSO a shortcut (e.g. # 'qcut'/'cut') gets expanded away in (1) before detection, so its split would be lost # or misplaced. Matching the verbatim text catches it at its true offset (no projection # needed). Non-shortcut triggers yield the same offset in both scans and dedupe to one. for m in split_pattern.finditer(text): effect, placement = _effect(m.group(0)) orig = m.start() if placement == "start" else m.end() cuts.append((orig, effect)) # dedupe boundaries that map to the same original offset (e.g. several triggers inside # one shortcut expansion, or the same trigger seen by both scans) and drop any at the start. seen, uniq = set(), [] for off, eff in sorted(cuts, key=lambda c: c[0]): if off <= 0 or off in seen: continue seen.add(off); uniq.append((off, eff)) if not uniq: return _single() positions = [0] + [c[0] for c in uniq] + [len(text)] chunks = [text[positions[i]:positions[i + 1]].strip() for i in range(len(positions) - 1)] scene_texts = chunks[1:] effects = [c[1] for c in uniq] # effects[i] precedes scene_texts[i] scenes = [{"index": i, "text": t} for i, t in enumerate(scene_texts)] transitions = [ {"after_scene": i - 1, "visual_effect": effects[i]} for i in range(1, len(scene_texts)) if effects[i] ] return {"anchor": chunks[0], "scenes": scenes, "transitions": transitions} def refinement_keys_in_text(text): """The set of non-default refinement keys whose shortcuts fire in `text`. Pure read of one prompt chunk: expand its shortcuts and collect the refinement key bound to each shortcut that fired. No splitting, no scene logic — the caller hands us one scene's text and we say which keys belong to it.""" if not str(text or "").strip(): return set() try: _, pieces = _expand_with_map(str(text)) except Exception: return set() return {str(pc.get("refinement_key") or "") for pc in pieces if pc.get("is_shortcut") and str(pc.get("refinement_key") or "")} def split_scenes(prompt, placement="start"): """THE canonical scene split — one source of truth for editing, preview, key attribution, session reset and generation. Walks the ordered shortcut-expansion pieces (`_expand_with_map`) ONCE — no offset round-trip, no re-scanning of expansion content. A scene boundary is created ONLY by a real transition trigger: a shortcut whose trigger word is in the transition DB (e.g. `qcut`/`cut`), or a literal transition word the user typed. The hard-coded generic `scene ` pattern is intentionally NOT used — incidental "scene 2" text inside an expansion must never force a cut. Each non-transition keyed shortcut contributes its refinement key to the CURRENT scene; the leading run (before the first transition) is the anchor, whose keys fold into every scene. Returns: { "anchor": , "anchor_expanded": , "scenes": [ {"index": i, "raw": , # editing "expanded": , # generation encode "keys": ,# attribution / reset "effect": }, ... ], } A prompt with no transition triggers yields a single scene (the whole prompt). Because every consumer reads this ONE split, scene counts and boundaries can never disagree.""" text = str(prompt or "") try: from .templates import load_custom_transition_triggers except ImportError: from templates import load_custom_transition_triggers try: custom_map = load_custom_transition_triggers() or {} except Exception: custom_map = {} expanded_full, pieces = _expand_with_map(text) segs = [{"raw": "", "exp": "", "keys": set(), "effect": None}] def _norm(s): return re.sub(r"\s+", " ", str(s or "").strip().lower()) def _open(effect): eff = None if effect in (None, "none", "") else effect segs.append({"raw": "", "exp": "", "keys": set(), "effect": eff}) def _transition(raw_t, exp_t, info): place = (info.get("placement") or placement or "start") effect = info.get("visual_effect") if place == "end": # trigger tails the previous scene, then cut segs[-1]["raw"] += raw_t; segs[-1]["exp"] += exp_t _open(effect) elif place == "silent": # trigger dropped entirely _open(effect) else: # "start": trigger leads the new scene _open(effect) segs[-1]["raw"] += raw_t; segs[-1]["exp"] += exp_t # Literal-text trigger matcher: the user's custom triggers PLUS the generic `scene ` label. # Scoped to LITERAL text only (this matcher never runs over shortcut expansions), so a "scene 2" # the user types — or, crucially, the one the Movie Editor INJECTS into the generation prompt to # force a split for a scene with no leading trigger (see build_combined_prompt for_generation) — # still cuts, while a "scene 2" buried inside a shortcut's expansion does NOT. def _tp(t): end = r"\b" if re.search(r"\w$", t) else r"(?=\s|$)" return r"\b" + re.escape(t) + end _pats = [_tp(t) for t in sorted((t for t in custom_map.keys() if t), key=len, reverse=True)] _pats.append(_GENERIC_SCENE_LABEL_PATTERN) try: lit_pat = re.compile("|".join(_pats), re.IGNORECASE) except re.error: lit_pat = None for pc in pieces: raw = text[pc.get("orig_start", 0):pc.get("orig_end", 0)] exp = expanded_full[pc.get("exp_start", 0):pc.get("exp_end", 0)] if pc.get("is_shortcut"): # A shortcut is a transition when its TRIGGER word is in the DB (e.g. a literal # "qcut" entry) OR when its EXPANSION resolves to a transition phrase (the common # case: a "cut" shortcut expanding to the "Scene cut." split phrase). Match either, # so an expansion-keyed split marker still cuts here exactly as it does in the # verbatim editor split. Generic "scene " is intentionally NOT matched on # expansions — only user-defined custom triggers — so incidental labels never cut. info = custom_map.get(_norm(raw)) if info is None: info = custom_map.get(_norm(exp)) if info is not None: # transition-trigger shortcut _transition(raw, exp, info) else: # content shortcut: text + its key segs[-1]["raw"] += raw; segs[-1]["exp"] += exp key = str(pc.get("refinement_key") or "").strip() if key: segs[-1]["keys"].add(key) elif lit_pat is not None: # literal text may hold typed triggers last = 0 for m in lit_pat.finditer(raw): segs[-1]["raw"] += raw[last:m.start()]; segs[-1]["exp"] += raw[last:m.start()] info = custom_map.get(_norm(m.group(0))) if info is None: # Generic `scene ` label: a pure split DELIMITER (the Movie Editor injects # one to force a boundary for a scene with no leading trigger; a user may also # type it). It is NEVER content, so cut on it but DROP it ("silent") instead of # letting it lead the new scene's text. Without this the literal "scene 1", # "scene 2", … get encoded into each scene's conditioning (prompt pollution). info = {"placement": "silent"} _transition(m.group(0), m.group(0), info) last = m.end() segs[-1]["raw"] += raw[last:]; segs[-1]["exp"] += raw[last:] else: segs[-1]["raw"] += raw; segs[-1]["exp"] += exp anchor = segs[0] anchor_keys = set(anchor["keys"]) body = segs[1:] if not body: # no transitions -> one scene return {"anchor": "", "anchor_expanded": "", "scenes": [{"index": 0, "raw": anchor["raw"].strip(), "expanded": anchor["exp"].strip(), "keys": anchor_keys, "effect": None}]} scenes = [{"index": i, "raw": s["raw"].strip(), "expanded": s["exp"].strip(), "keys": anchor_keys | s["keys"], "effect": s["effect"]} for i, s in enumerate(body)] return {"anchor": anchor["raw"].strip(), "anchor_expanded": anchor["exp"].strip(), "scenes": scenes} def split_scenes_from_segments(anchor, segments, placement="start", postfix=""): """Canonical scene structure from EXPLICIT per-scene boundaries supplied by the caller (the Movie Editor's own scene list) instead of re-deriving boundaries from in-text delimiters. The editor already KNOWS where each scene starts (they are separate Scene objects), so it hands us that list directly — no `scene N` marker ever has to be injected into the prompt. `anchor` is the shared prefix text; `segments` is one raw text per editor scene (each may carry its own leading transition trigger, which supplies that scene's effect). Each segment becomes EXACTLY one scene; its shortcut expansion + refinement keys + effect come from split_scenes (reused per item, where there is no boundary ambiguity), and the anchor keys fold into every scene — identical semantics to split_scenes(), same output shape.""" def _flatten(split_out): # Collapse a split_scenes() result to one scene: any leading anchor text + every # sub-scene joined; first non-None effect wins; keys unioned. raw_parts, exp_parts, keys, effect = [], [], set(), None ar = (split_out.get("anchor") or "").strip() ae = (split_out.get("anchor_expanded") or "").strip() if ar: raw_parts.append(ar) if ae: exp_parts.append(ae) for s in (split_out.get("scenes") or []): r = (s.get("raw") or "").strip() e = (s.get("expanded") or "").strip() if r: raw_parts.append(r) if e: exp_parts.append(e) keys |= set(s.get("keys") or set()) if effect is None and s.get("effect"): effect = s.get("effect") return " ".join(raw_parts).strip(), " ".join(exp_parts).strip(), keys, effect a_raw, a_exp, anchor_keys, _ = _flatten(split_scenes(str(anchor or ""), placement=placement)) # Postfix mirrors the anchor: shortcut-expanded once, its keys fold into every scene, and # its expanded text is appended to each scene downstream (see the structural fold in V2). p_raw, p_exp, postfix_keys, _ = _flatten(split_scenes(str(postfix or ""), placement=placement)) shared_keys = anchor_keys | postfix_keys scenes = [] for i, seg in enumerate(segments or []): r, e, k, eff = _flatten(split_scenes(str(seg or ""), placement=placement)) scenes.append({"index": i, "raw": r, "expanded": e, "keys": shared_keys | k, "effect": eff}) return {"anchor": a_raw, "anchor_expanded": a_exp, "postfix": p_raw, "postfix_expanded": p_exp, "scenes": scenes} def refinement_key_pool_for(prompt): """Every non-default refinement key the prompt activates (anchor + all scenes) — the exact set a session reset wipes. Derived from the one canonical split.""" pool = set() for scene in split_scenes(prompt).get("scenes", []) or []: pool |= scene.get("keys") or set() return pool def resolve_scene_refinement_keys(raw_positive): """Per-scene non-default refinement keys, read straight off the canonical scene split. Returns a list of sets aligned 1:1 with `split_scenes(raw_positive)["scenes"]`; an empty set means that scene steers/trains with the project DEFAULT key. There is no scene-count parameter and no divergence fallback: every consumer reads the ONE canonical split, so attribution can't disagree with the preview or with generation. Anchor keys are already folded into each scene by `split_scenes`.""" try: return [scene.get("keys") or set() for scene in split_scenes(str(raw_positive or "")).get("scenes", []) or []] except Exception as error: print(f"[FunPackStudio] Scene refinement-key attribution failed: {error}") return [] def parse_timeline_segments(prompt): """Return {"anchor": str, "scenes": [...], "transitions": [...]} for timeline preview. Mirrors _v2_transition_scene_texts: segments[0] is the character anchor prepended to every scene, segments[1:] are the actual generated scenes. If no transitions found, the single segment is treated as scene 1 with no anchor. """ segments = split_prompt_by_transitions(prompt, placement="silent") if len(segments) == 1: return { "anchor": "", "scenes": [{"index": 0, "text": segments[0][0]}], "transitions": [], } anchor = segments[0][0].strip() scenes = [{"index": i, "text": text.strip()} for i, (text, _) in enumerate(segments[1:])] transitions = [ {"after_scene": i - 1, "visual_effect": effect} for i, (_, effect) in enumerate(segments[1:]) if i > 0 ] return {"anchor": anchor, "scenes": scenes, "transitions": transitions} class _ClipTokenizerAdapter: """Presents a live Comfy tokenizer through the slice of the HuggingFace API the refinement code uses: call it for `input_ids`, decode ids back to text. Comfy's inner tokenizers take text and nothing else, so truncation and the leading start token are handled here. Dropping the start token matters: concept spans are found by matching a phrase's ids inside the whole prompt's ids, and a BOS on the phrase never matches.""" def __init__(self, sd_tokenizer): self._t = sd_tokenizer.tokenizer self._start = getattr(sd_tokenizer, "start_token", None) self.name_or_path = str(getattr(sd_tokenizer, "embedding_key", "") or "CLIP") def __call__(self, text, add_special_tokens=True, truncation=False, max_length=None, **_): ids = list((self._t(text) or {}).get("input_ids") or []) if not add_special_tokens and self._start is not None: while ids and ids[0] == self._start: ids.pop(0) if truncation and max_length: ids = ids[: int(max_length)] return {"input_ids": ids} def encode(self, text, **kwargs): return self(text, **kwargs)["input_ids"] def decode(self, token_ids, skip_special_tokens=True): try: return self._t.decode(list(token_ids), skip_special_tokens=skip_special_tokens) except TypeError: return self._t.decode(list(token_ids)) class FunPackVideoRefiner: LATENT_OUTPUT_INDEX = 6 NO_LATENT_REFERENCE_ERROR = "No available latent to operate. Please connect reference latent to input of Video Refiner." WRONG_LATENT_ERROR = ( "Video Refiner latent input must be a plain video LATENT with tensor samples. " "Audio latents and LTX audio/video combined NestedTensor latents are not supported here. " "Connect only the video latent to Video Refiner, then feed the refined video latent into " "LTXVConcatAVLatent as video_latent." ) SAVED_LATENT_ONLY_STATUS = ( "Running refinement on saved latent. Changing reference latent shape and size will cause no effect to generation." ) _tokenizers = {} _tokenizer_sources = { "ltx2": [ ("DreamFast/gemma-3-12b-it-heretic-v2", { "trust_remote_code": True, "use_fast": True, }), ], # MiniMax H3 encodes with Qwen3-VL, not Gemma. Falling through to the ltx2 entry # downloaded a 12B Gemma tokenizer for a Qwen model: every token span it produced was # measured with the wrong vocabulary, and it fetched ~50 MB over the network to do it. "minimax_h3": [ ("Qwen/Qwen3-VL-32B-Instruct", {"use_fast": True}), ("Qwen/Qwen3-32B", {"use_fast": True}), ], "wan": [ ("Wan-AI/Wan2.2-T2V-A14B", { "subfolder": "google/umt5-xxl", "use_fast": True, }), ("Wan-AI/Wan2.2-I2V-A14B", { "subfolder": "google/umt5-xxl", "use_fast": True, }), ("Wan-AI/Wan2.2-Animate-14B", { "subfolder": "google/umt5-xxl", "use_fast": True, }), ("Wan-AI/Wan2.1-T2V-1.3B", { "subfolder": "google/umt5-xxl", "use_fast": True, }), ("Runware/Wan2.2-TI2V-5B", { "subfolder": "google/umt5-xxl", "use_fast": True, }), ("ddwf/Wan2.2-Animate-14B", { "subfolder": "google/umt5-xxl", "use_fast": True, }), ], } @classmethod def _get_tokenizer_sources(cls, mode="ltx2"): mode = (mode or "ltx2").lower() sources = list(cls._tokenizer_sources.get(mode, cls._tokenizer_sources["ltx2"])) if mode == "wan": local_sources = [] models_dir = getattr(folder_paths, "models_dir", None) if models_dir: local_sources.extend( (path, {}) for path in glob.glob(os.path.join(models_dir, "Wan", "*", "google", "umt5-xxl")) if os.path.isdir(path) ) text_encoder_tokenizer = os.path.join(models_dir, "text_encoders", "google", "umt5-xxl") if os.path.isdir(text_encoder_tokenizer): local_sources.append((text_encoder_tokenizer, {})) if local_sources: sources = local_sources + sources return sources @staticmethod def _clip_text_tokenizer(clip): """The tokenizer that actually produced the conditioning, or None. `clip.tokenizer` is a Comfy SD1Tokenizer holding one named sub-tokenizer — `gemma3_12b` on LTX-2.3, `gemma4` on LTX-2.5, `umt5_xxl` on WAN. The name is on `.clip`.""" top = getattr(clip, "tokenizer", None) name = getattr(top, "clip", None) if top is not None else None sub = getattr(top, name, None) if isinstance(name, str) else None return sub if getattr(sub, "tokenizer", None) is not None else None @staticmethod def _encoder_family_changed(stored, current): """Whether learned state from `stored` may still be applied under `current`. Either side being empty means unknown — an older session file, or a run with no CLIP connected — and unknown must never trigger a reset, or every conditioning-only run would wipe the key it was meant to use.""" return bool(current and stored and stored != current) @classmethod def _encoder_family(cls, clip): """Short name of the text encoder behind `clip` ("gemma3_12b", "gemma4", ...), or "".""" sub = cls._clip_text_tokenizer(clip) return str(getattr(sub, "embedding_key", "") or "") if sub is not None else "" @classmethod def _get_tokenizer(cls, mode="ltx2", clip=None): """Prefer the CLIP's own tokenizer. A downloaded one has to guess the encoder family, and guessing wrong is silent: the spans still resolve, they just point at the wrong positions. LTX-2.3 encodes with Gemma3 and LTX-2.5 with Gemma4.""" if clip is not None: sub = cls._clip_text_tokenizer(clip) if sub is not None: try: return _ClipTokenizerAdapter(sub) except Exception as e: print(f"[FunPackVideoRefiner] CLIP tokenizer unusable, falling back to download: {e}") mode = (mode or "ltx2").lower() cached = cls._tokenizers.get(mode) if cached is not None: return cls._tokenizers[mode] sources = cls._get_tokenizer_sources(mode) # Local cache FIRST, every source, before any network call. `from_pretrained` reaches # HuggingFace with no token, no timeout and no progress, on ComfyUI's execution # thread — mid-run that is an unbounded stall with nothing in the log to explain it, # and on a rate-limited unauthenticated request it can be a long one. A tokenizer # that is already on disk should never pay for that. for local_only in (True, False): for model_id, kwargs in sources: if not local_only: print(f"[FunPackVideoRefiner] tokenizer for '{mode}' is not in the local " f"HuggingFace cache — DOWNLOADING '{model_id}'. This blocks the run " f"until it finishes; set HF_TOKEN to avoid unauthenticated rate limits.") try: cls._tokenizers[mode] = AutoTokenizer.from_pretrained( model_id, **dict(kwargs, local_files_only=local_only)) return cls._tokenizers[mode] except Exception as e: if not local_only: print(f"[FunPackVideoRefiner] Tokenizer load failed for mode '{mode}' " f"from '{model_id}': {e}") return None def _normalize_prompt_for_mode(self, prompt: str, mode: str) -> str: prompt = (prompt or "").strip() if mode == "wan": return re.sub(r"\s+", " ", prompt) return prompt def _prompt_looks_like_refusal(self, prompt: str) -> bool: text = re.sub(r"\s+", " ", str(prompt or "").strip().lower()) if not text: return False text = text.replace("’", "'").replace("`", "'") prefix = text[:520] refusal_patterns = ( r"^(?:i'?m|i am)?\s*sorry\b.{0,120}\b(?:can(?:not|'?t)|unable|not able|won'?t)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate|do that|with this request)\b", r"^(?:i|we)\s+(?:can(?:not|'?t)|won'?t|am unable|are unable|am not able|are not able)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate|do that|with this request)\b", r"^as an ai\b.{0,160}\b(?:can(?:not|'?t)|unable|not able|won'?t)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate)\b", r"^(?:i|we)\s+(?:must|have to|need to)\s+(?:decline|refuse)\b", ) return any(re.search(pattern, prefix) for pattern in refusal_patterns) def _v2_run_looks_like_refusal(self, run): if not isinstance(run, dict): return False return ( self._prompt_looks_like_refusal(run.get("prompt", "")) or self._prompt_looks_like_refusal(run.get("encoded_prompt", "")) ) def _get_conditioning_seq_len(self, conditioning: torch.Tensor) -> int: if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1: return 0 return conditioning.shape[1] if conditioning.dim() == 3 else conditioning.shape[0] def _get_conditioning_token_mask(self, conditioning: torch.Tensor): if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1: return None if conditioning.dim() == 3: token_energy = conditioning.detach().abs().sum(dim=-1) mask = token_energy.gt(1e-12).any(dim=0) else: token_energy = conditioning.detach().abs().sum(dim=-1) mask = token_energy.gt(1e-12) if not bool(mask.any()): seq_len = self._get_conditioning_seq_len(conditioning) return torch.ones(seq_len, dtype=torch.bool, device=conditioning.device) return mask def _get_effective_seq_len(self, token_mask, fallback_seq_len: int) -> int: if token_mask is None: return fallback_seq_len active_positions = torch.nonzero(token_mask, as_tuple=False).flatten() if active_positions.numel() == 0: return fallback_seq_len return min(fallback_seq_len, int(active_positions[-1].item()) + 1) def _mask_to_embedding_dims(self, token_mask, reference: torch.Tensor): if token_mask is None or not isinstance(reference, torch.Tensor) or reference.dim() <= 1: return None token_mask = token_mask.to(device=reference.device, dtype=reference.dtype) if reference.dim() == 3: return token_mask.view(1, -1, 1) return token_mask.view(-1, 1) def _masked_sequence_mean(self, conditioning: torch.Tensor, token_mask): if not isinstance(conditioning, torch.Tensor): return conditioning if conditioning.dim() <= 1 or token_mask is None: return conditioning mask = self._mask_to_embedding_dims(token_mask, conditioning) if conditioning.dim() == 3: denom = mask.sum(dim=1).clamp_min(1.0) return (conditioning * mask).sum(dim=1) / denom denom = mask.sum(dim=0).clamp_min(1.0) return (conditioning * mask).sum(dim=0) / denom def _tokenize_ids(self, tokenizer, text: str, add_special_tokens: bool, max_length: Optional[int] = None): if not tokenizer or not text: return [] kwargs = {"add_special_tokens": add_special_tokens} if max_length is not None and max_length > 0: kwargs["truncation"] = True kwargs["max_length"] = max_length try: tokenized = tokenizer(text, **kwargs) input_ids = tokenized.get("input_ids", tokenized) if isinstance(input_ids, list) and input_ids and isinstance(input_ids[0], list): input_ids = input_ids[0] return input_ids if isinstance(input_ids, list) else list(input_ids) except Exception: try: return tokenizer.encode(text, **kwargs) except Exception: return [] def _iter_prompt_segments(self, prompt: str): if not prompt: return last_end = 0 for match in PROTECTED_PHRASE_RE.finditer(prompt): start, end = match.span() if start > last_end: yield ("text", prompt[last_end:start]) protected_text = (match.group(1) or match.group(2) or match.group(3) or "").strip() if protected_text: yield ("protected", protected_text) last_end = end if last_end < len(prompt): yield ("text", prompt[last_end:]) def _mask_quoted_text(self, prompt: str): if not prompt: return "" return PROTECTED_PHRASE_RE.sub(lambda match: " " * len(match.group(0)), prompt) def _fallback_word_groups_from_prompt(self, prompt: str, seq_len: int, token_mask=None, existing_groups=None): if not prompt or seq_len <= 0: return [] effective_seq_len = self._get_effective_seq_len(token_mask, seq_len) if effective_seq_len <= 0: return [] active_positions = list(range(effective_seq_len)) if token_mask is not None: active_positions = [ index for index, enabled in enumerate(token_mask[:effective_seq_len].detach().cpu().tolist()) if enabled ] if not active_positions: return [] existing = { str(group[2]).strip().lower() for group in (existing_groups or []) if isinstance(group, (list, tuple)) and len(group) >= 3 } candidates = [] seen = set(existing) for segment_type, segment_text in self._iter_prompt_segments(prompt): if segment_type == "protected": pieces = [segment_text.strip()] else: pieces = re.findall(r"[\w'’.-]+", segment_text, flags=re.UNICODE) for piece in pieces: clean = piece.strip(" \t\r\n,;:!?()[]{}\"'") lower = clean.lower() if lower in seen or not self._is_valuable_token(clean): continue seen.add(lower) candidates.append(clean) if not candidates: return [] fallback = [] step_count = len(candidates) max_slots = len(active_positions) for index, word in enumerate(candidates[:max_slots]): if step_count == 1: pos_index = len(active_positions) // 2 else: pos_index = round(index * (len(active_positions) - 1) / max(1, min(step_count, max_slots) - 1)) start = int(active_positions[max(0, min(len(active_positions) - 1, pos_index))]) fallback.append((start, min(effective_seq_len, start + 1), word, [])) return fallback def _build_word_groups(self, prompt: str, tokenizer, seq_len: int, token_mask=None): if not prompt or seq_len <= 0: return [] if not tokenizer: return self._fallback_word_groups_from_prompt(prompt, seq_len, token_mask) effective_seq_len = self._get_effective_seq_len(token_mask, seq_len) token_mask_list = None if token_mask is not None: token_mask_list = token_mask[:effective_seq_len].detach().cpu().tolist() full_token_ids = self._tokenize_ids( tokenizer, prompt, add_special_tokens=True, max_length=effective_seq_len )[:effective_seq_len] word_groups = [] grouped_seen = set() for segment_type, segment_text in self._iter_prompt_segments(prompt): if segment_type != "protected": continue clean_phrase = segment_text.strip() lower = clean_phrase.lower() if lower in grouped_seen or len(clean_phrase) < 3 or not any(c.isalpha() for c in clean_phrase): continue grouped_seen.add(lower) phrase_token_list = self._tokenize_ids( tokenizer, clean_phrase, add_special_tokens=False ) if not phrase_token_list: continue for start in range(max(0, len(full_token_ids) - len(phrase_token_list) + 1)): end = start + len(phrase_token_list) if token_mask_list is not None and not all(token_mask_list[start:min(effective_seq_len, end)]): continue if full_token_ids[start:end] == phrase_token_list and start < effective_seq_len: word_groups.append((start, min(effective_seq_len, end), clean_phrase, phrase_token_list)) break raw_words = [w.strip() for w in self._mask_quoted_text(prompt).split() if w.strip()] for word in raw_words: clean_word = word lower = clean_word.lower() if lower in grouped_seen or len(clean_word) < 3 or not self._is_valuable_token(clean_word): continue grouped_seen.add(lower) word_token_list = self._tokenize_ids( tokenizer, clean_word, add_special_tokens=False ) if not word_token_list: continue found = False for start in range(max(0, len(full_token_ids) - len(word_token_list) + 1)): end = start + len(word_token_list) if token_mask_list is not None and not all(token_mask_list[start:min(effective_seq_len, end)]): continue if full_token_ids[start:end] == word_token_list and start < effective_seq_len: word_groups.append((start, min(effective_seq_len, end), clean_word, word_token_list)) found = True break if not found: continue if len(word_groups) < 2: word_groups.extend( self._fallback_word_groups_from_prompt( prompt, seq_len, token_mask, existing_groups=word_groups, ) ) return sorted( [group for group in word_groups if group[1] > group[0]], key=lambda group: (group[0], group[1], group[2].lower()), ) def _missing_axes_for_rating_key(self, rating_key): for profile in RATING_PROFILES.values(): if profile.get("key") == rating_key: return set(profile.get("missing_axes", [])) return set() def _axis_counter_name(self, axis): if axis == "concept": return "missing_concept_count" if axis == "details": return "missing_detail_count" if axis == "quality": return "missing_quality_count" return None def _axis_memory_boost(self, missing_axes): axes = set(missing_axes or []) boost = 0.0 if "concept" in axes: boost += 0.42 if "details" in axes: boost += 0.30 if "quality" in axes: boost += 0.24 if len(axes) >= 2: boost += 0.16 return boost def _apply_missing_memory_pressure(self, item, missing_axes, iter_num): axes = set(missing_axes or []) if not isinstance(item, dict) or not axes: return item item["wanted_count"] = max(0, int(item.get("wanted_count", 0)) + 1) item["missing_count"] = max(0, int(item.get("missing_count", 0)) + 1) for axis in axes: counter = self._axis_counter_name(axis) if counter: item[counter] = max(0, int(item.get(counter, 0)) + 1) item["last_missing_iter"] = int(iter_num) return item def _phrase_position_score(self, item): likes = float(item.get("liked_count", 0)) neutral = float(item.get("neutral_count", 0)) awful = float(item.get("awful_count", 0)) wanted = float(item.get("wanted_count", 0)) missing = float(item.get("missing_count", 0)) return float(item.get("score", 0.0)) + likes * 0.45 + neutral * 0.06 + wanted * 0.26 + missing * 0.18 - awful * 0.85 def _rating_memory_deltas(self, rating_key): if rating_key == "like": return 0.70, 1, 0, 0 if rating_key == "missing_details": return 0.56, 0, 1, 0 if rating_key == "missing_concept": return 0.72, 0, 1, 0 if rating_key == "missing_quality": return 0.48, 0, 1, 0 if rating_key == "missing_details_concept": return 0.84, 0, 1, 0 if rating_key == "missing_details_quality": return 0.68, 0, 1, 0 if rating_key == "missing_concept_quality": return 0.80, 0, 1, 0 if rating_key == "awful": return -1.10, 0, 0, 1 if rating_key == "discover": return 0.0, 0, 1, 0 return 0.12, 0, 1, 0 def _history_modified_conditioning(self, history_entry, reference, device): if not isinstance(history_entry, dict): return None mod_data = history_entry.get("modified_embeds") if mod_data is None: return None try: candidate = serializable_to_tensor(mod_data).to(device) except Exception: return None if not isinstance(reference, torch.Tensor) or list(candidate.shape) != list(reference.shape): return None return candidate def _history_modified_conditioning_any_shape(self, history_entry, device, dtype=None): if not isinstance(history_entry, dict): return None mod_data = history_entry.get("modified_embeds") if mod_data is None: return None try: candidate = serializable_to_tensor(mod_data).to(device) if dtype is not None: candidate = candidate.to(dtype=dtype) except Exception: return None if not isinstance(candidate, torch.Tensor) or candidate.dim() <= 1: return None return candidate def _conditioning_canvas_compatible(self, candidate, template): if ( not isinstance(candidate, torch.Tensor) or not isinstance(template, torch.Tensor) or candidate.dim() <= 1 or template.dim() <= 1 or candidate.dim() != template.dim() or int(candidate.shape[-1]) != int(template.shape[-1]) ): return False if candidate.dim() == 3 and int(candidate.shape[0]) != int(template.shape[0]): return False return self._get_conditioning_seq_len(candidate) > 0 def _resize_conditioning_sequence_like(self, source, target): if not isinstance(source, torch.Tensor) or not isinstance(target, torch.Tensor): return None if source.dim() != target.dim() or source.dim() <= 1: return None if int(source.shape[-1]) != int(target.shape[-1]): return None if source.dim() == 3 and int(source.shape[0]) != int(target.shape[0]): return None if list(source.shape) == list(target.shape): return source.to(device=target.device, dtype=target.dtype) target_seq = self._get_conditioning_seq_len(target) if target_seq <= 0: return None try: working = source.to(device=target.device, dtype=target.dtype) if working.dim() == 3: resized = F.interpolate( working.transpose(1, 2), size=target_seq, mode="linear", align_corners=False, ).transpose(1, 2) else: resized = F.interpolate( working.transpose(0, 1).unsqueeze(0), size=target_seq, mode="linear", align_corners=False, ).squeeze(0).transpose(0, 1) except Exception: return None if list(resized.shape) != list(target.shape): return None return resized def _serialized_tensor_shape(self, payload): if not isinstance(payload, dict): return None shape = payload.get("shape") if not isinstance(shape, list) or not shape: return None try: return tuple(int(dim) for dim in shape) except (TypeError, ValueError): return None def _serialized_conditioning_compatible(self, payload, template): shape = self._serialized_tensor_shape(payload) if ( shape is None or not isinstance(template, torch.Tensor) or template.dim() <= 1 or len(shape) != int(template.dim()) or int(shape[-1]) != int(template.shape[-1]) ): return False if len(shape) == 3 and int(shape[0]) != int(template.shape[0]): return False return (shape[1] if len(shape) == 3 else shape[0]) > 0 def _serialized_conditioning_seq_len(self, payload): shape = self._serialized_tensor_shape(payload) if shape is None or len(shape) <= 1: return 0 return int(shape[1] if len(shape) == 3 else shape[0]) def _legacy_auto_inject_token_allowed(self, token): token = str(token or "").strip().lower() if not token: return False category = self._infer_concept_category([token]) if category in {"appearance", "subject", "environment", "character"}: return False return True def _get_scheduler_factors(self, mode, rating, reward, similarity, iter_num, total_iters, word_importance, word_groups, global_adaptive, device): if mode == "original": base_lr = 0.18 / (1 + 0.08 * (iter_num ** 0.5)) confidence = max(0.2, min(1.0, (rating - 3.0) / 5.0)) if similarity > 0.93: confidence *= 0.4 return base_lr, confidence, 1.0, {} confidence = max(0.15, min(1.0, (rating - 2.5) / 6.0)) if abs(reward) < 0.15 or similarity > 0.94: confidence *= 0.35 prodigy_d = global_adaptive.setdefault("prodigy_d", {}) d_coef = 1.0 if mode == "accurate" else 1.8 adaptive_lr = global_adaptive.get("prodigy_lr_base", 1.0) word_lr_mult = {} for _, _, full_word, _ in word_groups: wkey = full_word.lower() if wkey not in word_importance: continue g = abs(reward) + 1e-8 if wkey not in prodigy_d: prodigy_d[wkey] = g else: prodigy_d[wkey] = 0.9 * prodigy_d[wkey] + 0.1 * g word_lr_mult[wkey] = adaptive_lr / (prodigy_d[wkey] ** 0.5 + 1e-8) * d_coef current_step = global_adaptive.setdefault("current_step", 0) current_step = min(current_step + 1, 500) global_adaptive["current_step"] = current_step warmup = global_adaptive.get("warmup_steps", 8) progress = min(1.0, current_step / max(50, global_adaptive.get("total_steps_estimate", 150))) if current_step < warmup: lr_scale = current_step / max(1, warmup) else: lr_scale = 0.5 * (1.0 + math.cos(math.pi * (progress - warmup / (warmup + 50)))) if mode == "accurate": lr_scale *= 0.75 confidence *= 0.9 exploration_mult = max(0.3, 1.0 - 0.6 * progress) else: lr_scale = min(2.2, lr_scale * 1.6) confidence = min(1.0, confidence * 1.3) exploration_mult = max(0.6, 1.3 - 0.7 * progress) return lr_scale, confidence, exploration_mult, word_lr_mult # ========================================================================= # TOKEN / WORD UTILITIES # ========================================================================= def _get_top_tokens(self, token_dict, tokenizer, top_k=10): if not token_dict or not tokenizer: return "N/A" sorted_tokens = sorted(token_dict.items(), key=lambda x: x[1], reverse=True)[:top_k] top_list = [] for key, score in sorted_tokens: try: text = key if isinstance(key, str) else tokenizer.decode([int(key)], skip_special_tokens=True).strip() if text: top_list.append(f"{text}({score:.2f})") except Exception: top_list.append(f"{key}({score:.2f})") return ", ".join(top_list) if top_list else "None" def _is_valuable_token(self, token_text): if not token_text: return False t = token_text.strip() if '<' in t or '>' in t or len(t) < 3: return False t_lower = t.lower() stopwords = { "the", "a", "an", "and", "or", "but", "with", "for", "of", "in", "on", "at", "to", "from", "by", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "her", "his", "him", "she", "he", "it", "they", "them", "this", "that", "these", "those", "i", "you", "my", "your", "our", "their", "me", "us" } if t_lower in stopwords: return False if t in {",", ".", "!", "?", ":", ";", "-", "*", "(", ")", "[", "]", "{", "}", "'", "\"", "..."} or t.isdigit(): return False if not any(c.isalpha() for c in t): return False return True def _infer_concept_category(self, phrase_words: list): words = set(phrase_words or []) if not words: return "general" category_terms = { "quality": {"masterpiece", "best", "quality", "detailed", "highres", "high-res", "ultra", "perfect"}, "style": {"anime", "cinematic", "photorealistic", "painterly", "illustration", "stylized", "realistic", "film", "noir"}, "camera": {"closeup", "close-up", "wide", "shot", "angle", "zoom", "pan", "tracking", "dolly", "camera", "focus", "bokeh", "framing"}, "action": {"running", "walking", "flying", "jumping", "smiling", "turning", "dancing", "moving", "motion", "looking", "holding", "standing", "sitting"}, "environment": {"forest", "city", "street", "room", "beach", "mountain", "temple", "sunset", "night", "rain", "snow", "sky", "background"}, "appearance": {"hair", "eyes", "dress", "jacket", "armor", "face", "skin", "beard", "smile", "pose", "outfit"}, "subject": {"girl", "boy", "woman", "man", "person", "character", "robot", "dragon", "cat", "dog", "bird", "child"}, } best_category, best_score = "general", 0 for category, terms in category_terms.items(): score = len(words & terms) if score > best_score: best_category, best_score = category, score return best_category def _default_concept_cluster(self, phrase_words: list): category = self._infer_concept_category(phrase_words) return { "label": " ".join((phrase_words or [])[:6]), "anchor_words": list(phrase_words or []), "category": category, "category_source": "auto", "category_confidence": 0.35 if category == "general" else 0.65, "word_importance": {}, "presence_target": 1.0, "priority_weight": 1.0, "overrep_sensitivity": 1.0, "stability_weight": 1.0, "semantic_fidelity": 1.0, "user_affinity": 1.0, "question_history": [], "last_question_type": None, "last_question_iter": 0, "usage_count": 0, "last_seen_iter": 0, } def _ensure_concept_cluster_defaults(self, cluster: dict): if not isinstance(cluster, dict): return self._default_concept_cluster([]) anchor_words = list(cluster.get("anchor_words", [])) defaults = self._default_concept_cluster(anchor_words) defaults.update(cluster) defaults.setdefault("category_source", "auto") defaults.setdefault("category_confidence", 0.35 if defaults.get("category") == "general" else 0.65) if defaults.get("category_source") != "user" and ( not defaults.get("category") or defaults.get("category") == "general" ): defaults["category"] = self._infer_concept_category(defaults.get("anchor_words", [])) defaults["category_confidence"] = 0.35 if defaults["category"] == "general" else max(0.65, defaults.get("category_confidence", 0.0)) defaults["question_history"] = list(defaults.get("question_history", []))[-24:] return defaults def _clip_profile_value(self, value, low=0.5, high=1.8): return max(low, min(high, float(value))) def _feedback_question_specs(self): return { "presence": { "prompt": "How well is '{label}' represented in the output?", "legend": "1=absent 2=weak 3=slightly weak 4=slightly strong 5=perfect 6=overrepresented", }, "priority": { "prompt": "How important should '{label}' be relative to the other concepts?", "legend": "1=much less important 2=less important 3=slightly less 4=slightly more 5=important 6=top priority", }, "balance": { "prompt": "How balanced is '{label}' compared with nearby concepts?", "legend": "1=far too weak 2=too weak 3=slightly weak 4=slightly strong 5=balanced 6=overpowering", }, "fidelity": { "prompt": "How accurately does '{label}' match what you meant?", "legend": "1=wrong 2=mostly wrong 3=slightly off 4=close 5=correct 6=too literal", }, "stability": { "prompt": "How stable should '{label}' be across future outputs?", "legend": "1=very unstable 2=unstable 3=slightly unstable 4=slightly stable 5=stable 6=very rigid", }, "preference": { "prompt": "What is your preference for having '{label}' in future outputs?", "legend": "1=strongly less 2=less 3=slightly less 4=slightly more 5=more 6=much more", }, "category": { "prompt": "What kind of concept is '{label}'?", "legend": "1=general 2=concept 3=style 4=quality 5=character 6=details", }, } def _get_concept_mean_importance(self, cluster: dict): local_imp = cluster.get("word_importance", {}) if not local_imp: return 1.0 return sum(local_imp.values()) / max(1, len(local_imp)) def _get_question_base_weight(self, question_type: str, category: str): weights = { "presence": {"subject": 1.15, "appearance": 1.12, "action": 1.10, "environment": 1.08, "style": 0.95, "camera": 0.92, "quality": 0.90, "general": 1.0}, "priority": {"style": 1.15, "camera": 1.12, "quality": 1.08, "environment": 1.02, "subject": 0.98, "appearance": 0.98, "action": 1.0, "general": 1.0}, "balance": {"style": 1.18, "camera": 1.10, "quality": 1.08, "environment": 1.02, "subject": 0.98, "appearance": 1.0, "action": 1.0, "general": 1.0}, "fidelity": {"subject": 1.15, "action": 1.12, "environment": 1.08, "appearance": 1.05, "style": 0.95, "camera": 0.92, "quality": 0.90, "general": 1.0}, "stability": {"action": 1.10, "camera": 1.10, "style": 1.06, "subject": 1.04, "appearance": 1.0, "environment": 1.0, "quality": 0.95, "general": 1.0}, "preference": {"style": 1.14, "environment": 1.10, "appearance": 1.05, "subject": 1.03, "camera": 1.02, "action": 1.0, "quality": 0.95, "general": 1.0}, } return weights.get(question_type, {}).get(category, 1.0) def _score_question_type(self, question_type: str, cluster: dict, category: str, mean_imp: float, neighbor_mean: float, rating_shift: float, similarity: float, iter_num: int): uncertainty = max(0.0, 1.0 - min(1.0, abs(mean_imp - cluster.get("presence_target", 1.0)))) dominance = max(0.0, mean_imp - 1.15) neighbor_conflict = abs(mean_imp - neighbor_mean) freshness = 1.0 if cluster.get("last_question_type") == question_type: freshness -= 0.35 if iter_num - int(cluster.get("last_question_iter", 0)) < 3: freshness -= 0.20 freshness = max(0.25, freshness) if question_type == "presence": score = 0.50 * uncertainty + 0.22 * abs(cluster.get("presence_target", 1.0) - mean_imp) + 0.16 * rating_shift + 0.12 * max(0.0, 1.0 - similarity) elif question_type == "priority": score = 0.40 * rating_shift + 0.28 * abs(cluster.get("priority_weight", 1.0) - 1.0) + 0.18 * dominance + 0.14 * uncertainty elif question_type == "balance": score = 0.44 * dominance + 0.28 * neighbor_conflict + 0.16 * abs(cluster.get("overrep_sensitivity", 1.0) - 1.0) + 0.12 * rating_shift elif question_type == "fidelity": score = 0.38 * (2.0 - cluster.get("semantic_fidelity", 1.0)) + 0.24 * rating_shift + 0.20 * uncertainty + 0.18 * max(0.0, 1.0 - similarity) elif question_type == "stability": score = 0.35 * rating_shift + 0.30 * abs(cluster.get("stability_weight", 1.0) - 1.0) + 0.20 * max(0.0, 1.0 - similarity) + 0.15 * uncertainty elif question_type == "category": category_uncertainty = max(0.0, 1.0 - float(cluster.get("category_confidence", 0.35))) needs_category_discovery = ( cluster.get("category_source") != "user" and (category == "general" or category_uncertainty >= 0.50) ) if not needs_category_discovery: return 0.0 score = 0.65 + 0.25 * category_uncertainty + 0.10 * rating_shift else: # preference score = 0.34 * abs(cluster.get("user_affinity", 1.0) - 1.0) + 0.28 * rating_shift + 0.20 * uncertainty + 0.18 * abs(cluster.get("priority_weight", 1.0) - 1.0) return score * self._get_question_base_weight(question_type, category) * freshness def _select_feedback_question(self, ordered_concept_ids: list, concept_clusters: dict, concept_groups: dict, current_concept_labels: dict, last_rating: int, rating: int, similarity: float, iter_num: int): if not ordered_concept_ids: return None question_specs = self._feedback_question_specs() rating_shift = min(1.0, abs(rating - last_rating) / 4.0) candidates = [] category_candidates = [] for cid in ordered_concept_ids: cluster = concept_clusters.get(cid) if not cluster: continue cluster = self._ensure_concept_cluster_defaults(cluster) concept_clusters[cid] = cluster category = cluster.get("category", "general") mean_imp = self._get_concept_mean_importance(cluster) neighbor_ids = self._get_concept_neighbors(cid, ordered_concept_ids, radius=2) neighbor_means = [ self._get_concept_mean_importance(concept_clusters[nid]) for nid in neighbor_ids if nid in concept_clusters ] neighbor_mean = sum(neighbor_means) / len(neighbor_means) if neighbor_means else 1.0 chosen_group_id = None for gid, group in concept_groups.items(): if cid in group.get("concept_ids", []): chosen_group_id = gid break for question_type in question_specs.keys(): score = self._score_question_type( question_type, cluster, category, mean_imp, neighbor_mean, rating_shift, similarity, iter_num ) candidate = { "concept_id": cid, "concept_label": current_concept_labels.get(cid, cluster.get("label", "")), "question_type": question_type, "neighbor_ids": neighbor_ids, "group_id": chosen_group_id, "category": category, "score": score, } if question_type == "category": category_candidates.append(candidate) else: candidates.append(candidate) if category_candidates: category_candidate = max(category_candidates, key=lambda x: x["score"]) if category_candidate["score"] >= 0.60: return category_candidate if not candidates: return None return max(candidates, key=lambda x: x["score"]) def _format_feedback_question(self, concept_label: str, question_type: str): spec = self._feedback_question_specs().get(question_type, self._feedback_question_specs()["presence"]) return ( f"Concept: '{concept_label}'\n" f"{spec['prompt'].format(label=concept_label)}\n" f"{spec['legend']}" ) # ========================================================================= # MULTI-LEVEL CONCEPT SYSTEM # ========================================================================= def _parse_concepts(self, prompt: str): """ Level 2 → Level 3 boundary: split the prompt on commas/semicolons into concept phrases. Each phrase is returned as a list of significant lowercase words, preserving prompt order. Example: "masterpiece, anime girl, long hair, dark forest" -> [["masterpiece"], ["anime","girl"], ["long","hair"], ["dark","forest"]] """ if not prompt: return [] result = [] for segment_type, segment_text in self._iter_prompt_segments(prompt): if segment_type == "protected": protected_phrase = segment_text.strip().lower() if len(protected_phrase) >= 3 and any(c.isalpha() for c in protected_phrase): result.append([protected_phrase]) continue phrases = [p.strip() for p in re.split(r'[,;]', segment_text) if p.strip()] for phrase in phrases: words = [w.strip().lower() for w in phrase.split() if self._is_valuable_token(w.strip())] if words: result.append(words) return result def _concept_overlap_words(self, prompt: str): words = [] seen = set() for phrase_words in self._parse_concepts(prompt): for word in phrase_words: clean = str(word).strip().lower() if not clean or clean in seen: continue seen.add(clean) words.append(clean) return words def _ordered_prompt_words(self, prompt: str): words = [] for phrase_words in self._parse_concepts(prompt): for word in phrase_words: clean = str(word).strip().lower() if clean and self._is_valuable_token(clean): words.append(clean) return words def _ordered_prompt_phrases(self, prompt: str): phrases = [] if not prompt: return phrases phrase_texts = [] for segment_type, segment_text in self._iter_prompt_segments(prompt): if segment_type == "protected": phrase_texts.append(segment_text) else: phrase_texts.extend(p.strip() for p in re.split(r'[,;]', segment_text) if p.strip()) for phrase_text in phrase_texts: text = re.sub(r"\s+", " ", str(phrase_text or "").strip().lower()).strip(" ,;:.") if not text or len(text) < 3: continue phrase_tokens = [ token.strip().lower() for token in re.findall(r"[\w'’.-]+", text, flags=re.UNICODE) if self._is_valuable_token(token.strip()) ] if not phrase_tokens: continue phrases.append({ "text": text, "tokens": phrase_tokens, }) return phrases def _history_prompt_texts(self, prompt_key: str, history_entry: dict): texts = [] for key in ("canonical_prompt", "last_positive_prompt"): value = history_entry.get(key) if isinstance(value, str) and value.strip(): texts.append(value) if isinstance(prompt_key, str) and prompt_key.strip(): texts.append(prompt_key) for item in reversed(history_entry.get("history", [])[-4:]): value = item.get("prompt") if isinstance(item, dict) else None if isinstance(value, str) and value.strip(): texts.append(value) unique = [] seen = set() for text in texts: normalized = text.strip() if normalized and normalized not in seen: seen.add(normalized) unique.append(normalized) return unique def _prompt_history_concept_words(self, prompt_key: str, history_entry: dict): stored = history_entry.get("prompt_concept_words") if isinstance(stored, list) and stored: return [str(word).lower() for word in stored if str(word).strip()] words = [] seen = set() for text in self._history_prompt_texts(prompt_key, history_entry): for word in self._concept_overlap_words(text): if word not in seen: seen.add(word) words.append(word) return words def _conditioning_similarity(self, reference: torch.Tensor, current: torch.Tensor, token_mask=None): if not isinstance(reference, torch.Tensor) or not isinstance(current, torch.Tensor): return 0.0 if list(reference.shape) != list(current.shape): return 0.0 current = current.to(reference.device) if current.device != reference.device else current try: if reference.dim() >= 2 and current.dim() >= 2: ref_mean = self._masked_sequence_mean(reference, token_mask) cur_mean = self._masked_sequence_mean(current, token_mask) return float(F.cosine_similarity(ref_mean, cur_mean, dim=-1).mean().item()) return float(F.cosine_similarity( reference.flatten().unsqueeze(0), current.flatten().unsqueeze(0), dim=-1 ).mean().item()) except Exception: return 0.0 def _conditioning_history_similarity(self, history_entry: dict, raw_positive: torch.Tensor): source_data = history_entry.get("source_conditioning_embeds") or history_entry.get("reference_embeds") try: source_reference = serializable_to_tensor(source_data) except Exception: return 0.0 return self._conditioning_similarity(source_reference, raw_positive) def _find_prompt_variant_history(self, exact_prompt_key: str, current_words: list, raw_positive: torch.Tensor, prompt_histories: dict): if not exact_prompt_key or not current_words or not prompt_histories: return None, None current_set = set(current_words) if len(current_set) < 2: return None, None best_key = None best_info = None best_score = 0.0 for candidate_key, history_entry in prompt_histories.items(): if candidate_key == exact_prompt_key or not isinstance(history_entry, dict): continue candidate_words = self._prompt_history_concept_words(candidate_key, history_entry) candidate_set = set(candidate_words) if len(candidate_set) < 2: continue shared = current_set & candidate_set shared_count = len(shared) union = current_set | candidate_set overlap = shared_count / max(1, len(union)) coverage = shared_count / max(1, min(len(current_set), len(candidate_set))) semantic_score = max(overlap, coverage * 0.72) conditioning_similarity = self._conditioning_history_similarity(history_entry, raw_positive) if conditioning_similarity < 0.68: continue if shared_count < 3 and not (coverage >= 0.85 and conditioning_similarity >= 0.82): continue if not ( (overlap >= 0.46 and conditioning_similarity >= 0.78) or (coverage >= 0.68 and conditioning_similarity >= 0.72) or (overlap >= 0.60 and conditioning_similarity >= 0.68) ): continue score = semantic_score * 0.58 + conditioning_similarity * 0.42 if score > best_score: best_score = score best_key = candidate_key best_info = { "score": round(float(score), 4), "overlap": round(float(overlap), 4), "coverage": round(float(coverage), 4), "conditioning_similarity": round(float(conditioning_similarity), 4), "shared_words": sorted(shared)[:12], "matched_prompt_key": candidate_key, } return best_key, best_info def _remember_prompt_variant(self, active: dict, exact_prompt_key: str, positive_prompt: str, current_words: list, match_info: dict = None): active.setdefault("canonical_prompt", positive_prompt or exact_prompt_key) existing_words = set( str(word).lower() for word in active.get("prompt_concept_words", []) if str(word).strip() ) for word in current_words: existing_words.add(str(word).lower()) active["prompt_concept_words"] = sorted(existing_words)[:256] variants = list(active.get("prompt_variants", []))[-11:] variant = { "prompt_key": exact_prompt_key, "prompt": (positive_prompt or exact_prompt_key)[:240], } if match_info: variant.update({ "overlap": match_info.get("overlap"), "coverage": match_info.get("coverage"), "conditioning_similarity": match_info.get("conditioning_similarity"), }) variants.append(variant) active["prompt_variants"] = variants def _history_entry_prompt_words(self, entry: dict): if not isinstance(entry, dict): return [] stored = entry.get("prompt_words") if isinstance(stored, list) and stored: return [str(word).lower() for word in stored if str(word).strip()] prompt_text = entry.get("prompt_full") or entry.get("prompt") or "" return self._concept_overlap_words(prompt_text) def _retarget_prompt_history_to_source(self, active: dict, old_source: torch.Tensor, new_source: torch.Tensor): if ( not isinstance(active, dict) or not isinstance(old_source, torch.Tensor) or not isinstance(new_source, torch.Tensor) or list(old_source.shape) != list(new_source.shape) ): return 0 shift = new_source.to(old_source.device) - old_source migrated = 0 def _shift_serialized(field_name: str): nonlocal migrated payload = active.get(field_name) if payload is None: return try: tensor = serializable_to_tensor(payload).to(old_source.device) if list(tensor.shape) != list(old_source.shape): return active[field_name] = tensor_to_serializable(tensor + shift) migrated += 1 except Exception: return _shift_serialized("reference_embeds") _shift_serialized("liked_reference_embeds") for item in active.get("history", []): if not isinstance(item, dict) or item.get("modified_embeds") is None: continue try: tensor = serializable_to_tensor(item.get("modified_embeds")).to(old_source.device) if list(tensor.shape) != list(old_source.shape): continue item["modified_embeds"] = tensor_to_serializable(tensor + shift) item["retargeted_to_prompt_variant"] = True migrated += 1 except Exception: continue return migrated def _nudge_delta_words(self, words: list, delta: float, concept_clusters: dict, word_importance: dict, iter_num: int, role: str): touched = [] for raw_word in words: word = str(raw_word).strip().lower() if not self._is_valuable_token(word): continue cid, is_new = self._match_concept([word], concept_clusters, threshold=0.18) if cid is None: continue if is_new or cid not in concept_clusters: concept_clusters[cid] = self._default_concept_cluster([word]) else: concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid]) if word not in concept_clusters[cid].get("anchor_words", []): concept_clusters[cid]["anchor_words"].append(word) cluster = concept_clusters[cid] local_imp = cluster.setdefault("word_importance", {}) local_imp[word] = max(0.35, min(2.8, float(local_imp.get(word, 1.0)) + delta)) word_importance[word] = max(0.35, min(2.8, float(word_importance.get(word, 1.0)) + delta * 0.45)) cluster["presence_target"] = self._clip_profile_value( float(cluster.get("presence_target", 1.0)) + delta * 0.28 ) cluster["priority_weight"] = self._clip_profile_value( float(cluster.get("priority_weight", 1.0)) + delta * 0.20 ) if delta > 0: cluster["semantic_fidelity"] = self._clip_profile_value( float(cluster.get("semantic_fidelity", 1.0)) + delta * 0.12, low=0.6, high=1.8, ) else: cluster["overrep_sensitivity"] = self._clip_profile_value( float(cluster.get("overrep_sensitivity", 1.0)) + abs(delta) * 0.12 ) cluster["last_seen_iter"] = iter_num cluster["usage_count"] = int(cluster.get("usage_count", 0)) + 1 cluster["last_prompt_delta_role"] = role touched.append(word) return touched def _missing_axis_matches_category(self, axis: str, category: str): category = (category or "general").lower() if axis == "concept": return category in {"subject", "action", "appearance", "character", "concept", "general"} if axis == "details": return category in {"camera", "action", "environment", "appearance", "style", "general"} if axis == "quality": return category in {"quality", "style", "general"} return False def _apply_missing_axis_prompt_pressure(self, word_groups, word_to_concept, rating_profile, concept_clusters, word_importance, iter_num): missing_axes = set(rating_profile.get("missing_axes", [])) if not missing_axes or not word_groups: return "" axis_boosts = {"concept": 0.34, "details": 0.26, "quality": 0.22} touched = [] for _, _, full_word, _ in word_groups: word = str(full_word).strip().lower() if not self._is_valuable_token(word): continue cid = word_to_concept.get(word) category = "general" cluster = None if cid and cid in concept_clusters: concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid]) cluster = concept_clusters[cid] category = cluster.get("category", "general") matched_axes = [axis for axis in missing_axes if self._missing_axis_matches_category(axis, category)] pressure = sum(axis_boosts.get(axis, 0.0) for axis in matched_axes) if not pressure: pressure = sum(axis_boosts.get(axis, 0.0) for axis in missing_axes) * 0.35 if not pressure: continue word_importance[word] = max(0.35, min(2.8, float(word_importance.get(word, 1.0)) + pressure * 0.32)) if cluster is not None: local_imp = cluster.setdefault("word_importance", {}) local_imp[word] = max(0.35, min(2.8, float(local_imp.get(word, 1.0)) + pressure * 0.55)) cluster["missing_count"] = int(cluster.get("missing_count", 0)) + 1 for axis in missing_axes: counter = self._axis_counter_name(axis) if counter: cluster[counter] = int(cluster.get(counter, 0)) + 1 if "concept" in missing_axes or "details" in missing_axes: cluster["presence_target"] = self._clip_profile_value( float(cluster.get("presence_target", 1.0)) + pressure * 0.18 ) cluster["priority_weight"] = self._clip_profile_value( float(cluster.get("priority_weight", 1.0)) + pressure * 0.14 ) if "quality" in missing_axes: cluster["semantic_fidelity"] = self._clip_profile_value( float(cluster.get("semantic_fidelity", 1.0)) + pressure * 0.16, low=0.6, high=1.8, ) cluster["stability_weight"] = self._clip_profile_value( float(cluster.get("stability_weight", 1.0)) + pressure * 0.12 ) cluster["last_missing_iter"] = iter_num touched.append(word) if not touched: return "" axes = "+".join(sorted(missing_axes)) return f"Missing-axis pressure: {axes} reinforced {', '.join(touched[:10])}{'...' if len(touched) > 10 else ''}." def _apply_prompt_delta_attribution(self, history: list, current_words: list, rating_profile: dict, concept_clusters: dict, word_importance: dict, iter_num: int): if len(history) < 2: return "" previous_entry = history[-2] rated_entry = history[-1] previous_words = self._history_entry_prompt_words(previous_entry) rated_words = self._history_entry_prompt_words(rated_entry) or current_words if not previous_words or not rated_words: return "" previous_set = set(previous_words) rated_set = set(rated_words) added = sorted(rated_set - previous_set) removed = sorted(previous_set - rated_set) if not added and not removed: return "" previous_profile = normalize_refiner_rating(rated_entry.get("rating_label", rated_entry.get("rating", 0))) current_key = rating_profile.get("key", "") previous_missing_axes = set(previous_profile.get("missing_axes", [])) current_missing_axes = set(rating_profile.get("missing_axes", [])) word_importance = word_importance if isinstance(word_importance, dict) else {} boosted = [] softened = [] reason = "" if current_key == "like" and previous_missing_axes: boost = 0.22 + (0.10 if "concept" in previous_missing_axes else 0.0) boosted = self._nudge_delta_words( added[:24], boost, concept_clusters, word_importance, iter_num, "added_helped", ) softened = self._nudge_delta_words( removed[:24], -0.08, concept_clusters, word_importance, iter_num, "removed_after_missing", ) reason = f"{previous_profile.get('label', 'missing')} -> {rating_profile.get('label', 'Perfect')}" elif current_missing_axes and previous_profile.get("key") == "like": boost = 0.14 + (0.08 if "concept" in current_missing_axes else 0.0) boosted = self._nudge_delta_words( removed[:24], boost, concept_clusters, word_importance, iter_num, "removed_needed", ) softened = self._nudge_delta_words( added[:24], -0.10, concept_clusters, word_importance, iter_num, "added_hurt", ) reason = f"{previous_profile.get('label', 'Perfect')} -> {rating_profile.get('label', 'missing')}" if not boosted and not softened: return "" parts = [f"Prompt delta: {reason}."] if boosted: parts.append("boosted " + ", ".join(boosted[:8])) if softened: parts.append("softened " + ", ".join(softened[:8])) return " ".join(parts) def _build_prompt_fallback_concept(self, prompt: str, concept_clusters: dict): if not prompt: return None fallback_words = [] for segment_type, segment_text in self._iter_prompt_segments(prompt): if segment_type == "protected": protected_phrase = segment_text.strip().lower() if len(protected_phrase) >= 3 and any(c.isalpha() for c in protected_phrase): fallback_words.append(protected_phrase) continue fallback_words.extend( w.strip().lower() for w in re.split(r'[\s,;]+', segment_text) if self._is_valuable_token(w.strip()) ) if len(fallback_words) >= 8: break fallback_words = fallback_words[:8] if not fallback_words: return None fallback_id = "prompt_" + md5("|".join(fallback_words).encode()).hexdigest()[:10] if fallback_id not in concept_clusters: concept_clusters[fallback_id] = self._default_concept_cluster(fallback_words) concept_clusters[fallback_id]["label"] = " ".join(fallback_words[:5]) else: concept_clusters[fallback_id] = self._ensure_concept_cluster_defaults(concept_clusters[fallback_id]) return fallback_id def _force_feedback_fallback(self, ordered_concept_ids: list, concept_clusters: dict, concept_groups: dict, current_concept_labels: dict, rating_shift: float, similarity: float): if not ordered_concept_ids: return None ranked = [] for cid in ordered_concept_ids: cluster = concept_clusters.get(cid) if not cluster: continue cluster = self._ensure_concept_cluster_defaults(cluster) concept_clusters[cid] = cluster mean_imp = self._get_concept_mean_importance(cluster) dominance = abs(mean_imp - cluster.get("presence_target", 1.0)) ranked.append((dominance, cid, cluster)) if not ranked: return None _, chosen_cid, cluster = max(ranked, key=lambda x: x[0]) if rating_shift >= 2.0: question_type = "fidelity" elif similarity < 0.84: question_type = "stability" elif cluster.get("category") in {"style", "camera", "quality"}: question_type = "balance" else: question_type = "presence" chosen_group_id = None for gid, group in concept_groups.items(): if chosen_cid in group.get("concept_ids", []): chosen_group_id = gid break return { "concept_id": chosen_cid, "concept_label": current_concept_labels.get(chosen_cid, cluster.get("label", "")), "question_type": question_type, "neighbor_ids": self._get_concept_neighbors(chosen_cid, ordered_concept_ids, radius=2), "group_id": chosen_group_id, "category": cluster.get("category", "general"), "score": 0.0, } def _match_concept(self, phrase_words: list, concept_clusters: dict, threshold: float = 0.38): """ Match a phrase to an existing concept cluster using Jaccard similarity on anchor words. If no cluster clears `threshold`, a new cluster id is minted. Returns (cluster_id, is_new_cluster). Tune threshold: lower -> more isolated clusters; higher -> more merging. """ if not phrase_words: return None, False phrase_set = set(phrase_words) best_id, best_score = None, 0.0 for cid, cluster in concept_clusters.items(): anchor_set = set(cluster.get("anchor_words", [])) if not anchor_set: continue union = len(phrase_set | anchor_set) if union == 0: continue jaccard = len(phrase_set & anchor_set) / union if jaccard > best_score: best_score, best_id = jaccard, cid if best_score >= threshold: return best_id, False new_id = md5("|".join(sorted(phrase_words)).encode()).hexdigest()[:10] return new_id, True def _apply_category_feedback(self, cluster: dict, feedback_rating: int, iter_num: int = 0): category = CATEGORY_FEEDBACK_MAP.get(int(feedback_rating), "general") cluster["category"] = category cluster["category_source"] = "user" cluster["category_confidence"] = 1.0 if category in {"concept", "character", "details"}: cluster["presence_target"] = self._clip_profile_value(cluster.get("presence_target", 1.0) + 0.10) cluster["priority_weight"] = self._clip_profile_value(cluster.get("priority_weight", 1.0) + 0.10) cluster["semantic_fidelity"] = self._clip_profile_value( cluster.get("semantic_fidelity", 1.0) + 0.08, low=0.6, high=1.8, ) elif category == "quality": cluster["priority_weight"] = self._clip_profile_value(cluster.get("priority_weight", 1.0) + 0.06) cluster["overrep_sensitivity"] = self._clip_profile_value(cluster.get("overrep_sensitivity", 1.0) + 0.08) elif category == "style": cluster["stability_weight"] = self._clip_profile_value(cluster.get("stability_weight", 1.0) + 0.08) cluster["user_affinity"] = self._clip_profile_value(cluster.get("user_affinity", 1.0) + 0.05) cluster["question_history"] = list(cluster.get("question_history", []))[-23:] cluster["question_history"].append({ "iteration": iter_num, "type": "category", "rating": feedback_rating, "category": category, }) cluster["last_question_type"] = "category" cluster["last_question_iter"] = iter_num def _build_word_concept_map(self, prompt: str, concept_clusters: dict): """ Parse the prompt into concept phrases (Level 3), match or create clusters, and return: - word_to_concept: dict mapping each significant word -> cluster_id - ordered_concept_ids: list of cluster IDs in prompt order (no duplicates) Updates concept_clusters in-place. The ordered list is the backbone for group building and neighbour lookup at Level 4. """ word_to_concept = {} ordered_concept_ids = [] current_concept_labels = {} for phrase_words in self._parse_concepts(prompt): cid, is_new = self._match_concept(phrase_words, concept_clusters) if cid is None: continue phrase_label = " ".join(phrase_words[:6]) if is_new: concept_clusters[cid] = self._default_concept_cluster(phrase_words) else: concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid]) # Expand anchor vocabulary with words newly seen in this phrase existing = set(concept_clusters[cid]["anchor_words"]) for w in phrase_words: if w not in existing: concept_clusters[cid]["anchor_words"].append(w) if ( concept_clusters[cid].get("category_source") != "user" and concept_clusters[cid].get("category") == "general" ): concept_clusters[cid]["category"] = self._infer_concept_category( concept_clusters[cid]["anchor_words"] ) concept_clusters[cid]["category_confidence"] = ( 0.35 if concept_clusters[cid]["category"] == "general" else 0.65 ) concept_clusters[cid]["last_prompt_label"] = phrase_label current_concept_labels[cid] = phrase_label for w in phrase_words: word_to_concept[w] = cid if cid not in ordered_concept_ids: ordered_concept_ids.append(cid) return word_to_concept, ordered_concept_ids, current_concept_labels def _build_concept_groups(self, ordered_concept_ids: list, concept_clusters: dict, existing_groups: dict, current_concept_labels: dict, window: int = 3): """ Level 4: group consecutive concept phrases into semantic sentence-level units using a non-overlapping sliding window of size `window`. Groups model the "sentences" of the prompt — natural semantic blocks like (quality tags), (subject + appearance), (setting), (style + tech). Existing groups are matched by their concept_ids set and updated in-place to preserve their accumulated reward_ema and usage_count. New groups are created when a previously unseen concept combination appears. Returns the updated groups dict. """ if not ordered_concept_ids: return existing_groups groups = dict(existing_groups) for i in range(0, len(ordered_concept_ids), window): chunk = ordered_concept_ids[i:i + window] if not chunk: continue chunk_set = frozenset(chunk) # Try to match an existing group by concept set identity existing_gid = None for gid, g in groups.items(): if frozenset(g.get("concept_ids", [])) == chunk_set: existing_gid = gid break if existing_gid: # Preserve the canonical prompt order even if it shifted slightly groups[existing_gid]["concept_ids"] = chunk groups[existing_gid]["label"] = " | ".join( current_concept_labels.get(cid, concept_clusters.get(cid, {}).get("label", cid)) for cid in chunk ) else: labels = [current_concept_labels.get(cid, concept_clusters[cid]["label"]) for cid in chunk if cid in concept_clusters] gid = md5("|".join(chunk).encode()).hexdigest()[:10] groups[gid] = { "label": " | ".join(labels), "concept_ids": chunk, "reward_ema": 0.0, "usage_count": 0, "last_seen_iter": 0, } return groups def _get_concept_neighbors(self, concept_id: str, ordered_concept_ids: list, radius: int = 2): """ Return the concept IDs positionally adjacent to `concept_id` within `radius` steps in the prompt's ordered concept list. Adjacent concepts compete for embedding space: if "anime style" is overrepresented, it likely crowds its neighbours "cinematic lighting" and "detailed background". This adjacency is what makes the neighbour signal in _apply_concept_feedback meaningful. """ if concept_id not in ordered_concept_ids: return [] idx = ordered_concept_ids.index(concept_id) neighbors = [] for offset in range(-radius, radius + 1): if offset == 0: continue ni = idx + offset if 0 <= ni < len(ordered_concept_ids): neighbors.append(ordered_concept_ids[ni]) return neighbors def _get_dominant_concept(self, ordered_concept_ids: list, concept_clusters: dict): """ Return (cluster_id, avg_importance, label) for the concept with the highest mean word importance — i.e. what the embedding currently weighs most heavily. This is the "main thing forming the video" signal. """ best_cid, best_score, best_label = None, -1.0, "" for cid in ordered_concept_ids: if cid not in concept_clusters: continue imp_vals = list(concept_clusters[cid]["word_importance"].values()) if not imp_vals: continue avg = sum(imp_vals) / len(imp_vals) if avg > best_score: best_score, best_cid, best_label = avg, cid, concept_clusters[cid]["label"] return best_cid, best_score, best_label def _apply_concept_feedback(self, concept_id: str, feedback_rating: int, question_type: str, concept_clusters: dict, neighbor_ids: list, word_importance: dict, concept_groups: dict, iter_num: int = 0): """ Multi-level feedback propagation. Level 3 — rated concept phrase: All word importances inside the rated cluster shift by `direct_delta`. This is the primary, high-confidence signal. Level 3 — neighbour concepts (adjacent phrases in the prompt): Receive a dampened signal modelled on embedding-space competition. - Concept being boosted (absent/weak): neighbours get a small inhibitory nudge, giving the boosted concept more semantic room. - Concept being reduced (overrepresented): neighbours also receive a mild reduction — the whole semantic area is too heavy. Magnitude: 22% of direct_delta; same sign for overrep, opposite for absent. Level 4 — concept groups: The reward_ema of every group containing the rated concept is updated, enabling group-level health tracking over time. Level 2 — global word_importance fallback: Each change is also written at 40% strength into the flat global dict so the scheduler and prodigy system retain a valid signal. Feedback scale: 1 = absent -> direct_delta = +0.90 (strong boost) 2 = weak -> direct_delta = +0.50 3 = slightly weak -> direct_delta = +0.20 4 = slightly strong -> direct_delta = -0.15 5 = perfect -> direct_delta = +0.04 (small stability reward) 6 = overrepresented -> direct_delta = -0.55 (strong reduction) """ if concept_id not in concept_clusters: return cluster = self._ensure_concept_cluster_defaults(concept_clusters[concept_id]) concept_clusters[concept_id] = cluster question_type = question_type or "presence" if question_type == "category": self._apply_category_feedback(cluster, feedback_rating, iter_num) return direct_deltas = { "presence": {1: 0.90, 2: 0.50, 3: 0.20, 4: -0.15, 5: 0.04, 6: -0.55}, "priority": {1: -0.25, 2: -0.12, 3: -0.05, 4: 0.08, 5: 0.18, 6: 0.30}, "balance": {1: 0.72, 2: 0.40, 3: 0.16, 4: -0.10, 5: 0.00, 6: -0.62}, "fidelity": {1: 0.25, 2: 0.16, 3: 0.08, 4: 0.02, 5: 0.00, 6: -0.06}, "stability": {1: 0.10, 2: 0.06, 3: 0.03, 4: 0.00, 5: -0.02, 6: -0.06}, "preference": {1: -0.30, 2: -0.18, 3: -0.08, 4: 0.10, 5: 0.22, 6: 0.36}, } direct_delta = direct_deltas.get(question_type, direct_deltas["presence"]).get(feedback_rating, 0.0) centered = (feedback_rating - 3.5) / 2.5 # --- Level 3: rated concept --- local_imp = cluster["word_importance"] for wkey in list(local_imp.keys()): local_imp[wkey] = max(0.3, min(2.8, local_imp[wkey] + direct_delta)) if wkey in word_importance: word_importance[wkey] = max(0.3, min(2.8, word_importance[wkey] + direct_delta * 0.4)) if question_type == "presence": cluster["presence_target"] = self._clip_profile_value( cluster.get("presence_target", 1.0) + direct_delta * 0.22 ) elif question_type == "priority": cluster["priority_weight"] = self._clip_profile_value( cluster.get("priority_weight", 1.0) + centered * 0.18 ) elif question_type == "balance": cluster["overrep_sensitivity"] = self._clip_profile_value( cluster.get("overrep_sensitivity", 1.0) + max(0.0, centered) * 0.24 - max(0.0, -centered) * 0.12 ) cluster["presence_target"] = self._clip_profile_value( cluster.get("presence_target", 1.0) + direct_delta * 0.12 ) elif question_type == "fidelity": cluster["semantic_fidelity"] = self._clip_profile_value( cluster.get("semantic_fidelity", 1.0) + (centered * 0.16), low=0.6, high=1.8 ) elif question_type == "stability": cluster["stability_weight"] = self._clip_profile_value( cluster.get("stability_weight", 1.0) + centered * 0.18 ) elif question_type == "preference": cluster["user_affinity"] = self._clip_profile_value( cluster.get("user_affinity", 1.0) + centered * 0.20 ) cluster["priority_weight"] = self._clip_profile_value( cluster.get("priority_weight", 1.0) + centered * 0.08 ) cluster["presence_target"] = self._clip_profile_value( cluster.get("presence_target", 1.0) + centered * 0.10 ) # --- Level 3: neighbour concepts --- # Boosted concept -> inhibit neighbours (give it semantic space). # Reduced concept -> also soften neighbours (the whole area is too heavy). if direct_delta > 0: neighbor_delta = -direct_delta * 0.22 else: neighbor_delta = direct_delta * 0.22 for nid in neighbor_ids: if nid not in concept_clusters or nid == concept_id: continue concept_clusters[nid] = self._ensure_concept_cluster_defaults(concept_clusters[nid]) n_local = concept_clusters[nid]["word_importance"] for wkey in list(n_local.keys()): n_local[wkey] = max(0.3, min(2.8, n_local[wkey] + neighbor_delta)) if wkey in word_importance: word_importance[wkey] = max(0.3, min(2.8, word_importance[wkey] + neighbor_delta * 0.4)) # --- Level 4: concept group reward EMA --- # Maps feedback 1-6 onto a [-1, +1] reward signal. group_reward_map = {1: -1.0, 2: -0.5, 3: -0.1, 4: 0.4, 5: 1.0, 6: -0.6} normalized = group_reward_map.get(feedback_rating, 0.0) for gid, g in concept_groups.items(): if concept_id in g.get("concept_ids", []): g["reward_ema"] = 0.75 * g.get("reward_ema", 0.0) + 0.25 * normalized g["usage_count"] = g.get("usage_count", 0) + 1 cluster["question_history"] = list(cluster.get("question_history", []))[-23:] cluster["question_history"].append({ "iteration": iter_num, "type": question_type, "rating": feedback_rating, }) cluster["last_question_type"] = question_type cluster["last_question_iter"] = iter_num # ========================================================================= # SIGMA REFINEMENT # ========================================================================= def _ensure_sigma_state_defaults(self, global_adaptive: dict): global_adaptive.setdefault("sigma_profile", [0.0] * 32) global_adaptive.setdefault("last_applied_sigma_profile", [0.0] * 32) global_adaptive.setdefault("sigma_iterations", 0) global_adaptive.setdefault("sigma_avg_reward_ema", 0.0) global_adaptive.setdefault("sigma_exploration_base", 0.035) global_adaptive.setdefault("sigma_history", []) def _resolve_sigma_strength(self, strength_mode: str): strength_map = { "off": 0.0, "subtle": 0.18, "medium": 0.35, "strong": 0.60, "max": 1.0, } return strength_map.get((strength_mode or "subtle").lower(), 0.18) def _sigma_resample_profile(self, profile, target_len): if target_len <= 0: return np.zeros((0,), dtype=np.float32) arr = np.asarray(profile, dtype=np.float32) if arr.size == 0: arr = np.zeros((32,), dtype=np.float32) if arr.size == target_len: return arr.copy() if arr.size == 1: return np.full((target_len,), float(arr[0]), dtype=np.float32) src = np.linspace(0.0, 1.0, arr.size) dst = np.linspace(0.0, 1.0, target_len) return np.interp(dst, src, arr).astype(np.float32) def _sigma_smooth_noise(self, noise): if noise.size <= 2: return noise kernel = np.array([0.25, 0.5, 0.25], dtype=np.float32) smoothed = np.convolve(noise, kernel, mode="same") smoothed = np.convolve(smoothed, kernel, mode="same") return smoothed.astype(np.float32) def _sigma_enforce_monotonic(self, tuned: torch.Tensor, original: torch.Tensor): out = tuned.clone() count = int(out.shape[0]) if count <= 2: return out eps = max(1e-6, float((original[0] - original[-1]).abs().item()) * 1e-6) for i in range(1, count - 1): max_allowed = out[i - 1] - eps if out[i] >= max_allowed: out[i] = max_allowed out[-1] = original[-1] for i in range(count - 2, 0, -1): min_allowed = out[i + 1] + eps if out[i] <= min_allowed: out[i] = min_allowed out[0] = original[0] out[-1] = original[-1] return out def _refine_sigma_schedule(self, sigmas, rating: int, global_adaptive: dict, strength_mode: str, seed: int, rating_profile: Optional[dict] = None): if not isinstance(sigmas, torch.Tensor): return torch.FloatTensor([]), "Sigma refinement inactive." original_sigmas = sigmas.detach().clone() if original_sigmas.numel() <= 2: return original_sigmas, "Sigma refinement skipped: schedule too short." self._ensure_sigma_state_defaults(global_adaptive) rating_profile = rating_profile or normalize_refiner_rating(rating) rating_key = rating_profile.get("key", "") reward = float(rating_profile.get("reward", (rating - 5.5) / 4.5)) sigma_iterations = int(global_adaptive.get("sigma_iterations", 0)) profile = np.asarray(global_adaptive.get("sigma_profile", [0.0] * 32), dtype=np.float32) last_applied = np.asarray(global_adaptive.get("last_applied_sigma_profile", [0.0] * 32), dtype=np.float32) sigma_lr = 0.22 / (1.0 + 0.08 * math.sqrt(max(1, sigma_iterations))) profile = np.clip(profile + (last_applied * reward * sigma_lr), -1.0, 1.0) profile = np.clip(profile - profile.mean(), -1.0, 1.0) sigma_avg_reward_ema = float(global_adaptive.get("sigma_avg_reward_ema", 0.0)) sigma_avg_reward_ema = 0.85 * sigma_avg_reward_ema + 0.15 * reward sigma_exploration_base = float(global_adaptive.get("sigma_exploration_base", 0.035)) sigma_exploration_scale = max(0.18, 1.0 - max(0.0, sigma_avg_reward_ema) * 0.7) if seed != 0: np.random.seed(seed % (2 ** 32)) noise = np.random.normal( 0.0, sigma_exploration_base * sigma_exploration_scale, size=profile.shape[0] ).astype(np.float32) applied_profile = np.clip(profile + self._sigma_smooth_noise(noise), -1.0, 1.0) applied_profile = np.clip(applied_profile - applied_profile.mean(), -1.0, 1.0) middle_profile = self._sigma_resample_profile(applied_profile, max(0, original_sigmas.shape[0] - 2)) middle_profile_tensor = torch.tensor(middle_profile, dtype=original_sigmas.dtype, device=original_sigmas.device) tuned_sigmas = original_sigmas.clone() sigma_strength = self._resolve_sigma_strength(strength_mode) for idx in range(1, int(original_sigmas.shape[0]) - 1): delta = float(middle_profile_tensor[idx - 1].item()) current = original_sigmas[idx] prev_sigma = original_sigmas[idx - 1] next_sigma = original_sigmas[idx + 1] if delta >= 0.0: tuned_sigmas[idx] = current + (prev_sigma - current) * sigma_strength * delta else: tuned_sigmas[idx] = current + (current - next_sigma) * sigma_strength * delta tuned_sigmas = self._sigma_enforce_monotonic(tuned_sigmas, original_sigmas) global_adaptive["sigma_profile"] = profile.tolist() global_adaptive["last_applied_sigma_profile"] = applied_profile.tolist() global_adaptive["sigma_iterations"] = sigma_iterations + 1 global_adaptive["sigma_avg_reward_ema"] = sigma_avg_reward_ema global_adaptive["sigma_exploration_base"] = max( 0.012, min(0.05, sigma_exploration_base * (0.97 if rating_key == "like" else 1.02)) ) sigma_history = list(global_adaptive.get("sigma_history", []))[-119:] sigma_history.append({ "iteration": sigma_iterations + 1, "rating": int(rating), "rating_label": rating_profile.get("label", str(rating)), "reward": round(float(reward), 6), "lr": round(float(sigma_lr), 6), }) global_adaptive["sigma_history"] = sigma_history mean_shift = float((tuned_sigmas[1:-1] - original_sigmas[1:-1]).abs().mean().item()) if tuned_sigmas.numel() > 2 else 0.0 max_shift = float((tuned_sigmas[1:-1] - original_sigmas[1:-1]).abs().max().item()) if tuned_sigmas.numel() > 2 else 0.0 sigma_status = ( f"Sigma: iter {global_adaptive['sigma_iterations']} | strength {strength_mode} ({sigma_strength:.2f}) | " f"mean shift {mean_shift:.6f} | max shift {max_shift:.6f} | " f"endpoints preserved ({float(original_sigmas[0].item()):.6f} -> {float(original_sigmas[-1].item()):.6f})" ) return tuned_sigmas, sigma_status # ========================================================================= # LATENT REFINEMENT # ========================================================================= def _resize_tensor_like(self, tensor: torch.Tensor, reference: torch.Tensor): if not isinstance(tensor, torch.Tensor) or not isinstance(reference, torch.Tensor): return None out = tensor.to(device=reference.device, dtype=reference.dtype) if list(out.shape) == list(reference.shape): return out if out.dim() != reference.dim() or out.dim() not in {4, 5}: return None batch = reference.shape[0] if out.shape[0] < batch: reps = [1] * out.dim() reps[0] = math.ceil(batch / max(1, out.shape[0])) out = out.repeat(*reps) out = out[:batch] channels = reference.shape[1] if out.shape[1] < channels: pad_shape = list(out.shape) pad_shape[1] = channels - out.shape[1] out = torch.cat([out, torch.zeros(pad_shape, device=out.device, dtype=out.dtype)], dim=1) out = out[:, :channels] if out.dim() == 4: return F.interpolate(out, size=reference.shape[-2:], mode="bilinear", align_corners=False) return F.interpolate(out, size=reference.shape[-3:], mode="trilinear", align_corners=False) def _load_saved_latent(self, refinement_key, mode): path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt") if not os.path.exists(path): return None, "Latent: no saved latent for this key." try: data = torch.load(path, map_location="cpu", weights_only=False) except Exception as e: return None, f"Latent: failed to load saved latent ({e})." if not isinstance(data, dict) or not isinstance(data.get("samples"), torch.Tensor): return None, "Latent: saved latent file is invalid." return data, "Latent: saved latent loaded." def _value_links_to_output(self, value, source_id, output_index): if isinstance(value, (list, tuple)): if len(value) >= 2 and str(value[0]) == source_id: try: if int(value[1]) == output_index: return True except (TypeError, ValueError): pass return any(self._value_links_to_output(item, source_id, output_index) for item in value) if isinstance(value, dict): return any(self._value_links_to_output(item, source_id, output_index) for item in value.values()) return False def _is_output_connected(self, prompt, unique_id, output_index): if prompt is None or unique_id is None: return False source_id = str(unique_id) prompt_nodes = prompt.get("output", prompt) if isinstance(prompt, dict) else {} if not isinstance(prompt_nodes, dict): return False for node in prompt_nodes.values(): if not isinstance(node, dict): continue inputs = node.get("inputs", {}) if isinstance(inputs, dict) and self._value_links_to_output(inputs, source_id, output_index): return True workflow = prompt.get("workflow") if isinstance(prompt, dict) else None links = workflow.get("links", []) if isinstance(workflow, dict) else [] for link in links: if isinstance(link, (list, tuple)) and len(link) >= 3: try: if str(link[1]) == source_id and int(link[2]) == output_index: return True except (TypeError, ValueError): continue elif isinstance(link, dict): try: if str(link.get("origin_id")) == source_id and int(link.get("origin_slot")) == output_index: return True except (TypeError, ValueError): continue return False def _latent_refinement_disabled(self, latent): return clone_latent(latent), "Latent: disabled because latent refinement input/output path is not fully connected." def _raise_wrong_latent(self, latent): sample_type = latent_sample_type_name(latent) latent_type = latent.get("type", "unspecified") if isinstance(latent, dict) else "missing" raise ValueError(f"{self.WRONG_LATENT_ERROR} Received samples={sample_type}, type={latent_type}.") def _save_latent_reference(self, latent, refinement_key, mode): samples = latent_samples(latent) if samples is None: return False path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt") os.makedirs(os.path.dirname(path), exist_ok=True) bundle = cpu_tensor_bundle(latent) bundle["_meta"] = { "refinement_key": refinement_key, "mode": (mode or "ltx2").lower(), "samples_shape": list(samples.shape), } torch.save(bundle, path) return True def _delete_latent_reference(self, refinement_key, mode): path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt") try: os.remove(path) return True except FileNotFoundError: return False except OSError: return False def _refine_latent(self, latent, refinement_key, mode, rating, reward, global_adaptive, rating_profile: Optional[dict] = None): if isinstance(latent, dict) and not latent_is_plain_video_tensor(latent): self._raise_wrong_latent(latent) current = latent_samples(latent) saved_bundle, load_status = self._load_saved_latent(refinement_key, mode) if current is None: if saved_bundle is None: raise ValueError(self.NO_LATENT_REFERENCE_ERROR) saved_latent = latent_from_tensor_bundle(saved_bundle) saved_samples = latent_samples(saved_latent) if saved_samples is None: raise ValueError(self.NO_LATENT_REFERENCE_ERROR) latent_state = global_adaptive.setdefault("latent_refinement", {}) latent_state["last_shape"] = list(saved_samples.shape) latent_state["saved_shape"] = list(saved_samples.shape) latent_state["status"] = self.SAVED_LATENT_ONLY_STATUS return saved_latent, latent_state["status"] refined_latent = clone_latent(latent) if saved_bundle is None: self._save_latent_reference(latent, refinement_key, mode) global_adaptive.setdefault("latent_refinement", {})["last_shape"] = list(current.shape) return refined_latent, f"{load_status} Current latent saved as reference." raw_saved_samples = saved_bundle.get("samples") if list(raw_saved_samples.shape) != list(current.shape): self._save_latent_reference(latent, refinement_key, mode) latent_state = global_adaptive.setdefault("latent_refinement", {}) latent_state.pop("momentum", None) latent_state["last_shape"] = list(current.shape) latent_state["saved_shape"] = list(raw_saved_samples.shape) return refined_latent, "Latent: input shape changed. Reference rewritten, passthrough." saved_samples = raw_saved_samples.to(device=current.device, dtype=current.dtype) latent_state = global_adaptive.setdefault("latent_refinement", {}) latent_state["last_shape"] = list(current.shape) latent_state["saved_shape"] = list(raw_saved_samples.shape) momentum_data = latent_state.get("momentum") if isinstance(momentum_data, dict): try: momentum = serializable_to_tensor(momentum_data).to(device=current.device, dtype=current.dtype) momentum = self._resize_tensor_like(momentum, current) except Exception: momentum = None else: momentum = None if momentum is None: momentum = torch.zeros_like(current) nonzero_mask = current.ne(0) & saved_samples.ne(0) if not bool(nonzero_mask.any()): refined_latent["samples"] = current latent_state["status"] = "Latent: only zero-valued positions available, passthrough." return refined_latent, latent_state["status"] rating_profile = rating_profile or normalize_refiner_rating(rating) rating_key = rating_profile.get("key", "") missing_axes = set(rating_profile.get("missing_axes", [])) if rating_key == "like": lr = 0.035 elif missing_axes: lr = 0.018 if "details" in missing_axes: lr += 0.006 if "concept" in missing_axes: lr += 0.012 if "quality" in missing_axes: lr += 0.035 lr = min(0.065, lr) elif rating_key == "dislike": lr = 0.055 else: lr = 0.018 target_delta = (saved_samples - current) * nonzero_mask.to(dtype=current.dtype) update = target_delta * (reward * lr) momentum = 0.82 * momentum + 0.18 * update max_step = max(0.001, min(0.08, float(current.detach().abs().mean().item()) * 0.18)) latent_delta = torch.clamp(momentum, min=-max_step, max=max_step) refined_samples = current + latent_delta refined_samples = torch.where(nonzero_mask, refined_samples, current) refined_samples = torch.nan_to_num(refined_samples, nan=0.0) range_min = torch.minimum(current, saved_samples) range_max = torch.maximum(current, saved_samples) refined_samples = torch.maximum(torch.minimum(refined_samples, range_max), range_min) refined_latent["samples"] = refined_samples latent_state["momentum"] = tensor_to_serializable(momentum.detach().cpu()) latent_state["last_rating"] = int(rating) latent_state["last_rating_label"] = rating_profile.get("label", str(rating)) latent_state["last_reward"] = round(float(reward), 4) latent_state["nonzero_ratio"] = round(float(nonzero_mask.float().mean().item()), 6) latent_state["status"] = ( f"Latent: adjusted shape {tuple(current.shape)} | nonzero {latent_state['nonzero_ratio']:.1%} | " f"max step {max_step:.5f} | bounded to input/reference range" ) return refined_latent, latent_state["status"] # ========================================================================= # LORA WEIGHT SUGGESTIONS # ========================================================================= def _lora_state_id(self, lora_name: str, lora_type: str): return md5(f"{lora_name}::{lora_type}".encode("utf-8")).hexdigest()[:16] FunPackGemmaEmbeddingRefiner = FunPackVideoRefiner # A vision block leaves exactly one tag spare (the tags come from the tokenizer's INPUT # sequence, the conditioning is what came back). More than a couple means the two describe # different sequences, not a rounding difference. _TAG_SURPLUS_IS_ROUTINE = 2 def _tag_composition(meta, count): """(text positions, image positions) among the first `count` tags. Best-effort.""" try: tags = meta.get("minimax_token_tags") flat = tags.reshape(-1)[:count] vision = int((flat == 0).sum().item()) return int(flat.shape[0]) - vision, vision except Exception: # noqa: BLE001 return -1, -1 def _h3_reconcile_token_tags(conditioning, label=""): """Keep MiniMax H3's `minimax_token_tags` aligned with the conditioning it rides on. The DiT walks the tag vector position-by-position, so a conditioning whose token count no longer matches its tags is an IndexError on the FIRST forward — reported as a bare "list index out of range" with nothing pointing at the prompt or the conditioning. Studio edits conditioning in many places, and H3 tokenizes reference images INTO the same span (ref_image_size "match" scales them to the output resolution), so the count moves when either the prompt or the resolution changes — which is what made this look intermittent. Grow the tags when tokens were appended, trim them when there are more tags than tokens. Both keep position i meaning position i, which is all the DiT asks of them. Trimming matters most on image prompts: the tags are built from the tokenizer's INPUT sequence and the conditioning is what came back, so a vision block normally leaves one tag spare — and dropping the whole vector there would throw away the modality-0 marks that tell the DiT those positions are an image, not text. Say so either way. Non-H3 conditioning has no tags and passes straight through. """ if not isinstance(conditioning, list): return conditioning try: from .minimax_h3 import extend_token_tags, tags_match, token_tags_length, trim_token_tags except ImportError: # loaded as a top-level module (tests, direct import) from minimax_h3 import extend_token_tags, tags_match, token_tags_length, trim_token_tags out = [] for entry in conditioning: if not (isinstance(entry, (list, tuple)) and len(entry) >= 2 and isinstance(entry[1], dict) and hasattr(entry[0], "shape")): out.append(entry) continue cond, meta = entry[0], entry[1] if tags_match(cond, meta): out.append(entry) continue have = token_tags_length(meta) want = int(cond.shape[1]) if want > have: meta = extend_token_tags(meta, want - have) print(f"[FunPackStudio] MiniMax H3: {label} conditioning grew " f"{have} -> {want} tokens; extended minimax_token_tags to match.") else: meta = trim_token_tags(meta, want) note = (f"[FunPackStudio] MiniMax H3: {label} conditioning is {want} tokens but " f"minimax_token_tags describes {have} — trimmed the {have - want} spare " f"tag(s) off the tail. Modality marks for the kept positions are " f"unchanged.") # A vision block leaves ONE tag spare. A surplus larger than that is not the # routine case this trim was written for: the tags describe a sequence with room # for a picture and the conditioning does not, which means the reference's # embeddings are not in this tensor. Trimming the tail then leaves the kept marks # claiming the PROMPT positions are an image. Report what survived, because # "trimmed N tags" alone reads like housekeeping. if have - want > _TAG_SURPLUS_IS_ROUTINE: text_n, vision_n = _tag_composition(meta, want) note += (f"\n[FunPackStudio] That surplus is far past the one spare tag a " f"vision block leaves. The tags describe {have} positions and the " f"conditioning holds only the first {want} — of which {text_n} are " f"marked text and {vision_n} image. A leading run that is almost " f"all image is the REFERENCE block; the {have - want} positions " f"trimmed away are the encoded PROMPT, and they are not in this " f"tensor. The prompt has been cut, not the tags.") print(note) out.append([cond, meta] + list(entry[2:])) return out # Movie Editor sends this when the user did not rate before regenerating: apply session # memory / repairs but do not learn from a synthetic rating (unlike "-Just forget it-"). # Must appear in V2_RATING_LABELS so ComfyUI /prompt validation accepts editor overrides. MOVIE_EDITOR_CONTINUE_RATING = "__funpack_continue__" # Movie Editor sends this when the prompt changed since the last queue: keep refinement # training/history but drop stale active-repair axes so old "missing action" repairs # do not override the new prompt. No session reset. MOVIE_EDITOR_FRESH_PROMPT_RATING = "__funpack_fresh_prompt__" _BASE_RATING_LABELS = [ "-Just forget it-", "Disliked: bad image", "Disliked: bad composition", "Perfect", "Nailed it", "Missing details", "Missing action", "Missing quality", "Missing details + action", "Wrong details", "Wrong action", "Wrong action + quality", "Wrong details + action", "Wrong appearance", "Missing details + quality", "Missing action + quality", "Awful", ] _NO_LOVED_LABELS = { "-Just forget it-", "Perfect", "Awful", "Missing quality", "Disliked: bad image", "Disliked: bad composition", "Missing details + quality", "Missing action + quality", "Wrong action + quality", MOVIE_EDITOR_CONTINUE_RATING, MOVIE_EDITOR_FRESH_PROMPT_RATING, } V2_RATING_LABELS = _BASE_RATING_LABELS + [ l + "|loved" for l in _BASE_RATING_LABELS if l not in _NO_LOVED_LABELS ] + [MOVIE_EDITOR_CONTINUE_RATING, MOVIE_EDITOR_FRESH_PROMPT_RATING] + [str(n) for n in range(1, 11)] # The "rating" node input is a COMBO the graph builder validates against this exact list # (movie_editor/backend/builder.py's _live_value) -- a value that isn't in it gets silently # swapped for the node's declared default ("Missing action") before Python ever sees it, the # same protection that stops a saved-but-uninstalled LoRA from queuing a broken prompt. The # H3 scale sends bare digit strings, so without them listed here every H3 rating was silently # discarded and replaced with the same "Missing action" (+0.05) regardless of what was picked. V2_RATING_PROFILES = { "-Just forget it-": {"key": "forget", "reward": 0.0, "level": 0, "missing_axes": [], "skip_learning": True}, MOVIE_EDITOR_CONTINUE_RATING: { "key": "continue", "reward": 0.0, "level": 4, "missing_axes": [], "skip_learning": True, "label": "Continue refining", }, MOVIE_EDITOR_FRESH_PROMPT_RATING: { "key": "fresh_prompt", "reward": 0.0, "level": 4, "missing_axes": [], "skip_learning": True, "label": "Fresh prompt (keep training)", }, "Initial discovery": {"key": "discover", "reward": 0.0, "level": 4, "missing_axes": []}, "Perfect": {"key": "like", "reward": 1.0, "level": 8, "missing_axes": []}, "Nailed it": {"key": "nailed_it", "reward": 0.75, "level": 7, "missing_axes": []}, "Loved it": {"key": "loved_it", "reward": 0.85, "level": 7, "missing_axes": []}, "Missing details": {"key": "missing_details", "reward": 0.35, "level": 6, "missing_axes": ["details"]}, "Missing action": {"key": "missing_action", "reward": 0.05, "level": 5, "missing_axes": ["action"]}, "Missing quality": {"key": "missing_quality", "reward": -0.30, "level": 4, "missing_axes": ["quality"]}, "Missing details + action": {"key": "missing_details_action", "reward": -0.10, "level": 3, "missing_axes": ["details", "action"]}, # Wrong-* are quality-neutral signals ("good gen, but the words/identity were off"), not # quality rewards. `skip_value_function` keeps them out of value-function + Absolute-taste # reward training (where a 0.0/low reward on a visually-good gen poisons the learned quality # landscape and the ascent direction → drift/wrong-character). They STILL drive the relative # direction/category memory via their missing_axes/wrong_axes. "Wrong details": {"key": "wrong_details", "reward": 0.20, "level": 5, "missing_axes": ["details"], "wrong_axes": ["details"], "skip_value_function": True}, "Wrong action": {"key": "wrong_action", "reward": -0.10, "level": 4, "missing_axes": ["action"], "wrong_axes": ["action"], "skip_value_function": True}, "Wrong action + quality": {"key": "wrong_action_quality", "reward": -0.40, "level": 3, "missing_axes": ["quality"], "wrong_axes": ["action"], "skip_value_function": True}, "Wrong details + action": {"key": "wrong_details_action", "reward": 0.00, "level": 3, "missing_axes": ["details", "action"], "wrong_axes": ["details", "action"], "skip_value_function": True}, # "Wrong appearance" = character consistency wasn't preserved. Drives the consistency anchor # (_v2_update_appearance_anchor / _v2_apply_conditioning_memory_impl): pull the next gen toward # the last good-rated conditioning for this key and repel from the rejected drift. Not a reward # (skip_value_function) — it's a conditioning nudge, not a quality score. "Wrong appearance": { "key": "wrong_appearance", "reward": 0.0, "level": 4, "missing_axes": [], "skip_value_function": True, }, "Missing details + quality": {"key": "missing_details_quality", "reward": -0.40, "level": 2, "missing_axes": ["details", "quality"]}, "Missing action + quality": {"key": "missing_action_quality", "reward": -0.55, "level": 1, "missing_axes": ["action", "quality"]}, "Awful": {"key": "awful", "reward": -0.90, "level": 0, "missing_axes": ["details", "action", "quality"]}, # H3 follow-ups to Disliked: same key/reward as Awful, so every other consumer treats them # as Awful; `axis` only tells the rated-dial learners which half was at fault. "Disliked: bad image": {"key": "awful", "reward": -0.90, "level": 0, "axis": "image", "missing_axes": ["details", "action", "quality"]}, "Disliked: bad composition": {"key": "awful", "reward": -0.90, "level": 0, "axis": "composition", "missing_axes": ["details", "action", "quality"]}, } V2_FEEDBACK_AXES = ("details", "action", "quality") V2_ADVISOR_MODES = ["Off", "Only diagnostics", "Only prompt", "Full"] # outcome key -> reward, for the path planner's Beta posteriors (liked vs disliked rate per arm). _V2_KEY_REWARD = { str(profile["key"]): float(profile.get("reward", 0.0)) for profile in V2_RATING_PROFILES.values() if isinstance(profile, dict) and profile.get("key") } # Liked / disliked reward thresholds — shared with the Absolute store (_v2_learn_absolute) so the # planner and the global-taste prior agree on what counts as "more of this" vs "not this". V2_PATH_LIKED_REWARD = 0.3 V2_PATH_DISLIKED_REWARD = 0.0 V2_PROMPT_ADVISOR_SYSTEM_PROMPT = """You are a video prompt editor. Rewrite the prompt provided in the user message to fix the specific issue described. Rules: - Output only the corrected prompt text - no explanations, labels, or extra commentary - Use comma-separated concrete visual descriptors, actions, and camera terms - No metaphors, no prose, no emotional or poetic language - Every phrase must describe something a camera can directly capture: motion, appearance, lighting, objects, actions""" #: The prompt enhancer's default instructions. Separate from the repair advisor above: that #: one is triggered by a rating and fixes a named fault, this one runs on every generation and #: only elaborates. Kept short so a user editing it can actually read it. # Filler that says nothing about which shortcut a prompt means. Without this, every shortcut # containing "the" or "with" would match every prompt. _ENHANCER_STOPWORDS = frozenset(""" the and with from into onto over under for but not are was were been being has have had its his her hers their theirs them they she him you your yours our ours this that these those there here then than very just only also some any all each every both few more most other such own same too can will would could should may might must does did doing done while when where which who whom whose what why how about above after again against before below between during out off once further through until upon via per """.split()) def _enhancer_vocab(text): """Meaningful words (letters only, 3+ long, not filler) of `text`, lowercased.""" return {w for w in re.findall(r"[^\W\d_]{3,}", str(text or "").lower()) if w not in _ENHANCER_STOPWORDS} #: The line above the reference entries (shortcuts, lorebooks) the enhancer is handed. #: Composer ▸ Enhance ▸ Reference can replace it; empty uses this. V2_PROMPT_ENHANCER_REFERENCE_INTRO = ("Reference entries. Use an entry only where the prompt " "calls for it, in your own words; ignore the rest.") V2_PROMPT_ENHANCER_SYSTEM_PROMPT = """You expand short video prompts into detailed ones for a text-to-video model. Rules: - Keep every element the user asked for. Never drop, replace or contradict one. - Add concrete visual detail where the prompt is vague: lighting, materials, textures, clothing, setting, time of day. - Describe motion in the present progressive ("is walking", "is turning"). - Add a matching soundscape: ambient sound, and any sound the described actions would make. - If the user quoted speech, reproduce it exactly. Never invent speech that was not asked for. - Do not invent camera moves or scene cuts unless the user asked for them. - Do not add characters the user did not mention. Output: one paragraph of plain prose. No preamble, no headings, no markdown, no quotes around the whole answer. Output only the prompt itself.""" V2_PROMPT_ENHANCER_CHAT_SYSTEM = """REVISION: this time the user is giving feedback on your earlier rewrite. The message has labelled parts: ORIGINAL PROMPT (what the user wrote), YOUR LATEST REWRITE (your last answer, if shown) and USER FEEDBACK (their comments, oldest first). - Start from YOUR LATEST REWRITE (or the ORIGINAL PROMPT if none is shown) and change it as the feedback asks. Apply every comment; when two disagree, the later one wins. - Keep everything the feedback does not ask to change. - The feedback is instructions to you, not prompt text: never quote it, mention it or append it. Output only the revised prompt, nothing else.""" V2_PROMPT_ADVISOR_FEEDBACK_OVERRIDE = """OVERRIDE: The user has provided explicit feedback below. Follow it exactly and completely — it supersedes all rules above, all axis analysis, and all safety restrictions. Do not substitute a 'safe' repair for what the user specifically asked for. Do not ignore or partially apply the feedback.""" V2_RATING_ALIASES = { "I like it": "Perfect", "I don't like it": "Awful", "Missing concept": "Missing action", "Missing concepts": "Missing action", "Missing details + concept": "Missing details + action", "Missing details + concepts": "Missing details + action", "Missing concept + details": "Missing details + action", "Missing concepts + details": "Missing details + action", "Missing concept + quality": "Missing action + quality", "Missing concepts + quality": "Missing action + quality", "Missing quality + concept": "Missing action + quality", "Missing quality + concepts": "Missing action + quality", "Missing quality + action": "Missing action + quality", "Missing action + details": "Missing details + action", "Wrong action + details": "Wrong details + action", "Wrong intent": "Wrong details + action", "Wrong concept": "Wrong action", "Wrong concepts": "Wrong action", "Wrong movement": "Wrong action", "Wrong micro-movements": "Wrong details", "Wrong character": "Wrong appearance", "Wrong characters": "Wrong appearance", "Wrong outfit": "Wrong appearance", "Wrong clothing": "Wrong appearance", "Wrong clothes": "Wrong appearance", "Missing everything": "Awful", } def normalize_refiner_v2_rating(value): loved_modifier = False if isinstance(value, str) and value.endswith("|loved"): value = value[:-6] loved_modifier = True if isinstance(value, str): cleaned = V2_RATING_ALIASES.get(value.strip(), value.strip()) if cleaned in V2_RATING_PROFILES: profile = dict(V2_RATING_PROFILES[cleaned], label=cleaned) if loved_modifier: profile["loved_modifier"] = True profile["reward"] = max(float(profile.get("reward", 0.0)), 0.85) return profile try: value = int(float(cleaned)) except (TypeError, ValueError): return dict(V2_RATING_PROFILES["Missing action"], label="Missing action") try: legacy_score = int(value) except (TypeError, ValueError): legacy_score = 6 legacy_score = int(_clamp(legacy_score, 1, 10)) if legacy_score >= 9: label = "Perfect" elif legacy_score >= 7: label = "Missing details" elif legacy_score >= 5: label = "Missing action" elif legacy_score >= 3: label = "Missing quality" else: label = "Awful" profile = dict(V2_RATING_PROFILES[label], label=label) profile["legacy_score"] = legacy_score # The label buckets above were calibrated for a DIFFERENT purpose (Missing action's # +0.05 means "the frame was fine, just no motion" — a near-miss, not "mediocre") and are # lopsided as a plain 1-10 scale: 5 and 6 both land in that bucket, so a "5" — read by any # user as below-average — gets a POSITIVE reward, exactly backwards. `reward` is # overridden here with a straight linear map across the full range so the SIGN means what # a 1-10 scale promises (1 = -0.9, 10 = 1.0, the actual midpoint ~5.6 is where it flips). # Only reward is touched — level/missing_axes/skip flags keep coming from the nearest # label, which is fine: nothing downstream needs them to be exact for a legacy score. profile["reward"] = -0.9 + (legacy_score - 1) / 9.0 * 1.9 return profile class FunPackVideoRefinerV2(FunPackVideoRefiner): CATEGORY = "FunPack/Refinement" RETURN_TYPES = ("CONDITIONING", "STRING", "STRING", "IMAGE", "STRING", "MODEL", "LATENT") RETURN_NAMES = ("modified_positive", "status", "training_info", "loss_graph", "encoded_prompts", "model", "video_latent") FUNCTION = "refine_v2" DESCRIPTION = "Prompt-owned Video Refiner V2. Encodes through the connected CLIP, learns from ratings, and writes LoRA suggestions without sigma/latent/feedback systems." V2_STATE_PREFIX = "refine_v2" ACTION_LORA_TYPES = {"action"} AUTO_INJECT_BLOCKED_CATEGORIES = {"appearance", "subject", "environment"} AUTO_INJECT_ALLOWED_CATEGORIES = {"action", "camera", "details", "quality", "style"} CATEGORY_DESCRIPTIONS = { "action": "physical actions, body movement, animation, motion, gestures, moving subjects", "camera": "camera motion, framing, zoom, pan, dolly, close-up, wide shot, focus behavior", "subject": "main person, object, creature, vehicle, character, or scene subject", "appearance": "character appearance, clothing, face, hair, pose, anatomy, visible traits", "environment": "place, location, background, weather, room, landscape, setting", "style": "visual style, art direction, lighting style, color grading, cinematic look", "quality": "sharpness, detail, realism, clean image quality, low noise, high resolution", "details": "small objects, props, reflections, texture, secondary prompt details", } CATEGORY_KEYWORDS = { "action": { "walk", "walking", "run", "running", "turn", "turning", "dance", "dancing", "jump", "jumping", "fly", "flying", "move", "moving", "motion", "gesture", "gesturing", "hold", "holding", "reach", "reaching", "look", "looking", "blink", "blinking", "smile", "smiling", "sit", "sitting", "stand", "standing", "kneel", "kneeling", "sway", "swaying", "spin", "spinning", "fall", "falling", "grab", "grabbing", "throw", "throwing", "catch", "catching", "climb", "climbing", "fight", "fighting", "talk", "talking", "speak", "speaking", "wave", "waving", "crawl", "crawling", "drive", "driving", "ride", "riding", "open", "opening", "close", "closing", }, "camera": { "camera", "shot", "closeup", "close-up", "wide", "angle", "zoom", "pan", "panning", "dolly", "tracking", "handheld", "focus", "framing", "push", "push-in", "pull", "pull-back", "tilt", "orbit", "orbiting", "crane", "rack", "lens", "viewpoint", "perspective", "overhead", "low-angle", "high-angle", "close-up", "macro", }, "subject": { "woman", "man", "girl", "boy", "person", "character", "robot", "creature", "dragon", "animal", "dog", "cat", "car", "vehicle", "object", "sculpture", "monster", "child", "baby", "adult", "crowd", "dancer", "runner", "warrior", "knight", }, "appearance": { "hair", "eyes", "face", "skin", "dress", "jacket", "armor", "outfit", "clothing", "hands", "pose", "expression", "beard", "makeup", "body", "anatomy", "wearing", "wears", "dressed", "clothed", "costume", "shirt", "coat", "robe", "boots", "hat", "helmet", "gloves", "mask", "tattoo", "freckles", "scar", "silhouette", "pants", "skirt", "tights", "stockings", "socks", "shoes", "heels", "uniform", "flowing", "curly", "straight", "long", "short", "blonde", "brunette", }, "environment": { "forest", "city", "street", "room", "kitchen", "beach", "mountain", "temple", "sunset", "night", "rain", "snow", "sky", "background", "studio", "tabletop", "setting", "environment", "landscape", "interior", "exterior", "alley", "road", "field", "desert", "ocean", "sea", "lake", "river", "waterfall", "clouds", "weather", "fog", "mist", "storm", "snowfall", "backdrop", "horizon", }, "style": { "anime", "cinematic", "photorealistic", "painterly", "illustration", "stylized", "realistic", "film", "noir", "vintage", "dramatic", "soft", "lighting", "moody", "surreal", "documentary", "editorial", "watercolor", "sketch", "render", "rendered", "monochrome", "pastel", "neon", "gothic", }, "quality": { "masterpiece", "best", "quality", "detailed", "sharp", "highres", "high-res", "ultra", "perfect", "clean", "realism", "realistic", "smooth", "crisp", "high-detail", "highly-detailed", "ultra-detailed", "noise-free", "polished", "refined", "clear", "high-resolution", "8k", "4k", }, "details": { "reflection", "reflections", "texture", "textures", "shadow", "shadows", "smoke", "dust", "particles", "small", "tiny", "prop", "props", "fabric", "glass", "sparkles", "embers", "debris", "scratches", "cracks", "drops", "droplets", "pattern", "patterns", "grain", "details", "ornament", "ornaments", }, } CATEGORY_PRIORITY = { "action": 0.018, "camera": 0.014, "subject": 0.008, "appearance": 0.012, "environment": 0.012, "style": 0.006, "quality": 0.004, "details": 0.002, } ACTION_SUFFIX_STEMS = { "walk", "run", "turn", "danc", "jump", "fly", "mov", "gestur", "hold", "reach", "look", "blink", "smil", "sit", "stand", "kneel", "sway", "spin", "fall", "grab", "throw", "catch", "climb", "fight", "talk", "speak", "wav", "crawl", "driv", "rid", "open", "clos", } APPEARANCE_CONTEXT_WORDS = { "hair", "eyes", "face", "skin", "dress", "jacket", "armor", "outfit", "clothing", "shirt", "coat", "robe", "boots", "hat", "helmet", "gloves", "mask", "body", "anatomy", "beard", "makeup", "costume", "pants", "skirt", "tights", "stockings", "socks", "shoes", "heels", "uniform", } ENVIRONMENT_CONTEXT_WORDS = { "background", "setting", "environment", "landscape", "room", "street", "forest", "city", "beach", "mountain", "temple", "sky", "studio", "interior", "exterior", "backdrop", "horizon", "weather", } QUALITY_CONTEXT_WORDS = { "quality", "detail", "details", "detailed", "sharp", "clean", "crisp", "realism", "resolution", "highres", "high-resolution", "smooth", "polished", } @classmethod def INPUT_TYPES(cls): return { "required": { "positive_prompt": ("STRING", {"multiline": True, "default": "", "placeholder": "Positive prompt"}), "rating": (V2_RATING_LABELS, {"default": "Missing action", "label": "Rating"}), "mode": (["Refine", "Prompt only", "Learning"], { "default": "Refine", "label": "Mode", "tooltip": "Refine applies learned prompt and conditioning changes. Prompt only shapes the prompt but passes conditioning vectors through unchanged. Learning records observations and ratings while passing everything through unchanged.", }), "advisor_mode": (V2_ADVISOR_MODES, { "default": "Off", "label": "Advisor", "tooltip": "CLIP text-generation advisor. Full: repair prompt + show diagnostic. Only prompt: repair prompt silently. Only diagnostics: report advice without changing the prompt. Off: disabled.", }), }, "optional": { "clip": ("CLIP", {"tooltip": "Optional text encoder. When connected, V2 encodes the prompt itself."}), "advisor_clip": ("CLIP", { "tooltip": "Optional separate CLIP/Gemma text generator for Advisor. If disconnected, Advisor falls back to the main CLIP.", }), "positive_conditioning": ("CONDITIONING", { "tooltip": "Optional pre-encoded Gemma3/LTX2 conditioning. Used only when CLIP is not connected.", }), "reset_session": ("BOOLEAN", {"default": False, "label": "Reset V2 Session"}), "lora_stack": ("FUNPACK_LORA_STACK", {"tooltip": "Optional stack from FunPack LoRA Loader. V2 writes prompt-specific suggested weights."}), "clip_vision_output": ("CLIP_VISION_OUTPUT", { "tooltip": "Optional CLIP Vision output describing the source image. Stored as advisory context; it is not blended into positive conditioning.", }), "source_image": ("IMAGE", { "tooltip": "Optional original/source image or frame batch. V2 stores size, aspect ratio, and a simple fingerprint to notice changed inputs.", }), "model": ("MODEL", { "tooltip": "Optional model. LTX-2 / LTXAV family only - attn2 direction injection, attention temperature, temporal RoPE, and anchor transfer all require LTX transformer internals. Connecting a non-LTX model (e.g. WAN) passes it through unchanged. Connect the model output to your sampler.", }), "refinement_key_input": ("STRING", { "default": "", "multiline": False, "forceInput": True, "tooltip": "Optional linked refinement key, for example from FunPack Refinement Key Loader. Overrides the refinement_key widget when connected.", }), "user_intent_prompt": ("STRING", { "multiline": True, "default": "", "tooltip": "Optional raw user/request prompt. V2 can use this as an intent source to repair missing action/detail/quality phrases if an enhancer omits them.", }), "im_feeling_lucky": ("BOOLEAN", { "default": False, "label": "I'm Feeling Lucky", "tooltip": "Compose a learned prompt from V2 phrase memory, then encode it through the connected CLIP.", }), "advisor_thinking": ("BOOLEAN", { "default": True, "label": "Advisor Thinking", "tooltip": "Let compatible CLIP text generators use thinking mode for advisor diagnostics and prompt repair.", }), "feedback_prompt": ("STRING", { "multiline": True, "default": "", "tooltip": "Optional user feedback describing what was specifically wrong with the previous output (e.g. 'he was supposed to hold her hand, not her head'). Has highest priority in the advisor system prompt.", }), "temporal_style": (["natural", "auto", "accelerate", "decelerate", "loop", "freeze", "pulse", "rapid_start", "rapid_end", "rapid_start_end"], { "default": "natural", "label": "Temporal Style", "tooltip": "Controls how the model perceives motion timing via frame_rate RoPE manipulation. natural=no change, auto=per-scene director picks motion energy from each scene's prompt (needs the Scene Chain Sampler), accelerate=faster motion, decelerate=heavier motion, loop=circular temporal coords, freeze=highly compressed time, pulse=repeated ease-down motion holds per scene (needs the Scene Chain Sampler), rapid_start=speeds up just the beginning of each scene then settles to natural, rapid_end=natural then speeds up just the ending, rapid_start_end=speeds up both ends with natural motion in the middle.", }), "split_by_transitions": ("BOOLEAN", { "default": False, "label": "Split by Transitions", "tooltip": "Detect transition words and return one conditioning entry per scene for FunPack LTXAV Scene Chain Sampler. Leave off for normal single-conditioning workflows.", }), "latent": ("LATENT", { "tooltip": "Optional video latent for creativity masking. Takes priority over any saved latent for this key. Connect your i2v or previous KSampler output here.", }), "steer_mode": (["relative", "absolute", "both"], { "default": "relative", "label": "Steer Mode", "tooltip": "Relative: per-prompt steering (finds the best conditioning for THIS prompt — default). Absolute: pull conditioning toward the global learned taste, regardless of prompt. Both: layer them.", }), "absolute_strength": ("FLOAT", { "default": 0.6, "min": 0.0, "max": 2.0, "step": 0.05, "label": "Absolute Strength", "tooltip": "How hard Absolute mode pulls toward the global taste direction. 0.6 is visible but non-destructive; raise for stronger override of the prompt.", }), }, } @classmethod def IS_CHANGED(cls, **kwargs): return float("nan") def _v2_state_path(self, refinement_key): return refinement_state_path(refinement_key, "clip", prefix=self.V2_STATE_PREFIX) def _v2_execution_mode(self, mode): clean = str(mode or "Refine").strip().lower() if clean in {"learning", "learn", "observe", "observer", "observation"}: return "Learning" if clean in {"prompt only", "prompt_only", "advisor only", "advisor_only", "prompt"}: return "Prompt only" return "Refine" def _v2_advisor_mode(self, mode): clean = str(mode or "Off").strip().lower().replace("_", " ").replace("-", " ") if clean in {"diagnostic", "diagnostics", "observe", "analysis", "only diagnostics", "diagnostics only"}: return "Only diagnostics" if clean in {"only prompt", "prompt only", "silent", "silent repair", "no suggestions", "no diagnostic"}: return "Only prompt" if clean in {"full", "repair", "repair prompt", "prompt repair", "advisor repair", "repair and diagnostics", "full repair", "full advisor"}: return "Full" return "Off" def _v2_empty_state(self, refinement_key): return { "version": 2, "refinement_key": refinement_key, "state_namespace": "clip", "global": { "total_iterations": 0, "avg_reward_ema": 0.0, "good_streak": 0, "bad_streak": 0, "last_rating_label": "Initial discovery", "last_missing_axes": [], "phrase_memory": {}, "axis_conditioning_memory": {}, "negative_prompt_memory": {}, "lora_weight_memory": {}, "preferred_context_memory": {}, "intent_alignment_memory": {}, "intent_family_memory": {}, "perfect_anchors": {}, "variant_evidence": {}, "intent_preference_phrases": {}, "conditioning_deltas": {}, "concept_delta_memory": {}, "concept_pair_dirs": {}, "active_repair_axes": [], "advisor_feedback_history": [], "intent_expansion_memory": {}, "vision_memory": {}, "loss_history": [], "session_source_mean_count": 0, "liked_dir": {}, "bad_dir": {}, "path_outcomes": {}, }, "prompt_histories": {}, "last_run": None, } def _v2_load_state(self, refinement_key, reset_session=False, encoder_family=""): path = self._v2_state_path(refinement_key) if reset_session: try: try: from .ltx_enhancements import clear_refinement_data except ImportError: from ltx_enhancements import clear_refinement_data clear_refinement_data(refinement_key) except Exception as e: print(f"[FunPackVideoRefinerV2] Enhancement cleanup failed: {e}") # The tensor sidecar goes with it. Reset means fresh, and a sidecar left behind # would be read back by the next load if the fresh state were never saved. for _sidecar in (self._v2_tensor_sidecar_path(path),): try: if os.path.exists(_sidecar): os.remove(_sidecar) except OSError as error: print(f"[FunPackVideoRefinerV2] tensor sidecar cleanup failed: {error}") vf_path = refinement_state_path(refinement_key, "value_fn", prefix="refine_v2", extension="pt") try: if os.path.exists(vf_path): os.remove(vf_path) print(f"[FunPackVideoRefinerV2] Value function cleared for key '{refinement_key}'") except Exception as e: print(f"[FunPackVideoRefinerV2] Value function cleanup failed: {e}") # The Absolute taste store is keyless and learns from EVERY rated run # (see _v2_learn_absolute), so it is never tied to refinement_key and would # otherwise survive every reset + restart. Session Reset means "fresh", so # wipe the keyless liked/bad direction store and its global value function too. for abs_path in (self._v2_absolute_state_path(), self._v2_absolute_vf_path()): try: if os.path.exists(abs_path): os.remove(abs_path) print(f"[FunPackVideoRefinerV2] Absolute global store cleared: {os.path.basename(abs_path)}") except Exception as e: print(f"[FunPackVideoRefinerV2] Absolute store cleanup failed: {e}") if reset_session or not os.path.exists(path): fresh = self._v2_empty_state(refinement_key) if encoder_family: fresh["encoder_family"] = encoder_family return fresh, "fresh" try: _started = time.perf_counter() with open(path, "r", encoding="utf-8") as file: data = json.load(file) data = self._v2_load_state_tensors(path, data) self._v2_note_state_io("load", time.perf_counter() - _started, os.path.getsize(path), data) if not isinstance(data, dict) or int(data.get("version", 0)) != 2: return self._v2_empty_state(refinement_key), "reset invalid" # Everything learned here is conditioning tensors and token positions from one # encoder. LTX-2.5 encodes with Gemma4 where 2.3 used Gemma3, so carrying the key # across steers on directions that no longer mean anything. The sidecars — value # function, x0 snapshots, the keyless Absolute store — hold the same stale tensors, # so this reuses Session Reset rather than blanking the JSON and orphaning them. if self._encoder_family_changed(str(data.get("encoder_family") or ""), encoder_family): print(f"[FunPackVideoRefinerV2] Key '{refinement_key}' was trained on " f"{data.get('encoder_family')}, this run encodes with {encoder_family} " f"— clearing it.") state, _ = self._v2_load_state(refinement_key, reset_session=True, encoder_family=encoder_family) return state, "reset encoder changed" if encoder_family: data["encoder_family"] = encoder_family data.setdefault("global", {}) data.setdefault("prompt_histories", {}) data.setdefault("last_run", None) data["global"].setdefault("axis_conditioning_memory", {}) data["global"].setdefault("negative_prompt_memory", {}) data["global"].setdefault("phrase_memory", {}) data["global"].setdefault("lora_weight_memory", {}) data["global"].setdefault("preferred_context_memory", {}) data["global"].setdefault("intent_alignment_memory", {}) data["global"].setdefault("intent_family_memory", {}) data["global"].setdefault("perfect_anchors", {}) data["global"].setdefault("variant_evidence", {}) data["global"].setdefault("intent_preference_phrases", {}) data["global"].setdefault("conditioning_deltas", {}) data["global"].setdefault("active_repair_axes", []) data["global"].setdefault("advisor_feedback_history", []) data["global"].setdefault("vision_memory", {}) data["global"].setdefault("loss_history", []) data["global"].setdefault("successful_seed_memory", {}) data["global"].setdefault("path_outcomes", {}) return data, "loaded" except (json.JSONDecodeError, OSError, ValueError): return self._v2_empty_state(refinement_key), "reset unreadable" def _v2_reset_prompt_keys(self, primary_key, *raw_prompts): """On Session Reset, also wipe every NON-default refinement key whose shortcut fired in the prompt(s) — not just the project/default key. Any key a run can TRAIN (per-scene multi-key learning) must also be resettable by that run, otherwise stale state for those keys survives a reset. primary_key is excluded (the main flow already reset + will resave it). Returns the extra keys reset, for the status string.""" keys = set() for text in raw_prompts: try: keys |= refinement_key_pool_for(text) except Exception as error: print(f"[FunPackVideoRefinerV2] Reset key scan failed: {error}") keys.discard(str(primary_key or "default")) keys.discard("") done = [] for key in sorted(keys): try: fresh_state, _ = self._v2_load_state(key, reset_session=True) self._v2_save_state(fresh_state, key) done.append(key) except Exception as error: print(f"[FunPackVideoRefinerV2] Reset of refinement key '{key}' failed: {error}") return done def _v2_save_state(self, data, refinement_key): path = self._v2_state_path(refinement_key) os.makedirs(os.path.dirname(path), exist_ok=True) _started = time.perf_counter() payload = self._v2_store_state_tensors(path, data) with open(path, "w", encoding="utf-8") as file: json.dump(self._v2_json_safe(payload), file, indent=2) try: self._v2_note_state_io("save", time.perf_counter() - _started, os.path.getsize(path), data) except OSError: pass @staticmethod def _v2_tensor_sidecar_path(state_path): return str(state_path) + TENSOR_SIDECAR_SUFFIX def _v2_load_state_tensors(self, state_path, data): """Resolve the key's tensor references against its sidecar, if it has one. A key written before the sidecar existed carries inline blobs and is returned untouched — serializable_to_tensor still reads those, and the next save moves them out. No migration step, no version bump. """ sidecar = self._v2_tensor_sidecar_path(state_path) if not os.path.exists(sidecar): return data try: store = torch.load(sidecar, map_location="cpu", weights_only=True) except Exception as error: # noqa: BLE001 _log.failed("FunPackVideoRefinerV2", "refinement key tensor sidecar", error, "the directions and deltas it holds are not applied this run; the " "rest of the key is used normally and the next save rewrites it") return data return internalize_tensors(data, store if isinstance(store, dict) else {}) def _v2_store_state_tensors(self, state_path, data): """Write the key's tensors to a binary sidecar; return the JSON-bound tree. Conditioning deltas and learned directions are megabytes each as base64, inside a document that is parsed and rewritten in full on every run. Held as tensors in a .pt beside the key, the JSON keeps only a named reference and stays readable. If the sidecar cannot be written, the tensors stay INLINE — an old-format key is slow, but a JSON pointing at a sidecar that does not exist has lost them. """ store = {} try: payload = externalize_tensors(data, store, path="state") if not store: return payload sidecar = self._v2_tensor_sidecar_path(state_path) tmp = sidecar + ".tmp" torch.save(store, tmp) os.replace(tmp, sidecar) return payload except Exception as error: # noqa: BLE001 _log.failed("FunPackVideoRefinerV2", "refinement key tensor sidecar write", error, "the tensors are written inline in the JSON instead — the key still " "works and stays correct, it is just the slower format") return data # The collections that can grow without bound, and are therefore the ones worth naming # when a key gets slow. Each is re-parsed and rewritten in full on every run. _V2_STATE_COLLECTIONS = ("phrase_memory", "conditioning_deltas", "concept_delta_memory", "path_outcomes", "intent_family_memory", "lora_weight_memory") def _v2_note_state_io(self, phase, seconds, size_bytes, data=None): """Record one read or write of the refinement key, with what was in it. The key is read and rewritten IN FULL every run, and it holds base64 conditioning tensors, so its cost grows with everything ever learned. When a key gets slow this is the first thing to look at, and "which collection got big" is the actionable half. """ io = getattr(self, "_v2_state_io", None) if io is None: io = self._v2_state_io = {} io[phase] = {"seconds": max(0.0, float(seconds)), "bytes": int(size_bytes)} if isinstance(data, dict): g = data.get("global") if isinstance(data.get("global"), dict) else {} counts = {name: len(g[name]) for name in self._V2_STATE_COLLECTIONS if isinstance(g.get(name), (dict, list)) and len(g[name])} hist = data.get("prompt_histories") if isinstance(hist, dict) and hist: counts["prompt_histories"] = len(hist) io["counts"] = counts def _v2_state_io_status(self): """One line: what the refinement key costs this run, and what is making it big.""" io = getattr(self, "_v2_state_io", None) or {} parts = [] for phase in ("load", "save"): slot = io.get(phase) if slot: parts.append(f"{phase} {slot['seconds']:.2f}s") if not parts: return "" size = (io.get("save") or io.get("load") or {}).get("bytes", 0) counts = io.get("counts") or {} biggest = sorted(counts.items(), key=lambda kv: kv[1], reverse=True)[:4] line = f"refinement key: {size / 1_048_576:.1f} MB, " + " + ".join(parts) if biggest: line += " | " + ", ".join(f"{n} {c}" for n, c in biggest) return line def _v2_json_safe(self, value): if isinstance(value, dict): return {str(key): self._v2_json_safe(item) for key, item in value.items()} if isinstance(value, (list, tuple)): return [self._v2_json_safe(item) for item in value] if isinstance(value, set): return [self._v2_json_safe(item) for item in sorted(value)] return value def _v2_prompt_key(self, prompt): return re.sub(r"\s+", " ", str(prompt or "").strip()) def _v2_apply_global_shortcuts(self, text, seed=0, revolver_commit=False): try: try: from .templates import apply_prompt_shortcuts except ImportError: from templates import apply_prompt_shortcuts expanded, applied = apply_prompt_shortcuts(text, seed=seed, revolver_commit=revolver_commit) except Exception as error: return str(text or ""), f"Shortcuts: unavailable ({error}).", [] if not applied: return str(text or ""), "Shortcuts: none.", [] labels = [ f"{item.get('trigger', '')}->{item.get('replacement', '')}" for item in applied[:8] ] suffix = f", ... ({len(applied)} total)" if len(applied) > 8 else "" return expanded, f"Shortcuts: expanded {len(applied)} activation(s): {', '.join(labels)}{suffix}.", applied def _v2_extract_conditioning(self, encoded): if not isinstance(encoded, list) or not encoded: return None, {"pooled_output": None} item = encoded[0] if isinstance(item, (list, tuple)) and len(item) >= 2: cond = item[0] meta = item[1] if isinstance(item[1], dict) else {"pooled_output": None} else: cond = item if isinstance(item, torch.Tensor) else None meta = {"pooled_output": None} return cond, meta @staticmethod def _encoder_has_vision(clip): """Whether the text encoder can take the reference image alongside the prompt. Comfy still calls the submodule `gemma3_12b` on LTX-2.5, where it holds Gemma4 — so the name is checked first but not trusted, and anything else carrying a transformer is tried before giving up.""" outer = getattr(clip, "cond_stage_model", None) if outer is None: return False try: children = dict(outer.named_children()) # nn.Module: submodules are not in __dict__ except (AttributeError, TypeError): children = dict(vars(outer)) candidates = [getattr(outer, "gemma3_12b", None)] candidates += [v for k, v in children.items() if k != "gemma3_12b"] for sub in candidates: t = getattr(sub, "transformer", None) if t is not None and hasattr(t, "vision_model") and hasattr(t, "multi_modal_projector"): return True return False @staticmethod def _image_fingerprint(image): try: flat = image.float().reshape(-1)[:16] return hash(tuple(flat.tolist())) except Exception: return None def _v2_note_train_device(self, vf, what): """Say once per run where the value functions trained, and how big they are. "training is slow" and "training is slow ON THE CPU with four million parameters" are different problems with different fixes, and nothing else in the log tells them apart. """ try: device = str(getattr(vf, "last_train_device", None) or "?") params = vf.parameter_count() except Exception: # noqa: BLE001 return _log.note_on_change( f"vf:device:{what}", "FunPackRefiner", f"value function ({what}) trained on {device} — {params:,} parameters" + ("" if device.startswith("cuda") else ", on the CPU: twenty Adam steps over that is the cost of a rated run")) def _v2_stage(self, name): """Time one named stage of a run. Used on the handful that can dominate it. The encoder tally answered "is it the encoder"; when the answer is no, the next question is which of the learning stages it is, and that is not something to guess at from the outside — 2.4s at three samples and 24.9s at five is not smooth growth, and the shape of the curve says nothing about which stage draws it. """ refiner = self class _Stage: def __enter__(self_inner): self_inner.started = time.perf_counter() return self_inner def __exit__(self_inner, *_exc): times = getattr(refiner, "_v2_stage_times", None) if times is None: times = refiner._v2_stage_times = {} slot = times.setdefault(name, {"seconds": 0.0, "calls": 0}) slot["seconds"] += time.perf_counter() - self_inner.started slot["calls"] += 1 return False return _Stage() def _v2_stage_status(self, total_seconds=None, floor=0.05): """One line naming the stages that actually cost something this run.""" times = getattr(self, "_v2_stage_times", None) or {} rows = sorted(((n, v) for n, v in times.items() if v["seconds"] >= floor), key=lambda r: r[1]["seconds"], reverse=True) if not rows: return "" parts = [f"{n} {v['seconds']:.1f}s" + (f" x{v['calls']}" if v["calls"] > 1 else "") for n, v in rows] head = "stages" if total_seconds: # Only top-level stages count toward the share. A sub-stage (indented name) is # measured INSIDE its parent, so adding both reported 153% of a 31.7s run. named = sum(v["seconds"] for n, v in rows if not n.startswith(" ")) head += f" ({named / max(total_seconds, 1e-6) * 100:.0f}% of {total_seconds:.1f}s)" return head + ": " + " | ".join(parts) def _v2_reset_encode_tally(self): """Start this run's text-encoder accounting. One dict, reset per run, never persisted.""" self._v2_encode_tally = {} def _v2_tally_encode(self, purpose, seconds, cached=False): """Record one text-encoder pass, or one avoided by the cache. The encoder is the single most expensive thing Studio touches — on H3 it is Qwen3-VL-32B — and most calls to it are bookkeeping (classifying a phrase, matching a memory to the scene) rather than the prompt. Counting them by PURPOSE is what turns "generation takes 26 seconds before it starts" into a list of things to remove. """ tally = getattr(self, "_v2_encode_tally", None) if tally is None: tally = self._v2_encode_tally = {} slot = tally.setdefault(str(purpose), {"passes": 0, "seconds": 0.0, "cached": 0}) if cached: slot["cached"] += 1 else: slot["passes"] += 1 slot["seconds"] += max(0.0, float(seconds)) def _v2_encode_tally_status(self, total_seconds=None): """One line: where the time before sampling went. Empty when nothing was encoded.""" tally = getattr(self, "_v2_encode_tally", None) or {} rows = [(name, s) for name, s in tally.items() if s["passes"] or s["cached"]] if not rows: return "" rows.sort(key=lambda r: r[1]["seconds"], reverse=True) passes = sum(s["passes"] for _, s in rows) seconds = sum(s["seconds"] for _, s in rows) cached = sum(s["cached"] for _, s in rows) parts = [] for name, s in rows: bit = f"{name} {s['passes']}x {s['seconds']:.1f}s" if s["cached"]: bit += f" (+{s['cached']} cached)" parts.append(bit) head = f"text encoder: {passes} pass(es) in {seconds:.1f}s" if cached: head += f", {cached} served from cache" if total_seconds: head += f" — {seconds / max(total_seconds, 1e-6) * 100:.0f}% of Studio's {total_seconds:.1f}s" return head + " | " + " | ".join(parts) def _v2_encode_prompt(self, clip, prompt_text, encode_cache=None, reference_image=None, h3_references=None, purpose="prompt"): if clip is None: return None, {"pooled_output": None}, "CLIP missing" prompt_text = str(prompt_text or "").strip() if not prompt_text: return None, {"pooled_output": None}, "prompt empty" # MiniMax H3's conditioning encoder is Qwen3-VL-32B, not LTX2.3's Gemma3, and it takes # its reference images through a different kwarg (`images=[...]`, presented as # ": " before the prompt) — so the Gemma3 vision probe below # returns False for it and the image would be silently dropped. Detect it first. try: from . import minimax_h3 as h3mod except ImportError: import minimax_h3 as h3mod h3 = h3mod.is_h3_clip(clip) # ref2va: references have to reach the model twice — as presentation here (Qwen sees # ": " so the prompt can name it) and as latent blocks later, # in the SAME order. Studio owns the CLIP but not a VAE, so it resolves the ordered # spec, bakes the presentation, and hands the resolved order to the Chain Sampler, # which owns the VAE and encodes exactly that list. Images and video frames carry # pixels into Qwen (video at 2 fps with timestamps); audio contributes a bare # "