import base64
import gc
import glob
import json
import math
import os
import random
import re
from datetime import datetime, timezone
from hashlib import md5
from typing import Optional

import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image, ImageDraw, ImageFont
from safetensors.torch import load_file
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer

import folder_paths

LORA_REFINER_TYPE_PROFILES = {
    "general": {"step": 0.025, "max_offset": 0.20, "min_offset": -0.35, "bad_max_offset": 0.45, "bad_min_offset": -1.35, "culprit_bias": 0.28},
    "action": {"step": 0.046, "max_offset": 0.35, "min_offset": -0.45, "bad_max_offset": 0.75, "bad_min_offset": -2.10, "culprit_bias": 0.16},
    "concept": {"step": 0.046, "max_offset": 0.35, "min_offset": -0.45, "bad_max_offset": 0.75, "bad_min_offset": -2.10, "culprit_bias": 0.16},
    "style": {"step": 0.032, "max_offset": 0.28, "min_offset": -0.38, "bad_max_offset": 0.58, "bad_min_offset": -1.55, "culprit_bias": 0.20},
    "quality": {"step": 0.022, "max_offset": 0.18, "min_offset": -0.30, "bad_max_offset": 0.38, "bad_min_offset": -1.20, "culprit_bias": 0.18},
    "character": {"step": 0.024, "max_offset": 0.20, "min_offset": -0.32, "bad_max_offset": 0.42, "bad_min_offset": -1.30, "culprit_bias": 0.10},
}

RATING_LABELS = [
    "-Just forget it-",
    "Perfect",
    "Missing details",
    "Missing concept",
    "Missing quality",
    "Missing details + concept",
    "Missing details + quality",
    "Missing concept + quality",
    "Awful",
]

RATING_PROFILES = {
    "-Just forget it-": {
        "key": "forget",
        "level": 0,
        "legacy_score": 0,
        "legacy_range": "ignored",
        "reward": 0.0,
        "quality_signal": 0.0,
        "concept_signal": 0.0,
        "detail_signal": 0.0,
        "missing_axes": [],
        "prompt_emphasis": 0.0,
        "skip_learning": True,
    },
    "Initial discovery": {
        "key": "discover",
        "level": 5,
        "legacy_score": 6,
        "legacy_range": "discovery",
        "reward": 0.0,
        "quality_signal": 0.0,
        "concept_signal": 0.0,
        "detail_signal": 0.0,
        "missing_axes": [],
        "prompt_emphasis": 0.0,
    },
    "Perfect": {
        "key": "like",
        "level": 8,
        "legacy_score": 10,
        "legacy_range": "keep",
        "reward": 1.0,
        "quality_signal": 1.0,
        "concept_signal": 1.0,
        "detail_signal": 1.0,
        "missing_axes": [],
        "prompt_emphasis": 0.45,
    },
    "Missing details": {
        "key": "missing_details",
        "level": 7,
        "legacy_score": 8,
        "legacy_range": "boost details",
        "reward": 0.45,
        "quality_signal": 0.85,
        "concept_signal": 0.70,
        "detail_signal": -1.0,
        "missing_axes": ["details"],
        "prompt_emphasis": 0.65,
    },
    "Missing concept": {
        "key": "missing_concept",
        "level": 6,
        "legacy_score": 6,
        "legacy_range": "boost concept",
        "reward": 0.10,
        "quality_signal": 0.85,
        "concept_signal": -1.0,
        "detail_signal": 0.20,
        "missing_axes": ["concept"],
        "prompt_emphasis": 0.95,
    },
    "Missing quality": {
        "key": "missing_quality",
        "level": 5,
        "legacy_score": 4,
        "legacy_range": "boost quality",
        "reward": -0.25,
        "quality_signal": -1.0,
        "concept_signal": 0.65,
        "detail_signal": 0.65,
        "missing_axes": ["quality"],
        "prompt_emphasis": 0.55,
    },
    "Missing details + concept": {
        "key": "missing_details_concept",
        "level": 4,
        "legacy_score": 5,
        "legacy_range": "boost details+concept",
        "reward": -0.05,
        "quality_signal": 0.75,
        "concept_signal": -1.0,
        "detail_signal": -1.0,
        "missing_axes": ["details", "concept"],
        "prompt_emphasis": 1.05,
    },
    "Missing details + quality": {
        "key": "missing_details_quality",
        "level": 3,
        "legacy_score": 3,
        "legacy_range": "boost details+quality",
        "reward": -0.35,
        "quality_signal": -1.0,
        "concept_signal": 0.50,
        "detail_signal": -1.0,
        "missing_axes": ["details", "quality"],
        "prompt_emphasis": 0.82,
    },
    "Missing concept + quality": {
        "key": "missing_concept_quality",
        "level": 2,
        "legacy_score": 2,
        "legacy_range": "boost concept+quality",
        "reward": -0.55,
        "quality_signal": -1.0,
        "concept_signal": -1.0,
        "detail_signal": 0.15,
        "missing_axes": ["concept", "quality"],
        "prompt_emphasis": 1.02,
    },
    "Awful": {
        "key": "awful",
        "level": 1,
        "legacy_score": 2,
        "legacy_range": "boost all",
        "reward": -0.85,
        "quality_signal": -1.0,
        "concept_signal": -1.0,
        "detail_signal": -1.0,
        "missing_axes": ["details", "concept", "quality"],
        "prompt_emphasis": 1.20,
        "rollback_on_failure": True,
    },
}

RATING_ALIASES = {
    "I like it": "Perfect",
    "I don't like it": "Awful",
    "Missing details + concepts": "Missing details + concept",
    "Missing concept + details": "Missing details + concept",
    "Missing quality + details": "Missing details + quality",
    "Missing quality + concept": "Missing concept + quality",
    "Missing everything": "Awful",
}

CATEGORY_FEEDBACK_MAP = {
    1: "general",
    2: "concept",
    3: "style",
    4: "quality",
    5: "character",
    6: "details",
}


def _clamp(value, low, high):
    return max(low, min(high, value))


def normalize_refiner_rating(value):
    if isinstance(value, str):
        cleaned = value.strip()
        cleaned = RATING_ALIASES.get(cleaned, cleaned)
        if cleaned in RATING_PROFILES:
            return dict(RATING_PROFILES[cleaned], label=cleaned)
        try:
            value = int(float(cleaned))
        except (TypeError, ValueError):
            return dict(RATING_PROFILES["Missing concept"], label="Missing concept")

    try:
        legacy_score = int(value)
    except (TypeError, ValueError):
        legacy_score = RATING_PROFILES["Missing concept"]["legacy_score"]

    legacy_score = int(_clamp(legacy_score, 1, 10))
    if legacy_score >= 9:
        label = "Perfect"
    elif legacy_score >= 7:
        label = "Missing details"
    elif legacy_score >= 5:
        label = "Missing concept"
    elif legacy_score >= 3:
        label = "Missing quality"
    else:
        label = "Awful"

    profile = dict(RATING_PROFILES[label], label=label)
    profile["legacy_score"] = legacy_score
    return profile


PROTECTED_PHRASE_RE = re.compile(r'"([^"]+)"|“([^”]+)”|\\([^\\]+)\\')
def tensor_to_serializable(t: torch.Tensor) -> dict:
    if not isinstance(t, torch.Tensor):
        raise TypeError(f"Expected torch.Tensor, got {type(t)}")
    arr = t.detach().cpu().numpy()
    return {
        "data": base64.b64encode(arr.tobytes()).decode("utf-8"),
        "shape": list(arr.shape),
        "dtype": str(arr.dtype)
    }

def serializable_to_tensor(d: dict) -> torch.Tensor:
    arr = np.frombuffer(base64.b64decode(d["data"]), dtype=d["dtype"]).reshape(d["shape"]).copy()
    tensor = torch.from_numpy(arr).to(dtype=torch.float32)
    if torch.cuda.is_available():
        tensor = tensor.cuda()
    return tensor

def _safe_float(value, fallback=0.0):
    try:
        return float(value)
    except (TypeError, ValueError):
        return fallback

def _safe_int(value, fallback=0):
    try:
        return int(value)
    except (TypeError, ValueError):
        return fallback

def _to_image_tensor(image):
    arr = np.asarray(image).astype(np.float32) / 255.0
    return torch.from_numpy(arr).unsqueeze(0)

def render_refinement_loss_graph(refinement_key, scheduler_mode, mode, total_iterations, latest_learning_loss, points, width=960, height=540):
    image = Image.new("RGBA", (width, height), (0, 0, 0, 255))
    draw = ImageDraw.Draw(image)
    font = ImageFont.load_default()

    bg = (24, 28, 34)
    panel = (30, 36, 44)
    grid = (64, 76, 92)
    axis = (122, 137, 156)
    text = (235, 239, 244)
    subtext = (168, 180, 194)
    line = (74, 201, 255)
    fill = (74, 201, 255, 72)
    point_color = (255, 181, 71)

    image.paste(bg, (0, 0, width, height))

    draw.rounded_rectangle((16, 16, width - 16, height - 16), radius=18, fill=panel, outline=(49, 58, 70), width=1)
    draw.text((32, 28), "FunPack Refinement Loss", fill=text, font=font)
    draw.text((32, 50), f"Embedding: {refinement_key}", fill=text, font=font)
    draw.text((32, 68), f"Scheduler: {scheduler_mode.upper()}   Mode: {mode.upper()}   Iterations: {total_iterations}   Latest loss: {latest_learning_loss:.4f}", fill=subtext, font=font)

    if not points:
        draw.text((32, 112), "No loss history is available yet.", fill=(255, 160, 122), font=font)
        return _to_image_tensor(image.convert("RGB"))

    left = 70
    top = 92
    right = width - 28
    bottom = height - 58
    graph_width = max(1, right - left)
    graph_height = max(1, bottom - top)

    draw.line((left, top, left, bottom), fill=axis, width=1)
    draw.line((left, bottom, right, bottom), fill=axis, width=1)

    y_values = [_safe_float(point.get("learning_loss"), 0.0) for point in points]
    x_values = [_safe_int(point.get("total_iteration"), index + 1) for index, point in enumerate(points)]

    y_min = min(y_values)
    y_max = max(y_values)
    if abs(y_max - y_min) < 1e-9:
        pad = 0.25 if y_max == 0 else abs(y_max) * 0.15
        y_min -= pad
        y_max += pad
    else:
        pad = max(0.02, (y_max - y_min) * 0.12)
        y_min -= pad
        y_max += pad

    x_min = min(x_values)
    x_max = max(x_values)
    if x_min == x_max:
        x_max = x_min + 1

    for i in range(5):
        y = top + (graph_height * i / 4.0)
        draw.line((left, y, right, y), fill=grid, width=1)
        y_label = y_max - ((y_max - y_min) * i / 4.0)
        draw.text((18, y - 6), f"{y_label:.3f}", fill=subtext, font=font)

    for i in range(5):
        x = left + (graph_width * i / 4.0)
        draw.line((x, top, x, bottom), fill=grid, width=1)
        x_label = round(x_min + ((x_max - x_min) * i / 4.0))
        draw.text((x - 10, bottom + 10), str(x_label), fill=subtext, font=font)

    coords = []
    for x_value, y_value in zip(x_values, y_values):
        norm_x = (x_value - x_min) / max(1e-9, (x_max - x_min))
        norm_y = (y_value - y_min) / max(1e-9, (y_max - y_min))
        px = left + norm_x * graph_width
        py = bottom - norm_y * graph_height
        coords.append((px, py))

    if len(coords) == 1:
        px, py = coords[0]
        draw.ellipse((px - 4, py - 4, px + 4, py + 4), fill=point_color)
    else:
        polygon = [(coords[0][0], bottom)] + coords + [(coords[-1][0], bottom)]
        draw.polygon(polygon, fill=fill)
        draw.line(coords, fill=line, width=3)
        px, py = coords[-1]
        draw.ellipse((px - 5, py - 5, px + 5, py + 5), fill=point_color, outline=(255, 243, 214))

    return _to_image_tensor(image.convert("RGB"))


def refinement_state_path(refinement_key, mode, prefix="refine", extension="json"):
    base_dir = os.path.dirname(os.path.abspath(__file__))
    refinements_dir = os.path.join(base_dir, "refinements")
    safe_key = md5(f"{(mode or 'ltx2').lower()}::{refinement_key}".encode("utf-8")).hexdigest()
    return os.path.join(refinements_dir, f"{prefix}_{safe_key}.{extension}")


def clone_latent(latent):
    if not isinstance(latent, dict):
        return None

    cloned = {}
    for key, value in latent.items():
        cloned[key] = value.detach().clone() if isinstance(value, torch.Tensor) else value
    return cloned


def latent_samples(latent):
    if not isinstance(latent, dict):
        return None
    samples = latent.get("samples")
    return samples if isinstance(samples, torch.Tensor) else None


def latent_sample_type_name(latent):
    if not isinstance(latent, dict):
        return "missing"
    samples = latent.get("samples")
    if samples is None:
        return "missing samples"
    shape = tuple(samples.shape) if isinstance(samples, torch.Tensor) else "unknown"
    return f"{type(samples).__module__}.{type(samples).__name__}, shape={shape}"


def latent_is_plain_video_tensor(latent):
    samples = latent_samples(latent)
    if samples is None:
        return False
    if samples.dim() == 5:
        return True
    if samples.dim() == 4 and latent.get("type") != "audio":
        return True
    return False


def cpu_tensor_bundle(latent):
    if not isinstance(latent, dict):
        return {}

    bundle = {}
    for key, value in latent.items():
        if isinstance(value, torch.Tensor):
            bundle[key] = value.detach().cpu().clone()
    return bundle


def latent_from_tensor_bundle(bundle):
    if not isinstance(bundle, dict):
        return None

    latent = {}
    for key, value in bundle.items():
        if key == "_meta":
            continue
        latent[key] = value.detach().clone() if isinstance(value, torch.Tensor) else value
    return latent if latent_samples(latent) is not None else None


class FunPackVideoRefiner:
    LATENT_OUTPUT_INDEX = 6
    NO_LATENT_REFERENCE_ERROR = "No available latent to operate. Please connect reference latent to input of Video Refiner."
    WRONG_LATENT_ERROR = (
        "Video Refiner latent input must be a plain video LATENT with tensor samples. "
        "Audio latents and LTX audio/video combined NestedTensor latents are not supported here. "
        "Connect only the video latent to Video Refiner, then feed the refined video latent into "
        "LTXVConcatAVLatent as video_latent."
    )
    SAVED_LATENT_ONLY_STATUS = (
        "Running refinement on saved latent. Changing reference latent shape and size will cause no effect to generation."
    )

    _tokenizers = {}
    _tokenizer_sources = {
        "ltx2": [
            ("DreamFast/gemma-3-12b-it-heretic-v2", {
                "trust_remote_code": True,
                "use_fast": True,
            }),
        ],
        "wan": [
            ("Wan-AI/Wan2.2-T2V-A14B", {
                "subfolder": "google/umt5-xxl",
                "use_fast": True,
            }),
            ("Wan-AI/Wan2.2-I2V-A14B", {
                "subfolder": "google/umt5-xxl",
                "use_fast": True,
            }),
            ("Wan-AI/Wan2.2-Animate-14B", {
                "subfolder": "google/umt5-xxl",
                "use_fast": True,
            }),
            ("Wan-AI/Wan2.1-T2V-1.3B", {
                "subfolder": "google/umt5-xxl",
                "use_fast": True,
            }),
            ("Runware/Wan2.2-TI2V-5B", {
                "subfolder": "google/umt5-xxl",
                "use_fast": True,
            }),
            ("ddwf/Wan2.2-Animate-14B", {
                "subfolder": "google/umt5-xxl",
                "use_fast": True,
            }),
        ],
    }

    @classmethod
    def _get_tokenizer_sources(cls, mode="ltx2"):
        mode = (mode or "ltx2").lower()
        sources = list(cls._tokenizer_sources.get(mode, cls._tokenizer_sources["ltx2"]))

        if mode == "wan":
            local_sources = []
            models_dir = getattr(folder_paths, "models_dir", None)
            if models_dir:
                local_sources.extend(
                    (path, {})
                    for path in glob.glob(os.path.join(models_dir, "Wan", "*", "google", "umt5-xxl"))
                    if os.path.isdir(path)
                )
                text_encoder_tokenizer = os.path.join(models_dir, "text_encoders", "google", "umt5-xxl")
                if os.path.isdir(text_encoder_tokenizer):
                    local_sources.append((text_encoder_tokenizer, {}))

            if local_sources:
                sources = local_sources + sources

        return sources

    @classmethod
    def _get_tokenizer(cls, mode="ltx2"):
        mode = (mode or "ltx2").lower()
        cached = cls._tokenizers.get(mode)
        if cached is not None:
            return cls._tokenizers[mode]

        sources = cls._get_tokenizer_sources(mode)
        for model_id, kwargs in sources:
            try:
                cls._tokenizers[mode] = AutoTokenizer.from_pretrained(model_id, **kwargs)
                return cls._tokenizers[mode]
            except Exception as e:
                print(f"[FunPackVideoRefiner] Tokenizer load failed for mode '{mode}' from '{model_id}': {e}")

        return None

    def _normalize_prompt_for_mode(self, prompt: str, mode: str) -> str:
        prompt = (prompt or "").strip()
        if mode == "wan":
            return re.sub(r"\s+", " ", prompt)
        return prompt

    def _prompt_looks_like_refusal(self, prompt: str) -> bool:
        text = re.sub(r"\s+", " ", str(prompt or "").strip().lower())
        if not text:
            return False
        text = text.replace("’", "'").replace("`", "'")
        prefix = text[:520]
        refusal_patterns = (
            r"^(?:i'?m|i am)?\s*sorry\b.{0,120}\b(?:can(?:not|'?t)|unable|not able|won'?t)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate|do that|with this request)\b",
            r"^(?:i|we)\s+(?:can(?:not|'?t)|won'?t|am unable|are unable|am not able|are not able)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate|do that|with this request)\b",
            r"^as an ai\b.{0,160}\b(?:can(?:not|'?t)|unable|not able|won'?t)\b.{0,120}\b(?:help|assist|comply|fulfill|provide|create|generate)\b",
            r"^(?:i|we)\s+(?:must|have to|need to)\s+(?:decline|refuse)\b",
        )
        return any(re.search(pattern, prefix) for pattern in refusal_patterns)

    def _v2_run_looks_like_refusal(self, run):
        if not isinstance(run, dict):
            return False
        return (
            self._prompt_looks_like_refusal(run.get("prompt", "")) or
            self._prompt_looks_like_refusal(run.get("encoded_prompt", ""))
        )

    def _get_conditioning_seq_len(self, conditioning: torch.Tensor) -> int:
        if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1:
            return 0
        return conditioning.shape[1] if conditioning.dim() == 3 else conditioning.shape[0]

    def _get_conditioning_token_mask(self, conditioning: torch.Tensor):
        if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1:
            return None

        if conditioning.dim() == 3:
            token_energy = conditioning.detach().abs().sum(dim=-1)
            mask = token_energy.gt(1e-12).any(dim=0)
        else:
            token_energy = conditioning.detach().abs().sum(dim=-1)
            mask = token_energy.gt(1e-12)

        if not bool(mask.any()):
            seq_len = self._get_conditioning_seq_len(conditioning)
            return torch.ones(seq_len, dtype=torch.bool, device=conditioning.device)

        return mask

    def _get_effective_seq_len(self, token_mask, fallback_seq_len: int) -> int:
        if token_mask is None:
            return fallback_seq_len
        active_positions = torch.nonzero(token_mask, as_tuple=False).flatten()
        if active_positions.numel() == 0:
            return fallback_seq_len
        return min(fallback_seq_len, int(active_positions[-1].item()) + 1)

    def _mask_to_embedding_dims(self, token_mask, reference: torch.Tensor):
        if token_mask is None or not isinstance(reference, torch.Tensor) or reference.dim() <= 1:
            return None

        token_mask = token_mask.to(device=reference.device, dtype=reference.dtype)
        if reference.dim() == 3:
            return token_mask.view(1, -1, 1)
        return token_mask.view(-1, 1)

    def _masked_sequence_mean(self, conditioning: torch.Tensor, token_mask):
        if not isinstance(conditioning, torch.Tensor):
            return conditioning
        if conditioning.dim() <= 1 or token_mask is None:
            return conditioning

        mask = self._mask_to_embedding_dims(token_mask, conditioning)
        if conditioning.dim() == 3:
            denom = mask.sum(dim=1).clamp_min(1.0)
            return (conditioning * mask).sum(dim=1) / denom

        denom = mask.sum(dim=0).clamp_min(1.0)
        return (conditioning * mask).sum(dim=0) / denom

    def _tokenize_ids(self, tokenizer, text: str, add_special_tokens: bool, max_length: Optional[int] = None):
        if not tokenizer or not text:
            return []

        kwargs = {"add_special_tokens": add_special_tokens}
        if max_length is not None and max_length > 0:
            kwargs["truncation"] = True
            kwargs["max_length"] = max_length

        try:
            tokenized = tokenizer(text, **kwargs)
            input_ids = tokenized.get("input_ids", tokenized)
            if isinstance(input_ids, list) and input_ids and isinstance(input_ids[0], list):
                input_ids = input_ids[0]
            return input_ids if isinstance(input_ids, list) else list(input_ids)
        except Exception:
            try:
                return tokenizer.encode(text, **kwargs)
            except Exception:
                return []

    def _iter_prompt_segments(self, prompt: str):
        if not prompt:
            return

        last_end = 0
        for match in PROTECTED_PHRASE_RE.finditer(prompt):
            start, end = match.span()
            if start > last_end:
                yield ("text", prompt[last_end:start])
            protected_text = (match.group(1) or match.group(2) or match.group(3) or "").strip()
            if protected_text:
                yield ("protected", protected_text)
            last_end = end

        if last_end < len(prompt):
            yield ("text", prompt[last_end:])

    def _mask_quoted_text(self, prompt: str):
        if not prompt:
            return ""
        return PROTECTED_PHRASE_RE.sub(lambda match: " " * len(match.group(0)), prompt)

    def _fallback_word_groups_from_prompt(self, prompt: str, seq_len: int, token_mask=None, existing_groups=None):
        if not prompt or seq_len <= 0:
            return []

        effective_seq_len = self._get_effective_seq_len(token_mask, seq_len)
        if effective_seq_len <= 0:
            return []

        active_positions = list(range(effective_seq_len))
        if token_mask is not None:
            active_positions = [
                index
                for index, enabled in enumerate(token_mask[:effective_seq_len].detach().cpu().tolist())
                if enabled
            ]
        if not active_positions:
            return []

        existing = {
            str(group[2]).strip().lower()
            for group in (existing_groups or [])
            if isinstance(group, (list, tuple)) and len(group) >= 3
        }
        candidates = []
        seen = set(existing)
        for segment_type, segment_text in self._iter_prompt_segments(prompt):
            if segment_type == "protected":
                pieces = [segment_text.strip()]
            else:
                pieces = re.findall(r"[\w'’.-]+", segment_text, flags=re.UNICODE)
            for piece in pieces:
                clean = piece.strip(" \t\r\n,;:!?()[]{}\"'")
                lower = clean.lower()
                if lower in seen or not self._is_valuable_token(clean):
                    continue
                seen.add(lower)
                candidates.append(clean)

        if not candidates:
            return []

        fallback = []
        step_count = len(candidates)
        max_slots = len(active_positions)
        for index, word in enumerate(candidates[:max_slots]):
            if step_count == 1:
                pos_index = len(active_positions) // 2
            else:
                pos_index = round(index * (len(active_positions) - 1) / max(1, min(step_count, max_slots) - 1))
            start = int(active_positions[max(0, min(len(active_positions) - 1, pos_index))])
            fallback.append((start, min(effective_seq_len, start + 1), word, []))
        return fallback

    def _build_word_groups(self, prompt: str, tokenizer, seq_len: int, token_mask=None):
        if not prompt or seq_len <= 0:
            return []

        if not tokenizer:
            return self._fallback_word_groups_from_prompt(prompt, seq_len, token_mask)

        effective_seq_len = self._get_effective_seq_len(token_mask, seq_len)
        token_mask_list = None
        if token_mask is not None:
            token_mask_list = token_mask[:effective_seq_len].detach().cpu().tolist()

        full_token_ids = self._tokenize_ids(
            tokenizer,
            prompt,
            add_special_tokens=True,
            max_length=effective_seq_len
        )[:effective_seq_len]

        word_groups = []
        grouped_seen = set()
        for segment_type, segment_text in self._iter_prompt_segments(prompt):
            if segment_type != "protected":
                continue

            clean_phrase = segment_text.strip()
            lower = clean_phrase.lower()
            if lower in grouped_seen or len(clean_phrase) < 3 or not any(c.isalpha() for c in clean_phrase):
                continue

            grouped_seen.add(lower)
            phrase_token_list = self._tokenize_ids(
                tokenizer,
                clean_phrase,
                add_special_tokens=False
            )
            if not phrase_token_list:
                continue

            for start in range(max(0, len(full_token_ids) - len(phrase_token_list) + 1)):
                end = start + len(phrase_token_list)
                if token_mask_list is not None and not all(token_mask_list[start:min(effective_seq_len, end)]):
                    continue
                if full_token_ids[start:end] == phrase_token_list and start < effective_seq_len:
                    word_groups.append((start, min(effective_seq_len, end), clean_phrase, phrase_token_list))
                    break

        raw_words = [w.strip() for w in self._mask_quoted_text(prompt).split() if w.strip()]
        for word in raw_words:
            clean_word = word
            lower = clean_word.lower()
            if lower in grouped_seen or len(clean_word) < 3 or not self._is_valuable_token(clean_word):
                continue

            grouped_seen.add(lower)
            word_token_list = self._tokenize_ids(
                tokenizer,
                clean_word,
                add_special_tokens=False
            )
            if not word_token_list:
                continue

            found = False
            for start in range(max(0, len(full_token_ids) - len(word_token_list) + 1)):
                end = start + len(word_token_list)
                if token_mask_list is not None and not all(token_mask_list[start:min(effective_seq_len, end)]):
                    continue
                if full_token_ids[start:end] == word_token_list and start < effective_seq_len:
                    word_groups.append((start, min(effective_seq_len, end), clean_word, word_token_list))
                    found = True
                    break

            if not found:
                continue

        if len(word_groups) < 2:
            word_groups.extend(
                self._fallback_word_groups_from_prompt(
                    prompt,
                    seq_len,
                    token_mask,
                    existing_groups=word_groups,
                )
            )

        return sorted(
            [group for group in word_groups if group[1] > group[0]],
            key=lambda group: (group[0], group[1], group[2].lower()),
        )

    def _void_empty_status(self, enabled=False):
        return f"Void: {'on' if enabled else 'off'} | idle | tokens: none"

    def _void_token_embedding(self, conditioning: torch.Tensor, start: int, end: int):
        if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1 or end <= start:
            return None
        if conditioning.dim() == 3:
            return conditioning[:, start:end, :].detach().mean(dim=(0, 1))
        return conditioning[start:end, :].detach().mean(dim=0)

    def _void_bank_score(self, item):
        likes = float(item.get("liked_count", 0))
        neutral = float(item.get("neutral_count", 0))
        awful = float(item.get("awful_count", 0))
        wanted = float(item.get("wanted_count", 0))
        missing = float(item.get("missing_count", 0))
        concept_missing = float(item.get("missing_concept_count", 0))
        detail_missing = float(item.get("missing_detail_count", 0))
        quality_missing = float(item.get("missing_quality_count", 0))
        missing_pressure = missing * 0.22 + concept_missing * 0.14 + detail_missing * 0.10 + quality_missing * 0.08
        return float(item.get("score", 0.0)) + likes * 0.40 + neutral * 0.08 + wanted * 0.30 + missing_pressure - awful * 0.95

    def _missing_axes_for_rating_key(self, rating_key):
        for profile in RATING_PROFILES.values():
            if profile.get("key") == rating_key:
                return set(profile.get("missing_axes", []))
        return set()

    def _axis_counter_name(self, axis):
        if axis == "concept":
            return "missing_concept_count"
        if axis == "details":
            return "missing_detail_count"
        if axis == "quality":
            return "missing_quality_count"
        return None

    def _axis_memory_boost(self, missing_axes):
        axes = set(missing_axes or [])
        boost = 0.0
        if "concept" in axes:
            boost += 0.42
        if "details" in axes:
            boost += 0.30
        if "quality" in axes:
            boost += 0.24
        if len(axes) >= 2:
            boost += 0.16
        return boost

    def _apply_missing_memory_pressure(self, item, missing_axes, iter_num):
        axes = set(missing_axes or [])
        if not isinstance(item, dict) or not axes:
            return item
        item["wanted_count"] = max(0, int(item.get("wanted_count", 0)) + 1)
        item["missing_count"] = max(0, int(item.get("missing_count", 0)) + 1)
        for axis in axes:
            counter = self._axis_counter_name(axis)
            if counter:
                item[counter] = max(0, int(item.get(counter, 0)) + 1)
        item["last_missing_iter"] = int(iter_num)
        return item

    def _ensure_void_token_bank(self, global_adaptive):
        bank = global_adaptive.get("void_token_bank")
        if not isinstance(bank, dict):
            bank = {}
        if bank and global_adaptive.get("void_token_bank_validated") is True:
            return bank
        cleaned = {}
        for token, item in bank.items():
            if not isinstance(token, str) or not isinstance(item, dict):
                continue
            vector = item.get("embedding")
            if not isinstance(vector, dict):
                continue
            try:
                shape = vector.get("shape", [])
                if len(shape) != 1 or int(shape[0]) <= 0:
                    continue
                item["score"] = round(max(-2.0, min(8.0, float(item.get("score", 0.0)))), 4)
                item["liked_count"] = max(0, int(item.get("liked_count", 0)))
                item["neutral_count"] = max(0, int(item.get("neutral_count", 0)))
                item["awful_count"] = max(0, int(item.get("awful_count", 0)))
                item["wanted_count"] = max(0, int(item.get("wanted_count", 0)))
                item["missing_count"] = max(0, int(item.get("missing_count", 0)))
                item["missing_concept_count"] = max(0, int(item.get("missing_concept_count", 0)))
                item["missing_detail_count"] = max(0, int(item.get("missing_detail_count", 0)))
                item["missing_quality_count"] = max(0, int(item.get("missing_quality_count", 0)))
                item["last_seen_iter"] = max(0, int(item.get("last_seen_iter", 0)))
                item["last_missing_iter"] = max(0, int(item.get("last_missing_iter", 0)))
                item["sample_count"] = max(0, int(item.get("sample_count", 0)))
                cleaned[token] = item
            except (TypeError, ValueError):
                continue
        global_adaptive["void_token_bank"] = cleaned
        global_adaptive["void_token_bank_validated"] = True
        return cleaned

    def _void_pair_key(self, left_token, right_token):
        return f"{str(left_token).strip().lower()}\t{str(right_token).strip().lower()}"

    def _void_pair_score(self, item):
        likes = float(item.get("liked_count", 0))
        neutral = float(item.get("neutral_count", 0))
        awful = float(item.get("awful_count", 0))
        wanted = float(item.get("wanted_count", 0))
        missing = float(item.get("missing_count", 0))
        return float(item.get("score", 0.0)) + likes * 0.35 + neutral * 0.05 + wanted * 0.22 + missing * 0.16 - awful * 0.85

    def _ensure_void_pair_bank(self, global_adaptive):
        bank = global_adaptive.get("void_token_pairs")
        if not isinstance(bank, dict):
            bank = {}
        if bank and global_adaptive.get("void_token_pairs_validated") is True:
            return bank
        cleaned = {}
        for key, item in bank.items():
            if not isinstance(key, str) or "\t" not in key or not isinstance(item, dict):
                continue
            try:
                left, right = key.split("\t", 1)
                if not self._is_valuable_token(left) or not self._is_valuable_token(right):
                    continue
                item["score"] = round(max(-3.0, min(6.0, float(item.get("score", 0.0)))), 4)
                item["liked_count"] = max(0, int(item.get("liked_count", 0)))
                item["neutral_count"] = max(0, int(item.get("neutral_count", 0)))
                item["awful_count"] = max(0, int(item.get("awful_count", 0)))
                item["wanted_count"] = max(0, int(item.get("wanted_count", 0)))
                item["missing_count"] = max(0, int(item.get("missing_count", 0)))
                item["missing_concept_count"] = max(0, int(item.get("missing_concept_count", 0)))
                item["missing_detail_count"] = max(0, int(item.get("missing_detail_count", 0)))
                item["missing_quality_count"] = max(0, int(item.get("missing_quality_count", 0)))
                item["last_seen_iter"] = max(0, int(item.get("last_seen_iter", 0)))
                item["last_missing_iter"] = max(0, int(item.get("last_missing_iter", 0)))
                cleaned[key] = item
            except (TypeError, ValueError):
                continue
        global_adaptive["void_token_pairs"] = cleaned
        global_adaptive["void_token_pairs_validated"] = True
        return cleaned

    def _lucky_context_score(self, item):
        likes = float(item.get("liked_count", 0))
        neutral = float(item.get("neutral_count", 0))
        awful = float(item.get("awful_count", 0))
        wanted = float(item.get("wanted_count", 0))
        missing = float(item.get("missing_count", 0))
        concept_missing = float(item.get("missing_concept_count", 0))
        detail_missing = float(item.get("missing_detail_count", 0))
        return float(item.get("score", 0.0)) + likes * 0.38 + neutral * 0.06 + wanted * 0.24 + missing * 0.18 + concept_missing * 0.10 + detail_missing * 0.08 - awful * 0.90

    def _ensure_lucky_context_memory(self, global_adaptive):
        memory = global_adaptive.get("lucky_context_memory")
        if not isinstance(memory, dict):
            memory = {}
        if memory and global_adaptive.get("lucky_context_memory_validated") is True:
            return memory

        cleaned = {}
        for anchor, neighbors in memory.items():
            anchor = str(anchor).strip().lower()
            if not self._is_valuable_token(anchor) or not isinstance(neighbors, dict):
                continue

            clean_neighbors = {}
            for neighbor, item in neighbors.items():
                neighbor = str(neighbor).strip().lower()
                if neighbor == anchor or not self._is_valuable_token(neighbor) or not isinstance(item, dict):
                    continue
                try:
                    item["score"] = round(max(-4.0, min(8.0, float(item.get("score", 0.0)))), 4)
                    item["liked_count"] = max(0, int(item.get("liked_count", 0)))
                    item["neutral_count"] = max(0, int(item.get("neutral_count", 0)))
                    item["awful_count"] = max(0, int(item.get("awful_count", 0)))
                    item["wanted_count"] = max(0, int(item.get("wanted_count", 0)))
                    item["missing_count"] = max(0, int(item.get("missing_count", 0)))
                    item["missing_concept_count"] = max(0, int(item.get("missing_concept_count", 0)))
                    item["missing_detail_count"] = max(0, int(item.get("missing_detail_count", 0)))
                    item["missing_quality_count"] = max(0, int(item.get("missing_quality_count", 0)))
                    item["last_seen_iter"] = max(0, int(item.get("last_seen_iter", 0)))
                    item["last_missing_iter"] = max(0, int(item.get("last_missing_iter", 0)))
                    item["co_count"] = max(0, int(item.get("co_count", 0)))
                    clean_neighbors[neighbor] = item
                except (TypeError, ValueError):
                    continue

            if clean_neighbors:
                cleaned[anchor] = clean_neighbors

        global_adaptive["lucky_context_memory"] = cleaned
        global_adaptive["lucky_context_memory_validated"] = True
        return cleaned

    def _phrase_position_score(self, item):
        likes = float(item.get("liked_count", 0))
        neutral = float(item.get("neutral_count", 0))
        awful = float(item.get("awful_count", 0))
        wanted = float(item.get("wanted_count", 0))
        missing = float(item.get("missing_count", 0))
        return float(item.get("score", 0.0)) + likes * 0.45 + neutral * 0.06 + wanted * 0.26 + missing * 0.18 - awful * 0.85

    def _ensure_lucky_phrase_placements(self, global_adaptive):
        memory = global_adaptive.get("lucky_phrase_placements")
        if not isinstance(memory, dict):
            memory = {}
        if memory and global_adaptive.get("lucky_phrase_placements_validated") is True:
            return memory

        cleaned = {}
        for phrase, item in memory.items():
            phrase_key = str(phrase).strip().lower()
            if len(phrase_key) < 3 or not isinstance(item, dict):
                continue
            positions = item.get("positions")
            if not isinstance(positions, dict):
                continue
            clean_positions = {}
            for pos, pos_item in positions.items():
                if not isinstance(pos_item, dict):
                    continue
                try:
                    pos_key = str(max(0, min(31, int(pos))))
                    pos_item["score"] = round(max(-4.0, min(8.0, float(pos_item.get("score", 0.0)))), 4)
                    pos_item["liked_count"] = max(0, int(pos_item.get("liked_count", 0)))
                    pos_item["neutral_count"] = max(0, int(pos_item.get("neutral_count", 0)))
                    pos_item["awful_count"] = max(0, int(pos_item.get("awful_count", 0)))
                    pos_item["wanted_count"] = max(0, int(pos_item.get("wanted_count", 0)))
                    pos_item["missing_count"] = max(0, int(pos_item.get("missing_count", 0)))
                    pos_item["missing_concept_count"] = max(0, int(pos_item.get("missing_concept_count", 0)))
                    pos_item["missing_detail_count"] = max(0, int(pos_item.get("missing_detail_count", 0)))
                    pos_item["missing_quality_count"] = max(0, int(pos_item.get("missing_quality_count", 0)))
                    pos_item["count"] = max(0, int(pos_item.get("count", 0)))
                    pos_item["last_seen_iter"] = max(0, int(pos_item.get("last_seen_iter", 0)))
                    pos_item["last_missing_iter"] = max(0, int(pos_item.get("last_missing_iter", 0)))
                    clean_positions[pos_key] = pos_item
                except (TypeError, ValueError):
                    continue
            if not clean_positions:
                continue
            cleaned[phrase_key] = {
                "text": str(item.get("text", phrase_key)).strip().lower() or phrase_key,
                "positions": clean_positions,
            }

        global_adaptive["lucky_phrase_placements"] = cleaned
        global_adaptive["lucky_phrase_placements_validated"] = True
        return cleaned

    def _rating_memory_deltas(self, rating_key):
        if rating_key == "like":
            return 0.70, 1, 0, 0
        if rating_key == "missing_details":
            return 0.56, 0, 1, 0
        if rating_key == "missing_concept":
            return 0.72, 0, 1, 0
        if rating_key == "missing_quality":
            return 0.48, 0, 1, 0
        if rating_key == "missing_details_concept":
            return 0.84, 0, 1, 0
        if rating_key == "missing_details_quality":
            return 0.68, 0, 1, 0
        if rating_key == "missing_concept_quality":
            return 0.80, 0, 1, 0
        if rating_key == "awful":
            return -1.10, 0, 0, 1
        if rating_key == "discover":
            return 0.0, 0, 1, 0
        return 0.12, 0, 1, 0

    def _update_lucky_context_memory(self, global_adaptive, tokens, rating_key, iter_num):
        memory = self._ensure_lucky_context_memory(global_adaptive)
        ordered_tokens = []
        seen = set()
        for token in tokens or []:
            token = str(token).strip().lower()
            if token in seen or not self._is_valuable_token(token):
                continue
            seen.add(token)
            ordered_tokens.append(token)

        if len(ordered_tokens) < 2:
            return memory

        score_delta, liked_delta, neutral_delta, awful_delta = self._rating_memory_deltas(rating_key)
        missing_axes = self._missing_axes_for_rating_key(rating_key)
        if missing_axes:
            score_delta += self._axis_memory_boost(missing_axes) * 0.55
        # Keep context learning local and ordered. A full all-to-all update makes
        # long Lucky prompts create thousands of JSON entries per click.
        window = 6
        for index, anchor in enumerate(ordered_tokens):
            neighbors = memory.setdefault(anchor, {})
            start = max(0, index - window)
            end = min(len(ordered_tokens), index + window + 1)
            for neighbor_index in range(start, end):
                if neighbor_index == index:
                    continue
                neighbor = ordered_tokens[neighbor_index]
                distance = abs(neighbor_index - index)
                if distance <= 0:
                    continue
                local_weight = 1.0 / float(distance)
                item = neighbors.get(neighbor, {})
                item["score"] = round(max(-4.0, min(8.0, float(item.get("score", 0.0)) + score_delta * local_weight)), 4)
                item["liked_count"] = max(0, int(item.get("liked_count", 0)) + liked_delta)
                item["neutral_count"] = max(0, int(item.get("neutral_count", 0)) + neutral_delta)
                item["awful_count"] = max(0, int(item.get("awful_count", 0)) + awful_delta)
                if missing_axes:
                    item = self._apply_missing_memory_pressure(item, missing_axes, iter_num)
                item["co_count"] = max(0, int(item.get("co_count", 0)) + 1)
                item["last_seen_iter"] = int(iter_num)
                neighbors[neighbor] = item

        return memory

    def _update_lucky_phrase_placements(self, global_adaptive, prompt_text, rating_key, iter_num):
        phrases = self._ordered_prompt_phrases(prompt_text)
        if not phrases:
            return self._ensure_lucky_phrase_placements(global_adaptive)

        memory = self._ensure_lucky_phrase_placements(global_adaptive)
        score_delta, liked_delta, neutral_delta, awful_delta = self._rating_memory_deltas(rating_key)
        missing_axes = self._missing_axes_for_rating_key(rating_key)
        if missing_axes:
            score_delta += self._axis_memory_boost(missing_axes) * 0.70
        for index, phrase in enumerate(phrases[:32]):
            text = str(phrase.get("text", "")).strip().lower()
            if len(text) < 3:
                continue
            entry = memory.setdefault(text, {"text": text, "positions": {}})
            positions = entry.setdefault("positions", {})
            pos_key = str(index)
            item = positions.get(pos_key, {})
            item["score"] = round(max(-4.0, min(8.0, float(item.get("score", 0.0)) + score_delta)), 4)
            item["liked_count"] = max(0, int(item.get("liked_count", 0)) + liked_delta)
            item["neutral_count"] = max(0, int(item.get("neutral_count", 0)) + neutral_delta)
            item["awful_count"] = max(0, int(item.get("awful_count", 0)) + awful_delta)
            if missing_axes:
                item = self._apply_missing_memory_pressure(item, missing_axes, iter_num)
            item["count"] = max(0, int(item.get("count", 0)) + 1)
            item["last_seen_iter"] = int(iter_num)
            positions[pos_key] = item
            entry["text"] = text
            memory[text] = entry

        global_adaptive["lucky_phrase_placements"] = memory
        return memory

    def _update_void_token_pairs(self, global_adaptive, word_groups, rating_key, iter_num):
        pair_bank = self._ensure_void_pair_bank(global_adaptive)
        ordered_tokens = []
        seen_positions = set()
        for start, _, full_word, _ in sorted(word_groups or [], key=lambda group: (group[0], group[1])):
            token = str(full_word).strip().lower()
            if not self._is_valuable_token(token):
                continue
            marker = (int(start), token)
            if marker in seen_positions:
                continue
            seen_positions.add(marker)
            ordered_tokens.append(token)

        if len(ordered_tokens) < 2:
            return pair_bank

        base_delta, liked_delta, neutral_delta, awful_delta = self._rating_memory_deltas(rating_key)
        missing_axes = self._missing_axes_for_rating_key(rating_key)
        score_delta = base_delta * 0.84
        if missing_axes:
            score_delta += self._axis_memory_boost(missing_axes) * 0.48

        for left, right in zip(ordered_tokens, ordered_tokens[1:]):
            if left == right:
                continue
            key = self._void_pair_key(left, right)
            item = pair_bank.get(key, {})
            item["left"] = left
            item["right"] = right
            item["score"] = round(max(-3.0, min(6.0, float(item.get("score", 0.0)) + score_delta)), 4)
            item["liked_count"] = max(0, int(item.get("liked_count", 0)) + liked_delta)
            item["neutral_count"] = max(0, int(item.get("neutral_count", 0)) + neutral_delta)
            item["awful_count"] = max(0, int(item.get("awful_count", 0)) + awful_delta)
            if missing_axes:
                item = self._apply_missing_memory_pressure(item, missing_axes, iter_num)
            item["last_seen_iter"] = int(iter_num)
            pair_bank[key] = item

        self._update_lucky_context_memory(global_adaptive, ordered_tokens, rating_key, iter_num)
        global_adaptive["void_token_pairs"] = pair_bank
        return global_adaptive["void_token_pairs"]

    def _void_pair_is_poor(self, global_adaptive, left_token, right_token):
        if not left_token or not right_token:
            return False
        pair_bank = global_adaptive.get("void_token_pairs")
        if not isinstance(pair_bank, dict):
            return False
        item = pair_bank.get(self._void_pair_key(left_token, right_token))
        if not isinstance(item, dict):
            return False
        awful_count = int(item.get("awful_count", 0))
        positive_count = int(item.get("liked_count", 0)) + int(item.get("neutral_count", 0))
        return awful_count > 0 and self._void_pair_score(item) < -0.05 and awful_count >= max(1, positive_count)

    def _update_void_token_bank(self, global_adaptive, word_groups, cur_positive,
                                rating_profile, iter_num, token_mask=None):
        bank = self._ensure_void_token_bank(global_adaptive)
        rating_key = rating_profile.get("key", "")
        if not word_groups or not isinstance(cur_positive, torch.Tensor) or rating_profile.get("skip_learning"):
            return bank

        score_delta, liked_delta, neutral_delta, awful_delta = self._rating_memory_deltas(rating_key)
        missing_axes = set(rating_profile.get("missing_axes", []))
        if missing_axes:
            score_delta += self._axis_memory_boost(missing_axes)

        token_mask_list = None
        if token_mask is not None:
            token_mask_list = token_mask.detach().cpu().tolist()

        for start, end, full_word, _ in word_groups:
            if not self._is_valuable_token(full_word):
                continue
            if token_mask_list is not None and not all(token_mask_list[start:min(len(token_mask_list), end)]):
                continue
            embedding = self._void_token_embedding(cur_positive, start, end)
            if embedding is None:
                continue

            token = full_word.strip().lower()
            item = bank.get(token, {})
            count = max(0, int(item.get("sample_count", 0)))
            if "embedding" in item and count > 0 and rating_key != "awful":
                try:
                    previous = serializable_to_tensor(item["embedding"]).to(embedding.device)
                    if list(previous.shape) == list(embedding.shape):
                        if rating_key == "like":
                            mix = 0.62
                        elif rating_key == "missing_details":
                            mix = 0.48
                        else:
                            mix = min(0.30, 1.0 / float(count + 1))
                        embedding = previous.lerp(embedding, mix)
                except Exception:
                    pass

            item["embedding"] = tensor_to_serializable(embedding.float().cpu())
            item["score"] = round(max(-2.0, min(8.0, float(item.get("score", 0.0)) + score_delta)), 4)
            item["liked_count"] = max(0, int(item.get("liked_count", 0)) + liked_delta)
            item["neutral_count"] = max(0, int(item.get("neutral_count", 0)) + neutral_delta)
            item["awful_count"] = max(0, int(item.get("awful_count", 0)) + awful_delta)
            if missing_axes:
                item = self._apply_missing_memory_pressure(item, missing_axes, iter_num)
            item["sample_count"] = count + (0 if rating_key == "awful" else 1)
            item["last_seen_iter"] = int(iter_num)
            bank[token] = item

        self._update_void_token_pairs(global_adaptive, word_groups, rating_key, iter_num)

        global_adaptive["void_token_bank"] = bank
        return global_adaptive["void_token_bank"]

    def _seed_void_token_bank(self, global_adaptive, word_groups, cur_positive, iter_num, token_mask=None):
        return self._update_void_token_bank(
            global_adaptive,
            word_groups,
            cur_positive,
            {"key": "discover", "skip_learning": False},
            iter_num,
            token_mask=token_mask,
        )

    def _lucky_groups_from_history(self, history_entry, seq_len):
        if not isinstance(history_entry, dict) or seq_len <= 0:
            return []
        lucky = history_entry.get("lucky") if isinstance(history_entry.get("lucky"), dict) else {}
        injections = lucky.get("injections", [])
        groups = []
        seen = set()
        for item in injections:
            if not isinstance(item, dict):
                continue
            token = str(item.get("token", "")).strip().lower()
            if not self._is_valuable_token(token):
                continue
            try:
                start = int(item.get("start", 0))
                end = int(item.get("end", start + 1))
            except (TypeError, ValueError):
                continue
            start = max(0, min(seq_len - 1, start))
            end = max(start + 1, min(seq_len, end))
            marker = (start, token)
            if marker in seen:
                continue
            seen.add(marker)
            groups.append((start, end, token, []))
        return groups

    def _history_modified_conditioning(self, history_entry, reference, device):
        if not isinstance(history_entry, dict):
            return None
        mod_data = history_entry.get("modified_embeds")
        if mod_data is None:
            return None
        try:
            candidate = serializable_to_tensor(mod_data).to(device)
        except Exception:
            return None
        if not isinstance(reference, torch.Tensor) or list(candidate.shape) != list(reference.shape):
            return None
        return candidate

    def _history_modified_conditioning_any_shape(self, history_entry, device, dtype=None):
        if not isinstance(history_entry, dict):
            return None
        mod_data = history_entry.get("modified_embeds")
        if mod_data is None:
            return None
        try:
            candidate = serializable_to_tensor(mod_data).to(device)
            if dtype is not None:
                candidate = candidate.to(dtype=dtype)
        except Exception:
            return None
        if not isinstance(candidate, torch.Tensor) or candidate.dim() <= 1:
            return None
        return candidate

    def _conditioning_canvas_compatible(self, candidate, template):
        if (
            not isinstance(candidate, torch.Tensor) or
            not isinstance(template, torch.Tensor) or
            candidate.dim() <= 1 or
            template.dim() <= 1 or
            candidate.dim() != template.dim() or
            int(candidate.shape[-1]) != int(template.shape[-1])
        ):
            return False
        if candidate.dim() == 3 and int(candidate.shape[0]) != int(template.shape[0]):
            return False
        return self._get_conditioning_seq_len(candidate) > 0

    def _resize_conditioning_sequence_like(self, source, target):
        if not isinstance(source, torch.Tensor) or not isinstance(target, torch.Tensor):
            return None
        if source.dim() != target.dim() or source.dim() <= 1:
            return None
        if int(source.shape[-1]) != int(target.shape[-1]):
            return None
        if source.dim() == 3 and int(source.shape[0]) != int(target.shape[0]):
            return None
        if list(source.shape) == list(target.shape):
            return source.to(device=target.device, dtype=target.dtype)

        target_seq = self._get_conditioning_seq_len(target)
        if target_seq <= 0:
            return None

        try:
            working = source.to(device=target.device, dtype=target.dtype)
            if working.dim() == 3:
                resized = F.interpolate(
                    working.transpose(1, 2),
                    size=target_seq,
                    mode="linear",
                    align_corners=False,
                ).transpose(1, 2)
            else:
                resized = F.interpolate(
                    working.transpose(0, 1).unsqueeze(0),
                    size=target_seq,
                    mode="linear",
                    align_corners=False,
                ).squeeze(0).transpose(0, 1)
        except Exception:
            return None

        if list(resized.shape) != list(target.shape):
            return None
        return resized

    def _serialized_tensor_shape(self, payload):
        if not isinstance(payload, dict):
            return None
        shape = payload.get("shape")
        if not isinstance(shape, list) or not shape:
            return None
        try:
            return tuple(int(dim) for dim in shape)
        except (TypeError, ValueError):
            return None

    def _serialized_conditioning_compatible(self, payload, template):
        shape = self._serialized_tensor_shape(payload)
        if (
            shape is None or
            not isinstance(template, torch.Tensor) or
            template.dim() <= 1 or
            len(shape) != int(template.dim()) or
            int(shape[-1]) != int(template.shape[-1])
        ):
            return False
        if len(shape) == 3 and int(shape[0]) != int(template.shape[0]):
            return False
        return (shape[1] if len(shape) == 3 else shape[0]) > 0

    def _serialized_conditioning_seq_len(self, payload):
        shape = self._serialized_tensor_shape(payload)
        if shape is None or len(shape) <= 1:
            return 0
        return int(shape[1] if len(shape) == 3 else shape[0])

    def _select_lucky_memory_canvas(self, prompt_histories, template, device, dtype):
        if not isinstance(prompt_histories, dict) or not isinstance(template, torch.Tensor):
            return None, ""

        candidates = []
        current_seq = self._get_conditioning_seq_len(template)

        def consider(payload, label, quality=0.0, iteration=0):
            if payload is None:
                return
            if not self._serialized_conditioning_compatible(payload, template):
                return
            seq_len = self._serialized_conditioning_seq_len(payload)
            candidates.append({
                "payload": payload,
                "label": label,
                "seq_len": int(seq_len),
                "quality": float(quality),
                "iteration": int(iteration or 0),
            })

        for prompt_key, prompt_state in prompt_histories.items():
            if not isinstance(prompt_state, dict):
                continue
            short_key = str(prompt_key)[:48] or "prompt"
            liked_count = int(prompt_state.get("liked_reference_count", 0) or 0)
            if liked_count > 0:
                consider(
                    prompt_state.get("liked_reference_embeds"),
                    f"liked '{short_key}'",
                    quality=2.0 + min(3.0, liked_count * 0.25),
                    iteration=prompt_state.get("liked_reference_last_iteration", 0),
                )
            consider(prompt_state.get("reference_embeds"), f"reference '{short_key}'", quality=0.5)
            consider(prompt_state.get("source_conditioning_embeds"), f"source '{short_key}'", quality=0.25)

            for entry in prompt_state.get("history", []) or []:
                if not isinstance(entry, dict):
                    continue
                profile = normalize_refiner_rating(entry.get("rating_label", entry.get("rating", 0)))
                key = profile.get("key", "")
                if key in {"forget", "awful"}:
                    continue
                quality = float(profile.get("reward", 0.0))
                if key == "like":
                    quality += 1.5
                elif key.startswith("missing"):
                    quality += 0.45
                consider(
                    entry.get("modified_embeds"),
                    f"history {entry.get('iteration', 0)} '{short_key}'",
                    quality=quality,
                    iteration=entry.get("iteration", 0),
                )

        if not candidates:
            return None, ""

        best = max(
            candidates,
            key=lambda item: (
                item["seq_len"] >= current_seq,
                item["seq_len"],
                item["quality"],
                item["iteration"],
            )
        )
        try:
            canvas = serializable_to_tensor(best["payload"]).to(device=device, dtype=dtype)
        except Exception:
            return None, ""
        if not self._conditioning_canvas_compatible(canvas, template):
            return None, ""
        return canvas.clone(), f"{best['label']} ({best['seq_len']} positions)"

    def _legacy_auto_inject_token_allowed(self, token):
        token = str(token or "").strip().lower()
        if not token:
            return False
        category = self._infer_concept_category([token])
        if category in {"appearance", "subject", "environment", "character"}:
            return False
        return True

    def _eligible_void_bank_items(self, global_adaptive, embedding_dim):
        bank = self._ensure_void_token_bank(global_adaptive)
        eligible = []
        for token, item in bank.items():
            if not self._legacy_auto_inject_token_allowed(token):
                continue
            if int(item.get("awful_count", 0)) >= 2 and int(item.get("liked_count", 0)) <= 0:
                continue
            score = self._void_bank_score(item)
            if score <= 0.35:
                continue
            try:
                vector = serializable_to_tensor(item["embedding"]).float()
            except Exception:
                continue
            if vector.dim() != 1 or int(vector.shape[0]) != int(embedding_dim):
                continue
            eligible.append((token, item, score))
        return eligible

    def _eligible_lucky_bank_items(self, global_adaptive, embedding_dim):
        bank = self._ensure_void_token_bank(global_adaptive)
        candidates = []
        for token, item in bank.items():
            if not self._legacy_auto_inject_token_allowed(token):
                continue
            score = self._void_bank_score(item)
            embedding_shape = self._serialized_tensor_shape(item.get("embedding"))
            if embedding_shape is None or len(embedding_shape) != 1 or int(embedding_shape[0]) != int(embedding_dim):
                continue
            candidates.append((token, item, score))

        return candidates

    def _lucky_prompt_anchor_tokens(self, word_groups, token_vectors, global_adaptive):
        context_memory = self._ensure_lucky_context_memory(global_adaptive)
        anchors = []
        seen = set()
        for _, _, full_word, _ in sorted(word_groups or [], key=lambda group: (group[0], group[1])):
            token = str(full_word).strip().lower()
            if token in seen or not self._is_valuable_token(token):
                continue
            if token in token_vectors or token in context_memory:
                anchors.append(token)
                seen.add(token)
        return anchors

    def _lucky_context_neighbors(self, global_adaptive, anchor_tokens, token_vectors, present_tokens=None):
        context_memory = self._ensure_lucky_context_memory(global_adaptive)
        present = set(present_tokens or [])
        scored = {}
        for anchor in anchor_tokens or []:
            for neighbor, item in context_memory.get(anchor, {}).items():
                if neighbor in present or neighbor not in token_vectors:
                    continue
                score = self._lucky_context_score(item)
                if score <= 0.15 and int(item.get("liked_count", 0)) <= 0:
                    continue
                if self._void_pair_is_poor(global_adaptive, anchor, neighbor):
                    continue
                scored[neighbor] = max(scored.get(neighbor, -999.0), score)
        return [
            token for token, _ in sorted(scored.items(), key=lambda pair: pair[1], reverse=True)
        ]

    def _lucky_global_tokens(self, lucky_pool):
        scored = sorted(lucky_pool, key=lambda item: item[2], reverse=True)
        preferred = [
            token for token, item, score in scored
            if score > 0.10 or int(item.get("liked_count", 0)) > 0 or int(item.get("neutral_count", 0)) > 0
        ]
        return preferred or [token for token, _, _ in scored]

    def _lucky_weighted_pick(self, token_scores, candidates, previous_token=None, global_adaptive=None):
        usable = []
        weights = []
        for token in candidates or []:
            token = str(token).strip().lower()
            if not token:
                continue
            if previous_token and global_adaptive is not None and self._void_pair_is_poor(global_adaptive, previous_token, token):
                continue
            usable.append(token)
            weights.append(max(0.05, float(token_scores.get(token, 0.0)) + 1.25))

        if not usable:
            return None

        if len(usable) == 1:
            return usable[0]

        weights_tensor = torch.tensor(weights, dtype=torch.float32)
        index = int(torch.multinomial(weights_tensor, 1).item())
        return usable[index]

    def _lucky_compose_tokens(self, global_adaptive, word_groups, lucky_pool, token_vectors, slot_count):
        token_scores = {token: score for token, _, score in lucky_pool}
        token_items = {token: item for token, item, _ in lucky_pool}
        present_tokens = {
            str(group[2]).strip().lower()
            for group in (word_groups or [])
            if isinstance(group, (list, tuple)) and len(group) >= 3 and self._is_valuable_token(str(group[2]).strip())
        }
        anchor_tokens = self._lucky_prompt_anchor_tokens(word_groups, token_vectors, global_adaptive)
        context_tokens = self._lucky_context_neighbors(
            global_adaptive,
            anchor_tokens,
            token_vectors,
            present_tokens=present_tokens,
        )
        global_tokens = self._lucky_global_tokens(lucky_pool)

        seed_sequence = []
        for token in anchor_tokens + context_tokens + global_tokens:
            if token in token_vectors and token not in seed_sequence:
                seed_sequence.append(token)

        if not seed_sequence:
            seed_sequence = [token for token, _, _ in lucky_pool]

        max_slots = max(0, int(slot_count))
        required_candidates = []
        for index, token in enumerate(anchor_tokens):
            item = token_items.get(token, {})
            missing_pressure = (
                int(item.get("wanted_count", 0)) +
                int(item.get("missing_count", 0)) +
                int(item.get("missing_concept_count", 0)) +
                int(item.get("missing_detail_count", 0)) +
                int(item.get("missing_quality_count", 0))
            )
            # Current prompt anchors are required; missing pressure decides which
            # ones win when the conditioning budget is tight.
            priority = float(token_scores.get(token, 0.0)) + missing_pressure * 0.55 + 0.12 / float(index + 1)
            required_candidates.append((priority, index, token))
        required_candidates = sorted(required_candidates, key=lambda item: (-item[0], item[1]))
        reserve_count = min(len(required_candidates), max_slots)
        if reserve_count > 3:
            reserve_count = min(reserve_count, max(3, min(16, int(math.ceil(max_slots * 0.40)))))
        required_tokens = [
            token for _, _, token in sorted(required_candidates[:reserve_count], key=lambda item: item[1])
        ]

        chosen_tokens = list(required_tokens)
        previous_token = None
        context_hits = 0
        if chosen_tokens:
            previous_token = chosen_tokens[-1]
        for index in range(len(chosen_tokens), max_slots):
            context_index = index - len(required_tokens)
            if context_tokens and 0 <= context_index < len(context_tokens):
                candidates = [context_tokens[context_index]]
            elif seed_sequence and random.random() < 0.72:
                candidates = seed_sequence
            else:
                candidates = [token for token, _, _ in lucky_pool]

            token = self._lucky_weighted_pick(token_scores, candidates, previous_token, global_adaptive)
            if token is None:
                token = self._lucky_weighted_pick(token_scores, [token for token, _, _ in lucky_pool], previous_token, global_adaptive)
            if token is None:
                continue

            if token in context_tokens:
                context_hits += 1
            chosen_tokens.append(token)
            previous_token = token

        if anchor_tokens and context_hits:
            source = "anchors+context"
        elif anchor_tokens:
            source = "prompt anchors"
        elif context_hits:
            source = "context"
        else:
            source = "global preferences"

        return chosen_tokens, {
            "source": source,
            "anchor_tokens": anchor_tokens,
            "required_tokens": required_tokens,
            "context_tokens": context_tokens,
            "context_hits": context_hits,
        }

    def _compose_lucky_prompt_text(self, global_adaptive, word_groups, embedding_dim,
                                   prompt_sequences=None, phrase_sequences=None, target_count=64):
        bank = self._ensure_void_token_bank(global_adaptive)
        lucky_pool = []
        for token, item in bank.items():
            if not self._legacy_auto_inject_token_allowed(token):
                continue
            embedding_shape = self._serialized_tensor_shape(item.get("embedding"))
            if embedding_shape is None or len(embedding_shape) != 1 or int(embedding_shape[0]) != int(embedding_dim):
                continue
            lucky_pool.append((token, item, self._void_bank_score(item)))

        if len(lucky_pool) < 2:
            return "", {
                "source": "bank too small",
                "anchor_tokens": [],
                "context_tokens": [],
                "context_hits": 0,
                "token_count": 0,
            }

        token_vectors = {token: True for token, _, _ in lucky_pool}

        if len(token_vectors) < 2:
            return "", {
                "source": "bank unreadable",
                "anchor_tokens": [],
                "context_tokens": [],
                "context_hits": 0,
                "token_count": 0,
            }

        count = max(4, min(int(target_count or 0), 32))
        chosen_tokens, compose_info = self._lucky_compose_tokens(
            global_adaptive,
            word_groups,
            lucky_pool,
            token_vectors,
            count,
        )

        ordered_sequences = []
        if isinstance(prompt_sequences, list):
            for sequence in prompt_sequences:
                if not isinstance(sequence, list):
                    continue
                ordered = [str(token).strip().lower() for token in sequence if str(token).strip().lower() in token_vectors]
                if len(ordered) >= 2:
                    ordered_sequences.append(ordered)

        if ordered_sequences and compose_info.get("source") == "global preferences":
            sequence = random.choice(ordered_sequences)
            chosen_tokens = [
                sequence[index % len(sequence)]
                if random.random() < 0.55 else chosen_tokens[index]
                for index in range(min(len(chosen_tokens), count))
            ]
            compose_info["source"] = "saved prompt order"

        token_scores = {token: score for token, _, score in lucky_pool}
        prompt_tokens = []
        seen = set()
        for token in chosen_tokens:
            token = str(token).strip().lower()
            if token in seen or not self._is_valuable_token(token):
                continue
            prompt_tokens.append(token)
            seen.add(token)

        if not prompt_tokens:
            return "", {
                **compose_info,
                "token_count": 0,
            }

        prompt_phrases = []
        phrase_seen = set()
        chosen_set = set(prompt_tokens)
        if isinstance(phrase_sequences, list):
            phrase_candidates = []
            placement_memory = self._ensure_lucky_phrase_placements(global_adaptive)
            for sequence_index, sequence in enumerate(phrase_sequences[-64:]):
                if not isinstance(sequence, list):
                    continue
                for phrase_index, phrase in enumerate(sequence):
                    if not isinstance(phrase, dict):
                        continue
                    text = str(phrase.get("text", "")).strip()
                    if not text:
                        continue
                    tokens = [
                        str(token).strip().lower()
                        for token in phrase.get("tokens", [])
                        if self._is_valuable_token(str(token).strip())
                    ]
                    if not tokens:
                        continue
                    overlap = chosen_set & set(tokens)
                    if not overlap:
                        continue
                    score = sum(max(0.05, token_scores.get(token, 0.0) + 1.25) for token in overlap)
                    score += 0.18 / float(phrase_index + 1)
                    score += 0.03 / float(max(1, len(phrase_sequences) - sequence_index))
                    preferred_position, placement_score = self._lucky_phrase_preferred_position(
                        placement_memory,
                        text,
                        fallback_position=phrase_index,
                    )
                    score += max(0.0, placement_score) * 0.20
                    phrase_candidates.append((score, preferred_position, phrase_index, text, tokens))

            sorted_candidates = sorted(phrase_candidates, key=lambda item: item[0], reverse=True)
            selected = []
            for score, preferred_position, phrase_index, text, tokens in sorted_candidates:
                key = text.lower()
                if key in phrase_seen:
                    continue
                selected.append({
                    "score": score,
                    "position": preferred_position,
                    "source_position": phrase_index,
                    "text": text,
                    "tokens": tokens,
                })
                phrase_seen.add(key)
                if len(selected) >= 12:
                    break
            prompt_phrases = [
                item["text"]
                for item in sorted(
                    selected,
                    key=lambda item: (item["position"], -item["score"], item["source_position"], item["text"]),
                )
            ]

        if not prompt_phrases:
            prompt_phrases = prompt_tokens

        prompt_text = ", ".join(prompt_phrases)
        return prompt_text, {
            **compose_info,
            "token_count": len(prompt_tokens),
            "tokens": prompt_tokens,
            "phrases": prompt_phrases,
        }

    def _lucky_phrase_preferred_position(self, placement_memory, phrase_text, fallback_position=0):
        phrase_key = str(phrase_text).strip().lower()
        entry = placement_memory.get(phrase_key) if isinstance(placement_memory, dict) else None
        if not isinstance(entry, dict):
            return int(fallback_position), 0.0
        positions = entry.get("positions")
        if not isinstance(positions, dict) or not positions:
            return int(fallback_position), 0.0

        candidates = []
        for pos_key, item in positions.items():
            if not isinstance(item, dict):
                continue
            try:
                pos = max(0, min(31, int(pos_key)))
            except (TypeError, ValueError):
                continue
            score = self._phrase_position_score(item)
            count_bonus = min(0.30, math.log1p(float(item.get("count", 0))) * 0.08)
            candidates.append((score + count_bonus, pos))

        if not candidates:
            return int(fallback_position), 0.0

        candidates = sorted(candidates, key=lambda item: item[0], reverse=True)
        top_score = candidates[0][0]
        close = [item for item in candidates if item[0] >= top_score - 0.18]
        if len(close) > 1 and random.random() < 0.35:
            weights = torch.tensor([max(0.05, item[0] + 1.25) for item in close], dtype=torch.float32)
            selected = close[int(torch.multinomial(weights, 1).item())]
        else:
            selected = candidates[0]
        return int(selected[1]), float(selected[0])

    def _encode_lucky_prompt_conditioning(self, clip, prompt_text, template, device, dtype):
        if clip is None or not prompt_text:
            return None, None, ""
        try:
            tokens = clip.tokenize(prompt_text)
            encoded = clip.encode_from_tokens_scheduled(tokens)
        except Exception as e:
            print(f"[FunPackVideoRefiner] Lucky prompt CLIP/Gemma encode failed: {e}")
            return None, None, f"encode failed: {e}"

        if not isinstance(encoded, list) or not encoded:
            return None, None, "encode returned empty conditioning"

        item = encoded[0]
        if isinstance(item, (list, tuple)) and len(item) >= 2:
            cond = item[0]
            meta = item[1] if isinstance(item[1], dict) else {"pooled_output": None}
        else:
            cond = item if isinstance(item, torch.Tensor) else None
            meta = {"pooled_output": None}

        if not isinstance(cond, torch.Tensor):
            return None, None, "encode returned invalid conditioning"

        cond = cond.to(device=device, dtype=dtype)
        if not self._conditioning_canvas_compatible(cond, template):
            return None, None, f"encoded shape {tuple(cond.shape)} incompatible"

        return cond, meta, f"encoded prompt ({self._get_conditioning_seq_len(cond)} positions)"

    def _select_void_tokens(self, global_adaptive, word_groups, cur_positive,
                            word_importance, token_mask=None, max_count=3):
        if not isinstance(cur_positive, torch.Tensor) or cur_positive.dim() <= 1:
            return [], "idle"
        embedding_dim = int(cur_positive.shape[-1])
        eligible = self._eligible_void_bank_items(global_adaptive, embedding_dim)
        if eligible:
            pool = sorted(eligible, key=lambda item: item[2], reverse=True)[:24]
            weights = torch.tensor([max(0.01, item[2]) for item in pool], dtype=torch.float32)
            count = min(max_count, max(1, int(torch.randint(1, min(3, len(pool)) + 1, (1,)).item())))
            picked_indexes = torch.multinomial(weights, count, replacement=False).tolist()
            picked = []
            for index in picked_indexes:
                token, item, _ = pool[index]
                picked.append((token, serializable_to_tensor(item["embedding"]).float()))
            return picked, "bank"

        fallback = []
        token_mask_list = None
        if token_mask is not None:
            token_mask_list = token_mask.detach().cpu().tolist()
        for start, end, full_word, _ in word_groups:
            token = full_word.strip().lower()
            if not self._is_valuable_token(token):
                continue
            if not self._legacy_auto_inject_token_allowed(token):
                continue
            if token_mask_list is not None and not all(token_mask_list[start:min(len(token_mask_list), end)]):
                continue
            score = float(word_importance.get(token, 1.0))
            if score < 1.05:
                continue
            embedding = self._void_token_embedding(cur_positive, start, end)
            if embedding is not None:
                fallback.append((token, embedding.float().cpu(), score))
        fallback = sorted(fallback, key=lambda item: item[2], reverse=True)[:max_count]
        return [(token, vector) for token, vector, _ in fallback], "current" if fallback else "empty"

    def _apply_into_the_void(self, new_positive, reference, global_adaptive, word_groups,
                             word_importance, into_the_void=False, im_feeling_lucky=False,
                             token_mask=None, prompt_sequences=None, lucky_canvas=None,
                             lucky_canvas_label="", lucky_prompt_text="", lucky_prompt_tokens=None):
        if not into_the_void and not im_feeling_lucky:
            return new_positive, self._void_empty_status(False), {}
        if not isinstance(new_positive, torch.Tensor) or not isinstance(reference, torch.Tensor):
            return new_positive, "Void: on | unavailable | tokens: none", {}
        if new_positive.dim() <= 1 or list(new_positive.shape) != list(reference.shape):
            return new_positive, "Void: on | incompatible conditioning | tokens: none", {}

        device = new_positive.device
        dtype = new_positive.dtype
        seq_len = new_positive.shape[1] if new_positive.dim() == 3 else new_positive.shape[0]
        active_len = self._get_effective_seq_len(token_mask, seq_len)
        active_len = max(1, min(seq_len, active_len))
        mixed = new_positive.clone()
        status_parts = []
        lucky_metadata = {}
        lucky_canvas_used = False
        lucky_canvas_status = ""

        if (
            im_feeling_lucky and
            self._conditioning_canvas_compatible(lucky_canvas, new_positive)
        ):
            try:
                canvas = lucky_canvas.to(device=device, dtype=dtype)
            except Exception:
                canvas = None
            if isinstance(canvas, torch.Tensor):
                mixed = canvas.clone()
                reference = canvas.clone()
                seq_len = mixed.shape[1] if mixed.dim() == 3 else mixed.shape[0]
                if token_mask is None or int(token_mask.numel()) != int(seq_len):
                    token_mask = self._get_conditioning_token_mask(mixed)
                active_len = self._get_effective_seq_len(token_mask, seq_len)
                active_len = max(1, min(seq_len, active_len))
                lucky_canvas_used = True
                lucky_canvas_status = lucky_canvas_label or f"memory ({seq_len} positions)"

        clamp_base = mixed.clone()

        if into_the_void:
            selected, source = self._select_void_tokens(
                global_adaptive,
                word_groups,
                mixed,
                word_importance,
                token_mask=token_mask,
            )
            if selected:
                positions = torch.linspace(0, active_len - 1, steps=len(selected), device=device).round().long()
                strength = min(0.055, 0.09 / max(1, len(selected)))

                for index, (_, vector) in enumerate(selected):
                    vector = vector.to(device=device, dtype=dtype)
                    if vector.dim() != 1 or int(vector.shape[0]) != int(mixed.shape[-1]):
                        continue
                    pos = int(positions[index].item())
                    if token_mask is not None and pos < token_mask.shape[0] and not bool(token_mask[pos].item()):
                        active_positions = torch.nonzero(token_mask[:active_len], as_tuple=False).flatten()
                        if active_positions.numel() == 0:
                            continue
                        pos = int(active_positions[min(index, active_positions.numel() - 1)].item())
                    if mixed.dim() == 3:
                        mixed[:, pos, :] = mixed[:, pos, :].lerp(vector.view(1, -1), strength)
                    else:
                        mixed[pos, :] = mixed[pos, :].lerp(vector, strength)

                token_list = ", ".join(token for token, _ in selected)
                status_parts.append(f"Void: on | {source} | tokens: {token_list}")
            else:
                status_parts.append(f"Void: on | {source} | tokens: none")
        else:
            status_parts.append("Void: off | idle | tokens: none")

        if im_feeling_lucky:
            active_positions = torch.arange(active_len, device=device)
            if token_mask is not None:
                active_positions = torch.nonzero(token_mask[:active_len], as_tuple=False).flatten()

            prompt_tokens = [
                str(token).strip().lower()
                for token in (lucky_prompt_tokens or [])
                if self._is_valuable_token(str(token).strip())
            ]
            if lucky_canvas_used and prompt_tokens:
                positions = torch.linspace(
                    0,
                    max(0, int(active_positions.numel()) - 1),
                    steps=min(len(prompt_tokens), max(1, int(active_positions.numel()))),
                    device=device,
                ).round().long()
                injections = []
                for index, token in enumerate(prompt_tokens[:int(positions.numel())]):
                    pos = int(active_positions[int(positions[index].item())].item()) if active_positions.numel() > 0 else index
                    injections.append({"token": token, "start": pos, "end": pos + 1})
                lucky_metadata = {
                    "enabled": True,
                    "source": "encoded prompt",
                    "tokens": prompt_tokens,
                    "unique_tokens": sorted(set(prompt_tokens)),
                    "injections": injections,
                    "anchor_tokens": [],
                    "required_tokens": prompt_tokens,
                    "context_tokens": [],
                    "context_hits": 0,
                    "canvas": lucky_canvas_status,
                    "prompt": lucky_prompt_text,
                }
                token_preview = ", ".join(prompt_tokens[:8])
                if len(prompt_tokens) > 8:
                    token_preview += ", ..."
                prompt_preview = re.sub(r"\s+", " ", lucky_prompt_text).strip()
                if len(prompt_preview) > 160:
                    prompt_preview = prompt_preview[:157].rstrip() + "..."
                status_parts.append(
                    f"Lucky: on | encoded prompt | {len(prompt_tokens)} prompt tokens | "
                    f"canvas: {lucky_canvas_status} | prompt: {prompt_preview} | tokens: {token_preview}"
                )
            else:
                lucky_pool = self._eligible_lucky_bank_items(global_adaptive, int(mixed.shape[-1]))
                if len(lucky_pool) < 2:
                    if lucky_canvas_used:
                        lucky_metadata = {
                            "enabled": True,
                            "source": "memory canvas",
                            "tokens": [],
                            "unique_tokens": [],
                            "injections": [],
                            "anchor_tokens": [],
                            "required_tokens": [],
                            "context_tokens": [],
                            "context_hits": 0,
                            "canvas": lucky_canvas_status,
                            "prompt": lucky_prompt_text,
                        }
                        status_parts.append(
                            f"Lucky: on | memory canvas {lucky_canvas_status} | bank too small ({len(lucky_pool)}/2 eligible) | tokens: none"
                        )
                    else:
                        status_parts.append(f"Lucky: on | bank too small ({len(lucky_pool)}/2 eligible) | tokens: none")
                elif active_positions.numel() <= 0:
                    if lucky_canvas_used:
                        lucky_metadata = {
                            "enabled": True,
                            "source": "memory canvas",
                            "tokens": [],
                            "unique_tokens": [],
                            "injections": [],
                            "anchor_tokens": [],
                            "required_tokens": [],
                            "context_tokens": [],
                            "context_hits": 0,
                            "canvas": lucky_canvas_status,
                            "prompt": lucky_prompt_text,
                        }
                        status_parts.append(f"Lucky: on | memory canvas {lucky_canvas_status} | no active positions | tokens: none")
                    else:
                        status_parts.append("Lucky: on | no active positions | tokens: none")
                else:
                    token_membership = {token: True for token, _, _ in lucky_pool}

                    ordered_sequences = []
                    if isinstance(prompt_sequences, list):
                        for sequence in prompt_sequences:
                            if not isinstance(sequence, list):
                                continue
                            ordered = [token for token in sequence if token in token_membership]
                            if len(ordered) >= 2:
                                ordered_sequences.append(ordered)

                    chosen_tokens, compose_info = self._lucky_compose_tokens(
                        global_adaptive,
                        word_groups,
                        lucky_pool,
                        token_membership,
                        int(active_positions.numel()),
                    )
                    if ordered_sequences and compose_info.get("source") == "global preferences":
                        sequence = random.choice(ordered_sequences)
                        chosen_tokens = [
                            sequence[index % len(sequence)]
                            if random.random() < 0.55 else chosen_tokens[index]
                            for index in range(min(len(chosen_tokens), int(active_positions.numel())))
                        ]
                        compose_info["source"] = "saved prompt order"

                    needed_tokens = set(chosen_tokens)
                    token_vectors = {}
                    for token, item, _ in lucky_pool:
                        if token not in needed_tokens:
                            continue
                        try:
                            vector = serializable_to_tensor(item["embedding"]).to(device=device, dtype=dtype)
                        except Exception:
                            continue
                        if vector.dim() == 1 and int(vector.shape[0]) == int(mixed.shape[-1]):
                            token_vectors[token] = vector

                    lucky_field = mixed.clone()
                    picked_tokens = []
                    injections = []
                    for pos_tensor, token in zip(active_positions, chosen_tokens):
                        vector = token_vectors.get(token)
                        if vector is None:
                            continue
                        pos = int(pos_tensor.item())
                        if lucky_field.dim() == 3:
                            lucky_field[:, pos, :] = vector.view(1, -1)
                        else:
                            lucky_field[pos, :] = vector
                        picked_tokens.append(token)
                        injections.append({"token": token, "start": pos, "end": pos + 1})

                    if picked_tokens:
                        strength = 0.030 if into_the_void else 0.040
                        mixed = mixed.lerp(lucky_field, strength)
                        token_preview = []
                        seen_tokens = set()
                        for token in picked_tokens:
                            if token in seen_tokens:
                                continue
                            token_preview.append(token)
                            seen_tokens.add(token)
                            if len(token_preview) >= 8:
                                break
                        token_list = ", ".join(token_preview)
                        if len(seen_tokens) < len(set(picked_tokens)):
                            token_list += ", ..."
                        anchor_preview = ", ".join(compose_info.get("anchor_tokens", [])[:4]) or "none"
                        required_preview = ", ".join(compose_info.get("required_tokens", [])[:6]) or "none"
                        context_preview = ", ".join(compose_info.get("context_tokens", [])[:6]) or "none"
                        lucky_metadata = {
                            "enabled": True,
                            "source": compose_info.get("source", "global preferences"),
                            "tokens": picked_tokens,
                            "unique_tokens": sorted(set(picked_tokens)),
                            "injections": injections,
                            "anchor_tokens": compose_info.get("anchor_tokens", []),
                            "required_tokens": compose_info.get("required_tokens", []),
                            "context_tokens": compose_info.get("context_tokens", []),
                            "context_hits": int(compose_info.get("context_hits", 0)),
                            "canvas": lucky_canvas_status if lucky_canvas_used else "current conditioning",
                            "prompt": lucky_prompt_text,
                        }
                        canvas_phrase = f" | canvas: {lucky_canvas_status}" if lucky_canvas_used else ""
                        prompt_preview = re.sub(r"\s+", " ", lucky_prompt_text).strip()
                        if len(prompt_preview) > 160:
                            prompt_preview = prompt_preview[:157].rstrip() + "..."
                        prompt_phrase = f" | prompt: {prompt_preview}" if prompt_preview else ""
                        status_parts.append(
                            f"Lucky: on | {lucky_metadata['source']} | field {len(picked_tokens)} positions from {len(set(picked_tokens))} tokens | "
                            f"anchors: {anchor_preview} | required: {required_preview} | context add: {context_preview}{canvas_phrase}{prompt_phrase} | tokens: {token_list}"
                        )
                    else:
                        if lucky_canvas_used:
                            lucky_metadata = {
                                "enabled": True,
                                "source": "memory canvas",
                                "tokens": [],
                                "unique_tokens": [],
                                "injections": [],
                                "anchor_tokens": [],
                                "required_tokens": [],
                                "context_tokens": [],
                                "context_hits": 0,
                                "canvas": lucky_canvas_status,
                                "prompt": lucky_prompt_text,
                            }
                            status_parts.append(f"Lucky: on | memory canvas {lucky_canvas_status} | bank unreadable | tokens: none")
                        else:
                            status_parts.append("Lucky: on | bank unreadable | tokens: none")
        else:
            status_parts.append("Lucky: off")

        delta = mixed - clamp_base
        reference_norm = reference.norm(dim=-1, keepdim=True).clamp_min(1e-8)
        max_delta = reference_norm * 0.08
        delta_norm = delta.norm(dim=-1, keepdim=True).clamp_min(1e-8)
        scale = torch.minimum(torch.ones_like(delta_norm), max_delta / delta_norm)
        mixed = clamp_base + delta * scale
        mixed = torch.clamp(mixed, min=-60.0, max=60.0)
        norm_factor = clamp_base.norm(dim=-1, keepdim=True).clamp_min(1e-8)
        mixed = mixed / mixed.norm(dim=-1, keepdim=True).clamp_min(1e-8) * norm_factor
        return mixed, " | ".join(status_parts), lucky_metadata

    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "positive_conditioning": ("CONDITIONING",),
                "mode": (["ltx2", "wan"], {"default": "ltx2", "label": "Tokenizer Mode"}),
                "rating": (RATING_LABELS, {"default": "Missing concept", "label": "Rating"}),
                "refinement_key": ("STRING", {"default": "my_style_v1", "multiline": False}),
                "scheduler_mode": (["original", "accurate", "aggressive"], {"default": "original"}),
            },
            "optional": {
                "positive_prompt": ("STRING", {
                    "multiline": True,
                    "default": "",
                    "placeholder": "Positive prompt"
                }),
                "clip": ("CLIP", {
                    "tooltip": "Optional text encoder. When connected, Lucky composes a learned prompt and re-encodes it before refinement."
                }),
                "sigmas": ("SIGMAS",),
                "sigma_strength": (["off", "subtle", "medium", "strong", "max"], {
                    "default": "subtle",
                    "label": "Sigma Refinement Strength"
                }),
                "reset_session": ("BOOLEAN", {"default": False, "label": "Reset Session (clears ALL history)"}),
                "unlimited_history": ("BOOLEAN", {
                    "default": False,
                    "label": "Unlimited History (never prunes)"
                }),
                "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "label": "Exploration Seed"}),
                "feedback_enabled": ("BOOLEAN", {"default": False, "label": "Enable Concept Feedback"}),
                "feedback_rating": ("INT", {
                    "default": 3,
                    "min": 1,
                    "max": 6,
                    "step": 1,
                    "display": "slider",
                    "label": "Feedback Response (follow the question scale)"
                }),
                "lora_stack": ("FUNPACK_LORA_STACK", {
                    "tooltip": "Optional stack from FunPack LoRA Loader. The refiner uses it to save prompt-specific suggested LoRA weights."
                }),
                "latent": ("LATENT", {
                    "tooltip": "Optional latent to refine. If no saved latent exists for this key, it passes through unchanged."
                }),
                "into_the_void": ("BOOLEAN", {
                    "default": False,
                    "tooltip": "Experimental: lightly mixes learned liked token embeddings into the final conditioning for preference discovery."
                }),
                "im_feeling_lucky": ("BOOLEAN", {
                    "default": False,
                    "label": "I'm Feeling Lucky",
                    "tooltip": "Composes conditioning from learned memory first, using stored canvases plus preferred token/context relationships."
                }),
            },
            "hidden": {
                "prompt": "PROMPT",
                "unique_id": "UNIQUE_ID",
            }
        }

    RETURN_TYPES = ("CONDITIONING", "STRING", "STRING", "STRING", "IMAGE", "SIGMAS", "LATENT")
    RETURN_NAMES = ("modified_positive", "status", "feedback_question", "training_info", "loss_graph", "refined_sigmas", "refined_latent")
    FUNCTION = "refine"
    CATEGORY = "FunPack/Refinement"

    @classmethod
    def IS_CHANGED(cls, **kwargs):
        return float("nan")

    # =========================================================================
    # SCHEDULER
    # =========================================================================

    def _get_scheduler_factors(self, mode, rating, reward, similarity, iter_num, total_iters,
                               word_importance, word_groups, global_adaptive, device):
        if mode == "original":
            base_lr = 0.18 / (1 + 0.08 * (iter_num ** 0.5))
            confidence = max(0.2, min(1.0, (rating - 3.0) / 5.0))
            if similarity > 0.93:
                confidence *= 0.4
            return base_lr, confidence, 1.0, {}

        confidence = max(0.15, min(1.0, (rating - 2.5) / 6.0))
        if abs(reward) < 0.15 or similarity > 0.94:
            confidence *= 0.35

        prodigy_d = global_adaptive.setdefault("prodigy_d", {})
        d_coef = 1.0 if mode == "accurate" else 1.8
        adaptive_lr = global_adaptive.get("prodigy_lr_base", 1.0)

        word_lr_mult = {}
        for _, _, full_word, _ in word_groups:
            wkey = full_word.lower()
            if wkey not in word_importance:
                continue
            g = abs(reward) + 1e-8
            if wkey not in prodigy_d:
                prodigy_d[wkey] = g
            else:
                prodigy_d[wkey] = 0.9 * prodigy_d[wkey] + 0.1 * g
            word_lr_mult[wkey] = adaptive_lr / (prodigy_d[wkey] ** 0.5 + 1e-8) * d_coef

        current_step = global_adaptive.setdefault("current_step", 0)
        current_step = min(current_step + 1, 500)
        global_adaptive["current_step"] = current_step

        warmup = global_adaptive.get("warmup_steps", 8)
        progress = min(1.0, current_step / max(50, global_adaptive.get("total_steps_estimate", 150)))

        if current_step < warmup:
            lr_scale = current_step / max(1, warmup)
        else:
            lr_scale = 0.5 * (1.0 + math.cos(math.pi * (progress - warmup / (warmup + 50))))

        if mode == "accurate":
            lr_scale *= 0.75
            confidence *= 0.9
            exploration_mult = max(0.3, 1.0 - 0.6 * progress)
        else:
            lr_scale = min(2.2, lr_scale * 1.6)
            confidence = min(1.0, confidence * 1.3)
            exploration_mult = max(0.6, 1.3 - 0.7 * progress)

        return lr_scale, confidence, exploration_mult, word_lr_mult

    # =========================================================================
    # TOKEN / WORD UTILITIES
    # =========================================================================

    def _get_top_tokens(self, token_dict, tokenizer, top_k=10):
        if not token_dict or not tokenizer:
            return "N/A"
        sorted_tokens = sorted(token_dict.items(), key=lambda x: x[1], reverse=True)[:top_k]
        top_list = []
        for key, score in sorted_tokens:
            try:
                text = key if isinstance(key, str) else tokenizer.decode([int(key)], skip_special_tokens=True).strip()
                if text:
                    top_list.append(f"{text}({score:.2f})")
            except Exception:
                top_list.append(f"{key}({score:.2f})")
        return ", ".join(top_list) if top_list else "None"

    def _is_valuable_token(self, token_text):
        if not token_text:
            return False
        t = token_text.strip()
        if '<' in t or '>' in t or len(t) < 3:
            return False
        t_lower = t.lower()
        stopwords = {
            "the", "a", "an", "and", "or", "but", "with", "for", "of", "in", "on", "at",
            "to", "from", "by", "is", "are", "was", "were", "be", "been", "being", "have",
            "has", "had", "do", "does", "did", "will", "would", "her", "his", "him", "she",
            "he", "it", "they", "them", "this", "that", "these", "those", "i", "you", "my",
            "your", "our", "their", "me", "us"
        }
        if t_lower in stopwords:
            return False
        if t in {",", ".", "!", "?", ":", ";", "-", "*", "(", ")", "[", "]", "{", "}", "'", "\"", "..."} or t.isdigit():
            return False
        if not any(c.isalpha() for c in t):
            return False
        return True

    def _infer_concept_category(self, phrase_words: list):
        words = set(phrase_words or [])
        if not words:
            return "general"

        category_terms = {
            "quality": {"masterpiece", "best", "quality", "detailed", "highres", "high-res", "ultra", "perfect"},
            "style": {"anime", "cinematic", "photorealistic", "painterly", "illustration", "stylized", "realistic", "film", "noir"},
            "camera": {"closeup", "close-up", "wide", "shot", "angle", "zoom", "pan", "tracking", "dolly", "camera", "focus", "bokeh", "framing"},
            "action": {"running", "walking", "flying", "jumping", "smiling", "turning", "dancing", "moving", "motion", "looking", "holding", "standing", "sitting"},
            "environment": {"forest", "city", "street", "room", "beach", "mountain", "temple", "sunset", "night", "rain", "snow", "sky", "background"},
            "appearance": {"hair", "eyes", "dress", "jacket", "armor", "face", "skin", "beard", "smile", "pose", "outfit"},
            "subject": {"girl", "boy", "woman", "man", "person", "character", "robot", "dragon", "cat", "dog", "bird", "child"},
        }

        best_category, best_score = "general", 0
        for category, terms in category_terms.items():
            score = len(words & terms)
            if score > best_score:
                best_category, best_score = category, score
        return best_category

    def _default_concept_cluster(self, phrase_words: list):
        category = self._infer_concept_category(phrase_words)
        return {
            "label": " ".join((phrase_words or [])[:6]),
            "anchor_words": list(phrase_words or []),
            "category": category,
            "category_source": "auto",
            "category_confidence": 0.35 if category == "general" else 0.65,
            "word_importance": {},
            "presence_target": 1.0,
            "priority_weight": 1.0,
            "overrep_sensitivity": 1.0,
            "stability_weight": 1.0,
            "semantic_fidelity": 1.0,
            "user_affinity": 1.0,
            "question_history": [],
            "last_question_type": None,
            "last_question_iter": 0,
            "usage_count": 0,
            "last_seen_iter": 0,
        }

    def _ensure_concept_cluster_defaults(self, cluster: dict):
        if not isinstance(cluster, dict):
            return self._default_concept_cluster([])

        anchor_words = list(cluster.get("anchor_words", []))
        defaults = self._default_concept_cluster(anchor_words)
        defaults.update(cluster)
        defaults.setdefault("category_source", "auto")
        defaults.setdefault("category_confidence", 0.35 if defaults.get("category") == "general" else 0.65)
        if defaults.get("category_source") != "user" and (
            not defaults.get("category") or defaults.get("category") == "general"
        ):
            defaults["category"] = self._infer_concept_category(defaults.get("anchor_words", []))
            defaults["category_confidence"] = 0.35 if defaults["category"] == "general" else max(0.65, defaults.get("category_confidence", 0.0))
        defaults["question_history"] = list(defaults.get("question_history", []))[-24:]
        return defaults

    def _clip_profile_value(self, value, low=0.5, high=1.8):
        return max(low, min(high, float(value)))

    def _feedback_question_specs(self):
        return {
            "presence": {
                "prompt": "How well is '{label}' represented in the output?",
                "legend": "1=absent  2=weak  3=slightly weak  4=slightly strong  5=perfect  6=overrepresented",
            },
            "priority": {
                "prompt": "How important should '{label}' be relative to the other concepts?",
                "legend": "1=much less important  2=less important  3=slightly less  4=slightly more  5=important  6=top priority",
            },
            "balance": {
                "prompt": "How balanced is '{label}' compared with nearby concepts?",
                "legend": "1=far too weak  2=too weak  3=slightly weak  4=slightly strong  5=balanced  6=overpowering",
            },
            "fidelity": {
                "prompt": "How accurately does '{label}' match what you meant?",
                "legend": "1=wrong  2=mostly wrong  3=slightly off  4=close  5=correct  6=too literal",
            },
            "stability": {
                "prompt": "How stable should '{label}' be across future outputs?",
                "legend": "1=very unstable  2=unstable  3=slightly unstable  4=slightly stable  5=stable  6=very rigid",
            },
            "preference": {
                "prompt": "What is your preference for having '{label}' in future outputs?",
                "legend": "1=strongly less  2=less  3=slightly less  4=slightly more  5=more  6=much more",
            },
            "category": {
                "prompt": "What kind of concept is '{label}'?",
                "legend": "1=general  2=concept  3=style  4=quality  5=character  6=details",
            },
        }

    def _get_concept_mean_importance(self, cluster: dict):
        local_imp = cluster.get("word_importance", {})
        if not local_imp:
            return 1.0
        return sum(local_imp.values()) / max(1, len(local_imp))

    def _get_question_base_weight(self, question_type: str, category: str):
        weights = {
            "presence": {"subject": 1.15, "appearance": 1.12, "action": 1.10, "environment": 1.08, "style": 0.95, "camera": 0.92, "quality": 0.90, "general": 1.0},
            "priority": {"style": 1.15, "camera": 1.12, "quality": 1.08, "environment": 1.02, "subject": 0.98, "appearance": 0.98, "action": 1.0, "general": 1.0},
            "balance": {"style": 1.18, "camera": 1.10, "quality": 1.08, "environment": 1.02, "subject": 0.98, "appearance": 1.0, "action": 1.0, "general": 1.0},
            "fidelity": {"subject": 1.15, "action": 1.12, "environment": 1.08, "appearance": 1.05, "style": 0.95, "camera": 0.92, "quality": 0.90, "general": 1.0},
            "stability": {"action": 1.10, "camera": 1.10, "style": 1.06, "subject": 1.04, "appearance": 1.0, "environment": 1.0, "quality": 0.95, "general": 1.0},
            "preference": {"style": 1.14, "environment": 1.10, "appearance": 1.05, "subject": 1.03, "camera": 1.02, "action": 1.0, "quality": 0.95, "general": 1.0},
        }
        return weights.get(question_type, {}).get(category, 1.0)

    def _score_question_type(self, question_type: str, cluster: dict, category: str,
                             mean_imp: float, neighbor_mean: float, rating_shift: float,
                             similarity: float, iter_num: int):
        uncertainty = max(0.0, 1.0 - min(1.0, abs(mean_imp - cluster.get("presence_target", 1.0))))
        dominance = max(0.0, mean_imp - 1.15)
        neighbor_conflict = abs(mean_imp - neighbor_mean)
        freshness = 1.0
        if cluster.get("last_question_type") == question_type:
            freshness -= 0.35
        if iter_num - int(cluster.get("last_question_iter", 0)) < 3:
            freshness -= 0.20
        freshness = max(0.25, freshness)

        if question_type == "presence":
            score = 0.50 * uncertainty + 0.22 * abs(cluster.get("presence_target", 1.0) - mean_imp) + 0.16 * rating_shift + 0.12 * max(0.0, 1.0 - similarity)
        elif question_type == "priority":
            score = 0.40 * rating_shift + 0.28 * abs(cluster.get("priority_weight", 1.0) - 1.0) + 0.18 * dominance + 0.14 * uncertainty
        elif question_type == "balance":
            score = 0.44 * dominance + 0.28 * neighbor_conflict + 0.16 * abs(cluster.get("overrep_sensitivity", 1.0) - 1.0) + 0.12 * rating_shift
        elif question_type == "fidelity":
            score = 0.38 * (2.0 - cluster.get("semantic_fidelity", 1.0)) + 0.24 * rating_shift + 0.20 * uncertainty + 0.18 * max(0.0, 1.0 - similarity)
        elif question_type == "stability":
            score = 0.35 * rating_shift + 0.30 * abs(cluster.get("stability_weight", 1.0) - 1.0) + 0.20 * max(0.0, 1.0 - similarity) + 0.15 * uncertainty
        elif question_type == "category":
            category_uncertainty = max(0.0, 1.0 - float(cluster.get("category_confidence", 0.35)))
            needs_category_discovery = (
                cluster.get("category_source") != "user" and
                (category == "general" or category_uncertainty >= 0.50)
            )
            if not needs_category_discovery:
                return 0.0
            score = 0.65 + 0.25 * category_uncertainty + 0.10 * rating_shift
        else:  # preference
            score = 0.34 * abs(cluster.get("user_affinity", 1.0) - 1.0) + 0.28 * rating_shift + 0.20 * uncertainty + 0.18 * abs(cluster.get("priority_weight", 1.0) - 1.0)

        return score * self._get_question_base_weight(question_type, category) * freshness

    def _select_feedback_question(self, ordered_concept_ids: list, concept_clusters: dict,
                                  concept_groups: dict, current_concept_labels: dict,
                                  last_rating: int, rating: int,
                                  similarity: float, iter_num: int):
        if not ordered_concept_ids:
            return None

        question_specs = self._feedback_question_specs()
        rating_shift = min(1.0, abs(rating - last_rating) / 4.0)
        candidates = []
        category_candidates = []

        for cid in ordered_concept_ids:
            cluster = concept_clusters.get(cid)
            if not cluster:
                continue
            cluster = self._ensure_concept_cluster_defaults(cluster)
            concept_clusters[cid] = cluster

            category = cluster.get("category", "general")
            mean_imp = self._get_concept_mean_importance(cluster)
            neighbor_ids = self._get_concept_neighbors(cid, ordered_concept_ids, radius=2)
            neighbor_means = [
                self._get_concept_mean_importance(concept_clusters[nid])
                for nid in neighbor_ids if nid in concept_clusters
            ]
            neighbor_mean = sum(neighbor_means) / len(neighbor_means) if neighbor_means else 1.0

            chosen_group_id = None
            for gid, group in concept_groups.items():
                if cid in group.get("concept_ids", []):
                    chosen_group_id = gid
                    break

            for question_type in question_specs.keys():
                score = self._score_question_type(
                    question_type, cluster, category, mean_imp,
                    neighbor_mean, rating_shift, similarity, iter_num
                )
                candidate = {
                    "concept_id": cid,
                    "concept_label": current_concept_labels.get(cid, cluster.get("label", "")),
                    "question_type": question_type,
                    "neighbor_ids": neighbor_ids,
                    "group_id": chosen_group_id,
                    "category": category,
                    "score": score,
                }
                if question_type == "category":
                    category_candidates.append(candidate)
                else:
                    candidates.append(candidate)

        if category_candidates:
            category_candidate = max(category_candidates, key=lambda x: x["score"])
            if category_candidate["score"] >= 0.60:
                return category_candidate

        if not candidates:
            return None

        return max(candidates, key=lambda x: x["score"])

    def _format_feedback_question(self, concept_label: str, question_type: str):
        spec = self._feedback_question_specs().get(question_type, self._feedback_question_specs()["presence"])
        return (
            f"Concept: '{concept_label}'\n"
            f"{spec['prompt'].format(label=concept_label)}\n"
            f"{spec['legend']}"
        )

    # =========================================================================
    # MULTI-LEVEL CONCEPT SYSTEM
    # =========================================================================

    def _parse_concepts(self, prompt: str):
        """
        Level 2 → Level 3 boundary: split the prompt on commas/semicolons into
        concept phrases. Each phrase is returned as a list of significant lowercase
        words, preserving prompt order.

        Example:
          "masterpiece, anime girl, long hair, dark forest"
          -> [["masterpiece"], ["anime","girl"], ["long","hair"], ["dark","forest"]]
        """
        if not prompt:
            return []
        result = []
        for segment_type, segment_text in self._iter_prompt_segments(prompt):
            if segment_type == "protected":
                protected_phrase = segment_text.strip().lower()
                if len(protected_phrase) >= 3 and any(c.isalpha() for c in protected_phrase):
                    result.append([protected_phrase])
                continue

            phrases = [p.strip() for p in re.split(r'[,;]', segment_text) if p.strip()]
            for phrase in phrases:
                words = [w.strip().lower() for w in phrase.split() if self._is_valuable_token(w.strip())]
                if words:
                    result.append(words)
        return result

    def _concept_overlap_words(self, prompt: str):
        words = []
        seen = set()
        for phrase_words in self._parse_concepts(prompt):
            for word in phrase_words:
                clean = str(word).strip().lower()
                if not clean or clean in seen:
                    continue
                seen.add(clean)
                words.append(clean)
        return words

    def _ordered_prompt_words(self, prompt: str):
        words = []
        for phrase_words in self._parse_concepts(prompt):
            for word in phrase_words:
                clean = str(word).strip().lower()
                if clean and self._is_valuable_token(clean):
                    words.append(clean)
        return words

    def _ordered_prompt_phrases(self, prompt: str):
        phrases = []
        if not prompt:
            return phrases
        phrase_texts = []
        for segment_type, segment_text in self._iter_prompt_segments(prompt):
            if segment_type == "protected":
                phrase_texts.append(segment_text)
            else:
                phrase_texts.extend(p.strip() for p in re.split(r'[,;]', segment_text) if p.strip())

        for phrase_text in phrase_texts:
            text = re.sub(r"\s+", " ", str(phrase_text or "").strip().lower()).strip(" ,;:.")
            if not text or len(text) < 3:
                continue
            phrase_tokens = [
                token.strip().lower()
                for token in re.findall(r"[\w'’.-]+", text, flags=re.UNICODE)
                if self._is_valuable_token(token.strip())
            ]
            if not phrase_tokens:
                continue
            phrases.append({
                "text": text,
                "tokens": phrase_tokens,
            })
        return phrases

    def _lucky_prompt_phrase_sequences(self, prompt_histories: dict):
        if not isinstance(prompt_histories, dict):
            return []

        sequences = []
        seen = set()
        for prompt_key, history_entry in prompt_histories.items():
            if not isinstance(history_entry, dict):
                continue

            history = history_entry.get("history", [])
            good_history = []
            if isinstance(history, list):
                for item in history[-16:]:
                    if not isinstance(item, dict):
                        continue
                    profile = normalize_refiner_rating(item.get("rating_label", item.get("rating", 0)))
                    if int(profile.get("level", 0)) >= 5:
                        good_history.append(item)

            texts = []
            for item in good_history:
                for key in ("prompt_full", "analysis_prompt", "prompt"):
                    text = item.get(key)
                    if isinstance(text, str) and text.strip():
                        texts.append(text)
            if good_history:
                texts.extend(self._history_prompt_texts(prompt_key, history_entry))

            for text in texts:
                phrases = self._ordered_prompt_phrases(text)
                if not phrases:
                    continue
                signature = tuple(phrase["text"] for phrase in phrases)
                if signature in seen:
                    continue
                seen.add(signature)
                sequences.append(phrases[:32])

        return sequences[-48:]

    def _lucky_prompt_sequences(self, prompt_histories: dict):
        if not isinstance(prompt_histories, dict):
            return []

        sequences = []
        seen = set()
        for prompt_key, history_entry in prompt_histories.items():
            if not isinstance(history_entry, dict):
                continue

            history = history_entry.get("history", [])
            good_history = []
            if isinstance(history, list):
                for item in history[-24:]:
                    if not isinstance(item, dict):
                        continue
                    profile = normalize_refiner_rating(item.get("rating_label", item.get("rating", 0)))
                    if int(profile.get("level", 0)) >= 5:
                        good_history.append(item)

            for item in good_history:
                for key in ("prompt_full", "analysis_prompt", "prompt"):
                    text = item.get(key)
                    if not isinstance(text, str) or not text.strip():
                        continue
                    ordered = self._ordered_prompt_words(text)
                    if len(ordered) < 2:
                        continue
                    signature = tuple(ordered)
                    if signature not in seen:
                        seen.add(signature)
                        sequences.append(ordered[:64])

            if good_history:
                for text in self._history_prompt_texts(prompt_key, history_entry):
                    ordered = self._ordered_prompt_words(text)
                    if len(ordered) < 2:
                        continue
                    signature = tuple(ordered)
                    if signature not in seen:
                        seen.add(signature)
                        sequences.append(ordered[:64])

        return sequences[-96:]

    def _history_prompt_texts(self, prompt_key: str, history_entry: dict):
        texts = []
        for key in ("canonical_prompt", "last_positive_prompt"):
            value = history_entry.get(key)
            if isinstance(value, str) and value.strip():
                texts.append(value)
        if isinstance(prompt_key, str) and prompt_key.strip():
            texts.append(prompt_key)
        for item in reversed(history_entry.get("history", [])[-4:]):
            value = item.get("prompt") if isinstance(item, dict) else None
            if isinstance(value, str) and value.strip():
                texts.append(value)

        unique = []
        seen = set()
        for text in texts:
            normalized = text.strip()
            if normalized and normalized not in seen:
                seen.add(normalized)
                unique.append(normalized)
        return unique

    def _prompt_history_concept_words(self, prompt_key: str, history_entry: dict):
        stored = history_entry.get("prompt_concept_words")
        if isinstance(stored, list) and stored:
            return [str(word).lower() for word in stored if str(word).strip()]

        words = []
        seen = set()
        for text in self._history_prompt_texts(prompt_key, history_entry):
            for word in self._concept_overlap_words(text):
                if word not in seen:
                    seen.add(word)
                    words.append(word)
        return words

    def _conditioning_similarity(self, reference: torch.Tensor, current: torch.Tensor, token_mask=None):
        if not isinstance(reference, torch.Tensor) or not isinstance(current, torch.Tensor):
            return 0.0

        if list(reference.shape) != list(current.shape):
            return 0.0

        current = current.to(reference.device) if current.device != reference.device else current
        try:
            if reference.dim() >= 2 and current.dim() >= 2:
                ref_mean = self._masked_sequence_mean(reference, token_mask)
                cur_mean = self._masked_sequence_mean(current, token_mask)
                return float(F.cosine_similarity(ref_mean, cur_mean, dim=-1).mean().item())
            return float(F.cosine_similarity(
                reference.flatten().unsqueeze(0),
                current.flatten().unsqueeze(0),
                dim=-1
            ).mean().item())
        except Exception:
            return 0.0

    def _conditioning_history_similarity(self, history_entry: dict, raw_positive: torch.Tensor):
        source_data = history_entry.get("source_conditioning_embeds") or history_entry.get("reference_embeds")
        try:
            source_reference = serializable_to_tensor(source_data)
        except Exception:
            return 0.0

        return self._conditioning_similarity(source_reference, raw_positive)

    def _find_prompt_variant_history(self, exact_prompt_key: str, current_words: list,
                                     raw_positive: torch.Tensor, prompt_histories: dict):
        if not exact_prompt_key or not current_words or not prompt_histories:
            return None, None

        current_set = set(current_words)
        if len(current_set) < 2:
            return None, None

        best_key = None
        best_info = None
        best_score = 0.0
        for candidate_key, history_entry in prompt_histories.items():
            if candidate_key == exact_prompt_key or not isinstance(history_entry, dict):
                continue

            candidate_words = self._prompt_history_concept_words(candidate_key, history_entry)
            candidate_set = set(candidate_words)
            if len(candidate_set) < 2:
                continue

            shared = current_set & candidate_set
            shared_count = len(shared)
            union = current_set | candidate_set
            overlap = shared_count / max(1, len(union))
            coverage = shared_count / max(1, min(len(current_set), len(candidate_set)))
            semantic_score = max(overlap, coverage * 0.72)
            conditioning_similarity = self._conditioning_history_similarity(history_entry, raw_positive)

            if conditioning_similarity < 0.68:
                continue
            if shared_count < 3 and not (coverage >= 0.85 and conditioning_similarity >= 0.82):
                continue
            if not (
                (overlap >= 0.46 and conditioning_similarity >= 0.78) or
                (coverage >= 0.68 and conditioning_similarity >= 0.72) or
                (overlap >= 0.60 and conditioning_similarity >= 0.68)
            ):
                continue

            score = semantic_score * 0.58 + conditioning_similarity * 0.42
            if score > best_score:
                best_score = score
                best_key = candidate_key
                best_info = {
                    "score": round(float(score), 4),
                    "overlap": round(float(overlap), 4),
                    "coverage": round(float(coverage), 4),
                    "conditioning_similarity": round(float(conditioning_similarity), 4),
                    "shared_words": sorted(shared)[:12],
                    "matched_prompt_key": candidate_key,
                }

        return best_key, best_info

    def _remember_prompt_variant(self, active: dict, exact_prompt_key: str, positive_prompt: str,
                                 current_words: list, match_info: dict = None):
        active.setdefault("canonical_prompt", positive_prompt or exact_prompt_key)
        existing_words = set(
            str(word).lower()
            for word in active.get("prompt_concept_words", [])
            if str(word).strip()
        )
        for word in current_words:
            existing_words.add(str(word).lower())
        active["prompt_concept_words"] = sorted(existing_words)[:256]

        variants = list(active.get("prompt_variants", []))[-11:]
        variant = {
            "prompt_key": exact_prompt_key,
            "prompt": (positive_prompt or exact_prompt_key)[:240],
        }
        if match_info:
            variant.update({
                "overlap": match_info.get("overlap"),
                "coverage": match_info.get("coverage"),
                "conditioning_similarity": match_info.get("conditioning_similarity"),
            })
        variants.append(variant)
        active["prompt_variants"] = variants

    def _history_entry_prompt_words(self, entry: dict):
        if not isinstance(entry, dict):
            return []
        stored = entry.get("prompt_words")
        if isinstance(stored, list) and stored:
            return [str(word).lower() for word in stored if str(word).strip()]
        prompt_text = entry.get("prompt_full") or entry.get("prompt") or ""
        return self._concept_overlap_words(prompt_text)

    def _retarget_prompt_history_to_source(self, active: dict, old_source: torch.Tensor,
                                           new_source: torch.Tensor):
        if (
            not isinstance(active, dict) or
            not isinstance(old_source, torch.Tensor) or
            not isinstance(new_source, torch.Tensor) or
            list(old_source.shape) != list(new_source.shape)
        ):
            return 0

        shift = new_source.to(old_source.device) - old_source
        migrated = 0

        def _shift_serialized(field_name: str):
            nonlocal migrated
            payload = active.get(field_name)
            if payload is None:
                return
            try:
                tensor = serializable_to_tensor(payload).to(old_source.device)
                if list(tensor.shape) != list(old_source.shape):
                    return
                active[field_name] = tensor_to_serializable(tensor + shift)
                migrated += 1
            except Exception:
                return

        _shift_serialized("reference_embeds")
        _shift_serialized("liked_reference_embeds")

        for item in active.get("history", []):
            if not isinstance(item, dict) or item.get("modified_embeds") is None:
                continue
            try:
                tensor = serializable_to_tensor(item.get("modified_embeds")).to(old_source.device)
                if list(tensor.shape) != list(old_source.shape):
                    continue
                item["modified_embeds"] = tensor_to_serializable(tensor + shift)
                item["retargeted_to_prompt_variant"] = True
                migrated += 1
            except Exception:
                continue

        return migrated

    def _nudge_delta_words(self, words: list, delta: float, concept_clusters: dict,
                           word_importance: dict, iter_num: int, role: str):
        touched = []
        for raw_word in words:
            word = str(raw_word).strip().lower()
            if not self._is_valuable_token(word):
                continue
            cid, is_new = self._match_concept([word], concept_clusters, threshold=0.18)
            if cid is None:
                continue
            if is_new or cid not in concept_clusters:
                concept_clusters[cid] = self._default_concept_cluster([word])
            else:
                concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
                if word not in concept_clusters[cid].get("anchor_words", []):
                    concept_clusters[cid]["anchor_words"].append(word)

            cluster = concept_clusters[cid]
            local_imp = cluster.setdefault("word_importance", {})
            local_imp[word] = max(0.35, min(2.8, float(local_imp.get(word, 1.0)) + delta))
            word_importance[word] = max(0.35, min(2.8, float(word_importance.get(word, 1.0)) + delta * 0.45))
            cluster["presence_target"] = self._clip_profile_value(
                float(cluster.get("presence_target", 1.0)) + delta * 0.28
            )
            cluster["priority_weight"] = self._clip_profile_value(
                float(cluster.get("priority_weight", 1.0)) + delta * 0.20
            )
            if delta > 0:
                cluster["semantic_fidelity"] = self._clip_profile_value(
                    float(cluster.get("semantic_fidelity", 1.0)) + delta * 0.12,
                    low=0.6,
                    high=1.8,
                )
            else:
                cluster["overrep_sensitivity"] = self._clip_profile_value(
                    float(cluster.get("overrep_sensitivity", 1.0)) + abs(delta) * 0.12
                )
            cluster["last_seen_iter"] = iter_num
            cluster["usage_count"] = int(cluster.get("usage_count", 0)) + 1
            cluster["last_prompt_delta_role"] = role
            touched.append(word)
        return touched

    def _missing_axis_matches_category(self, axis: str, category: str):
        category = (category or "general").lower()
        if axis == "concept":
            return category in {"subject", "action", "appearance", "character", "concept", "general"}
        if axis == "details":
            return category in {"camera", "action", "environment", "appearance", "style", "general"}
        if axis == "quality":
            return category in {"quality", "style", "general"}
        return False

    def _apply_missing_axis_prompt_pressure(self, word_groups, word_to_concept, rating_profile,
                                            concept_clusters, word_importance, iter_num):
        missing_axes = set(rating_profile.get("missing_axes", []))
        if not missing_axes or not word_groups:
            return ""

        axis_boosts = {"concept": 0.34, "details": 0.26, "quality": 0.22}
        touched = []
        for _, _, full_word, _ in word_groups:
            word = str(full_word).strip().lower()
            if not self._is_valuable_token(word):
                continue
            cid = word_to_concept.get(word)
            category = "general"
            cluster = None
            if cid and cid in concept_clusters:
                concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
                cluster = concept_clusters[cid]
                category = cluster.get("category", "general")

            matched_axes = [axis for axis in missing_axes if self._missing_axis_matches_category(axis, category)]
            pressure = sum(axis_boosts.get(axis, 0.0) for axis in matched_axes)
            if not pressure:
                pressure = sum(axis_boosts.get(axis, 0.0) for axis in missing_axes) * 0.35
            if not pressure:
                continue

            word_importance[word] = max(0.35, min(2.8, float(word_importance.get(word, 1.0)) + pressure * 0.32))
            if cluster is not None:
                local_imp = cluster.setdefault("word_importance", {})
                local_imp[word] = max(0.35, min(2.8, float(local_imp.get(word, 1.0)) + pressure * 0.55))
                cluster["missing_count"] = int(cluster.get("missing_count", 0)) + 1
                for axis in missing_axes:
                    counter = self._axis_counter_name(axis)
                    if counter:
                        cluster[counter] = int(cluster.get(counter, 0)) + 1
                if "concept" in missing_axes or "details" in missing_axes:
                    cluster["presence_target"] = self._clip_profile_value(
                        float(cluster.get("presence_target", 1.0)) + pressure * 0.18
                    )
                    cluster["priority_weight"] = self._clip_profile_value(
                        float(cluster.get("priority_weight", 1.0)) + pressure * 0.14
                    )
                if "quality" in missing_axes:
                    cluster["semantic_fidelity"] = self._clip_profile_value(
                        float(cluster.get("semantic_fidelity", 1.0)) + pressure * 0.16,
                        low=0.6,
                        high=1.8,
                    )
                    cluster["stability_weight"] = self._clip_profile_value(
                        float(cluster.get("stability_weight", 1.0)) + pressure * 0.12
                    )
                cluster["last_missing_iter"] = iter_num
            touched.append(word)

        if not touched:
            return ""
        axes = "+".join(sorted(missing_axes))
        return f"Missing-axis pressure: {axes} reinforced {', '.join(touched[:10])}{'...' if len(touched) > 10 else ''}."

    def _apply_prompt_delta_attribution(self, history: list, current_words: list,
                                        rating_profile: dict, concept_clusters: dict,
                                        word_importance: dict, iter_num: int):
        if len(history) < 2:
            return ""

        previous_entry = history[-2]
        rated_entry = history[-1]
        previous_words = self._history_entry_prompt_words(previous_entry)
        rated_words = self._history_entry_prompt_words(rated_entry) or current_words
        if not previous_words or not rated_words:
            return ""

        previous_set = set(previous_words)
        rated_set = set(rated_words)
        added = sorted(rated_set - previous_set)
        removed = sorted(previous_set - rated_set)
        if not added and not removed:
            return ""

        previous_profile = normalize_refiner_rating(rated_entry.get("rating_label", rated_entry.get("rating", 0)))
        current_key = rating_profile.get("key", "")
        previous_missing_axes = set(previous_profile.get("missing_axes", []))
        current_missing_axes = set(rating_profile.get("missing_axes", []))
        word_importance = word_importance if isinstance(word_importance, dict) else {}

        boosted = []
        softened = []
        reason = ""
        if current_key == "like" and previous_missing_axes:
            boost = 0.22 + (0.10 if "concept" in previous_missing_axes else 0.0)
            boosted = self._nudge_delta_words(
                added[:24],
                boost,
                concept_clusters,
                word_importance,
                iter_num,
                "added_helped",
            )
            softened = self._nudge_delta_words(
                removed[:24],
                -0.08,
                concept_clusters,
                word_importance,
                iter_num,
                "removed_after_missing",
            )
            reason = f"{previous_profile.get('label', 'missing')} -> {rating_profile.get('label', 'Perfect')}"
        elif current_missing_axes and previous_profile.get("key") == "like":
            boost = 0.14 + (0.08 if "concept" in current_missing_axes else 0.0)
            boosted = self._nudge_delta_words(
                removed[:24],
                boost,
                concept_clusters,
                word_importance,
                iter_num,
                "removed_needed",
            )
            softened = self._nudge_delta_words(
                added[:24],
                -0.10,
                concept_clusters,
                word_importance,
                iter_num,
                "added_hurt",
            )
            reason = f"{previous_profile.get('label', 'Perfect')} -> {rating_profile.get('label', 'missing')}"

        if not boosted and not softened:
            return ""

        parts = [f"Prompt delta: {reason}."]
        if boosted:
            parts.append("boosted " + ", ".join(boosted[:8]))
        if softened:
            parts.append("softened " + ", ".join(softened[:8]))
        return " ".join(parts)

    def _build_prompt_fallback_concept(self, prompt: str, concept_clusters: dict):
        if not prompt:
            return None

        fallback_words = []
        for segment_type, segment_text in self._iter_prompt_segments(prompt):
            if segment_type == "protected":
                protected_phrase = segment_text.strip().lower()
                if len(protected_phrase) >= 3 and any(c.isalpha() for c in protected_phrase):
                    fallback_words.append(protected_phrase)
                continue

            fallback_words.extend(
                w.strip().lower()
                for w in re.split(r'[\s,;]+', segment_text)
                if self._is_valuable_token(w.strip())
            )
            if len(fallback_words) >= 8:
                break

        fallback_words = fallback_words[:8]
        if not fallback_words:
            return None

        fallback_id = "prompt_" + md5("|".join(fallback_words).encode()).hexdigest()[:10]
        if fallback_id not in concept_clusters:
            concept_clusters[fallback_id] = self._default_concept_cluster(fallback_words)
            concept_clusters[fallback_id]["label"] = " ".join(fallback_words[:5])
        else:
            concept_clusters[fallback_id] = self._ensure_concept_cluster_defaults(concept_clusters[fallback_id])

        return fallback_id

    def _force_feedback_fallback(self, ordered_concept_ids: list, concept_clusters: dict,
                                 concept_groups: dict, current_concept_labels: dict,
                                 rating_shift: float, similarity: float):
        if not ordered_concept_ids:
            return None

        ranked = []
        for cid in ordered_concept_ids:
            cluster = concept_clusters.get(cid)
            if not cluster:
                continue
            cluster = self._ensure_concept_cluster_defaults(cluster)
            concept_clusters[cid] = cluster
            mean_imp = self._get_concept_mean_importance(cluster)
            dominance = abs(mean_imp - cluster.get("presence_target", 1.0))
            ranked.append((dominance, cid, cluster))

        if not ranked:
            return None

        _, chosen_cid, cluster = max(ranked, key=lambda x: x[0])
        if rating_shift >= 2.0:
            question_type = "fidelity"
        elif similarity < 0.84:
            question_type = "stability"
        elif cluster.get("category") in {"style", "camera", "quality"}:
            question_type = "balance"
        else:
            question_type = "presence"

        chosen_group_id = None
        for gid, group in concept_groups.items():
            if chosen_cid in group.get("concept_ids", []):
                chosen_group_id = gid
                break

        return {
            "concept_id": chosen_cid,
            "concept_label": current_concept_labels.get(chosen_cid, cluster.get("label", "")),
            "question_type": question_type,
            "neighbor_ids": self._get_concept_neighbors(chosen_cid, ordered_concept_ids, radius=2),
            "group_id": chosen_group_id,
            "category": cluster.get("category", "general"),
            "score": 0.0,
        }

    def _match_concept(self, phrase_words: list, concept_clusters: dict, threshold: float = 0.38):
        """
        Match a phrase to an existing concept cluster using Jaccard similarity on
        anchor words. If no cluster clears `threshold`, a new cluster id is minted.

        Returns (cluster_id, is_new_cluster).
        Tune threshold: lower -> more isolated clusters; higher -> more merging.
        """
        if not phrase_words:
            return None, False
        phrase_set = set(phrase_words)
        best_id, best_score = None, 0.0
        for cid, cluster in concept_clusters.items():
            anchor_set = set(cluster.get("anchor_words", []))
            if not anchor_set:
                continue
            union = len(phrase_set | anchor_set)
            if union == 0:
                continue
            jaccard = len(phrase_set & anchor_set) / union
            if jaccard > best_score:
                best_score, best_id = jaccard, cid
        if best_score >= threshold:
            return best_id, False
        new_id = md5("|".join(sorted(phrase_words)).encode()).hexdigest()[:10]
        return new_id, True

    def _apply_category_feedback(self, cluster: dict, feedback_rating: int, iter_num: int = 0):
        category = CATEGORY_FEEDBACK_MAP.get(int(feedback_rating), "general")
        cluster["category"] = category
        cluster["category_source"] = "user"
        cluster["category_confidence"] = 1.0

        if category in {"concept", "character", "details"}:
            cluster["presence_target"] = self._clip_profile_value(cluster.get("presence_target", 1.0) + 0.10)
            cluster["priority_weight"] = self._clip_profile_value(cluster.get("priority_weight", 1.0) + 0.10)
            cluster["semantic_fidelity"] = self._clip_profile_value(
                cluster.get("semantic_fidelity", 1.0) + 0.08,
                low=0.6,
                high=1.8,
            )
        elif category == "quality":
            cluster["priority_weight"] = self._clip_profile_value(cluster.get("priority_weight", 1.0) + 0.06)
            cluster["overrep_sensitivity"] = self._clip_profile_value(cluster.get("overrep_sensitivity", 1.0) + 0.08)
        elif category == "style":
            cluster["stability_weight"] = self._clip_profile_value(cluster.get("stability_weight", 1.0) + 0.08)
            cluster["user_affinity"] = self._clip_profile_value(cluster.get("user_affinity", 1.0) + 0.05)

        cluster["question_history"] = list(cluster.get("question_history", []))[-23:]
        cluster["question_history"].append({
            "iteration": iter_num,
            "type": "category",
            "rating": feedback_rating,
            "category": category,
        })
        cluster["last_question_type"] = "category"
        cluster["last_question_iter"] = iter_num

    def _build_word_concept_map(self, prompt: str, concept_clusters: dict):
        """
        Parse the prompt into concept phrases (Level 3), match or create clusters,
        and return:
          - word_to_concept: dict mapping each significant word -> cluster_id
          - ordered_concept_ids: list of cluster IDs in prompt order (no duplicates)

        Updates concept_clusters in-place. The ordered list is the backbone for
        group building and neighbour lookup at Level 4.
        """
        word_to_concept = {}
        ordered_concept_ids = []
        current_concept_labels = {}
        for phrase_words in self._parse_concepts(prompt):
            cid, is_new = self._match_concept(phrase_words, concept_clusters)
            if cid is None:
                continue
            phrase_label = " ".join(phrase_words[:6])
            if is_new:
                concept_clusters[cid] = self._default_concept_cluster(phrase_words)
            else:
                concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
                # Expand anchor vocabulary with words newly seen in this phrase
                existing = set(concept_clusters[cid]["anchor_words"])
                for w in phrase_words:
                    if w not in existing:
                        concept_clusters[cid]["anchor_words"].append(w)
                if (
                    concept_clusters[cid].get("category_source") != "user" and
                    concept_clusters[cid].get("category") == "general"
                ):
                    concept_clusters[cid]["category"] = self._infer_concept_category(
                        concept_clusters[cid]["anchor_words"]
                    )
                    concept_clusters[cid]["category_confidence"] = (
                        0.35 if concept_clusters[cid]["category"] == "general" else 0.65
                    )
            concept_clusters[cid]["last_prompt_label"] = phrase_label
            current_concept_labels[cid] = phrase_label
            for w in phrase_words:
                word_to_concept[w] = cid
            if cid not in ordered_concept_ids:
                ordered_concept_ids.append(cid)
        return word_to_concept, ordered_concept_ids, current_concept_labels

    def _build_concept_groups(self, ordered_concept_ids: list, concept_clusters: dict,
                               existing_groups: dict, current_concept_labels: dict,
                               window: int = 3):
        """
        Level 4: group consecutive concept phrases into semantic sentence-level
        units using a non-overlapping sliding window of size `window`.

        Groups model the "sentences" of the prompt — natural semantic blocks
        like (quality tags), (subject + appearance), (setting), (style + tech).

        Existing groups are matched by their concept_ids set and updated in-place
        to preserve their accumulated reward_ema and usage_count. New groups are
        created when a previously unseen concept combination appears.

        Returns the updated groups dict.
        """
        if not ordered_concept_ids:
            return existing_groups

        groups = dict(existing_groups)
        for i in range(0, len(ordered_concept_ids), window):
            chunk = ordered_concept_ids[i:i + window]
            if not chunk:
                continue
            chunk_set = frozenset(chunk)

            # Try to match an existing group by concept set identity
            existing_gid = None
            for gid, g in groups.items():
                if frozenset(g.get("concept_ids", [])) == chunk_set:
                    existing_gid = gid
                    break

            if existing_gid:
                # Preserve the canonical prompt order even if it shifted slightly
                groups[existing_gid]["concept_ids"] = chunk
                groups[existing_gid]["label"] = " | ".join(
                    current_concept_labels.get(cid, concept_clusters.get(cid, {}).get("label", cid))
                    for cid in chunk
                )
            else:
                labels = [current_concept_labels.get(cid, concept_clusters[cid]["label"])
                          for cid in chunk if cid in concept_clusters]
                gid = md5("|".join(chunk).encode()).hexdigest()[:10]
                groups[gid] = {
                    "label": " | ".join(labels),
                    "concept_ids": chunk,
                    "reward_ema": 0.0,
                    "usage_count": 0,
                    "last_seen_iter": 0,
                }

        return groups

    def _get_concept_neighbors(self, concept_id: str, ordered_concept_ids: list,
                                radius: int = 2):
        """
        Return the concept IDs positionally adjacent to `concept_id` within
        `radius` steps in the prompt's ordered concept list.

        Adjacent concepts compete for embedding space: if "anime style" is
        overrepresented, it likely crowds its neighbours "cinematic lighting"
        and "detailed background". This adjacency is what makes the neighbour
        signal in _apply_concept_feedback meaningful.
        """
        if concept_id not in ordered_concept_ids:
            return []
        idx = ordered_concept_ids.index(concept_id)
        neighbors = []
        for offset in range(-radius, radius + 1):
            if offset == 0:
                continue
            ni = idx + offset
            if 0 <= ni < len(ordered_concept_ids):
                neighbors.append(ordered_concept_ids[ni])
        return neighbors

    def _get_dominant_concept(self, ordered_concept_ids: list, concept_clusters: dict):
        """
        Return (cluster_id, avg_importance, label) for the concept with the
        highest mean word importance — i.e. what the embedding currently weighs
        most heavily. This is the "main thing forming the video" signal.
        """
        best_cid, best_score, best_label = None, -1.0, ""
        for cid in ordered_concept_ids:
            if cid not in concept_clusters:
                continue
            imp_vals = list(concept_clusters[cid]["word_importance"].values())
            if not imp_vals:
                continue
            avg = sum(imp_vals) / len(imp_vals)
            if avg > best_score:
                best_score, best_cid, best_label = avg, cid, concept_clusters[cid]["label"]
        return best_cid, best_score, best_label

    def _apply_concept_feedback(self, concept_id: str, feedback_rating: int,
                                 question_type: str,
                                 concept_clusters: dict, neighbor_ids: list,
                                 word_importance: dict, concept_groups: dict,
                                 iter_num: int = 0):
        """
        Multi-level feedback propagation.

        Level 3 — rated concept phrase:
          All word importances inside the rated cluster shift by `direct_delta`.
          This is the primary, high-confidence signal.

        Level 3 — neighbour concepts (adjacent phrases in the prompt):
          Receive a dampened signal modelled on embedding-space competition.
          - Concept being boosted (absent/weak): neighbours get a small inhibitory
            nudge, giving the boosted concept more semantic room.
          - Concept being reduced (overrepresented): neighbours also receive a mild
            reduction — the whole semantic area is too heavy.
          Magnitude: 22% of direct_delta; same sign for overrep, opposite for absent.

        Level 4 — concept groups:
          The reward_ema of every group containing the rated concept is updated,
          enabling group-level health tracking over time.

        Level 2 — global word_importance fallback:
          Each change is also written at 40% strength into the flat global dict
          so the scheduler and prodigy system retain a valid signal.

        Feedback scale:
          1 = absent           -> direct_delta = +0.90 (strong boost)
          2 = weak             -> direct_delta = +0.50
          3 = slightly weak    -> direct_delta = +0.20
          4 = slightly strong  -> direct_delta = -0.15
          5 = perfect          -> direct_delta = +0.04 (small stability reward)
          6 = overrepresented  -> direct_delta = -0.55 (strong reduction)
        """
        if concept_id not in concept_clusters:
            return

        cluster = self._ensure_concept_cluster_defaults(concept_clusters[concept_id])
        concept_clusters[concept_id] = cluster

        question_type = question_type or "presence"
        if question_type == "category":
            self._apply_category_feedback(cluster, feedback_rating, iter_num)
            return

        direct_deltas = {
            "presence": {1: 0.90, 2: 0.50, 3: 0.20, 4: -0.15, 5: 0.04, 6: -0.55},
            "priority": {1: -0.25, 2: -0.12, 3: -0.05, 4: 0.08, 5: 0.18, 6: 0.30},
            "balance": {1: 0.72, 2: 0.40, 3: 0.16, 4: -0.10, 5: 0.00, 6: -0.62},
            "fidelity": {1: 0.25, 2: 0.16, 3: 0.08, 4: 0.02, 5: 0.00, 6: -0.06},
            "stability": {1: 0.10, 2: 0.06, 3: 0.03, 4: 0.00, 5: -0.02, 6: -0.06},
            "preference": {1: -0.30, 2: -0.18, 3: -0.08, 4: 0.10, 5: 0.22, 6: 0.36},
        }
        direct_delta = direct_deltas.get(question_type, direct_deltas["presence"]).get(feedback_rating, 0.0)
        centered = (feedback_rating - 3.5) / 2.5

        # --- Level 3: rated concept ---
        local_imp = cluster["word_importance"]
        for wkey in list(local_imp.keys()):
            local_imp[wkey] = max(0.3, min(2.8, local_imp[wkey] + direct_delta))
            if wkey in word_importance:
                word_importance[wkey] = max(0.3, min(2.8, word_importance[wkey] + direct_delta * 0.4))

        if question_type == "presence":
            cluster["presence_target"] = self._clip_profile_value(
                cluster.get("presence_target", 1.0) + direct_delta * 0.22
            )
        elif question_type == "priority":
            cluster["priority_weight"] = self._clip_profile_value(
                cluster.get("priority_weight", 1.0) + centered * 0.18
            )
        elif question_type == "balance":
            cluster["overrep_sensitivity"] = self._clip_profile_value(
                cluster.get("overrep_sensitivity", 1.0) + max(0.0, centered) * 0.24 - max(0.0, -centered) * 0.12
            )
            cluster["presence_target"] = self._clip_profile_value(
                cluster.get("presence_target", 1.0) + direct_delta * 0.12
            )
        elif question_type == "fidelity":
            cluster["semantic_fidelity"] = self._clip_profile_value(
                cluster.get("semantic_fidelity", 1.0) + (centered * 0.16),
                low=0.6, high=1.8
            )
        elif question_type == "stability":
            cluster["stability_weight"] = self._clip_profile_value(
                cluster.get("stability_weight", 1.0) + centered * 0.18
            )
        elif question_type == "preference":
            cluster["user_affinity"] = self._clip_profile_value(
                cluster.get("user_affinity", 1.0) + centered * 0.20
            )
            cluster["priority_weight"] = self._clip_profile_value(
                cluster.get("priority_weight", 1.0) + centered * 0.08
            )
            cluster["presence_target"] = self._clip_profile_value(
                cluster.get("presence_target", 1.0) + centered * 0.10
            )

        # --- Level 3: neighbour concepts ---
        # Boosted concept -> inhibit neighbours (give it semantic space).
        # Reduced concept -> also soften neighbours (the whole area is too heavy).
        if direct_delta > 0:
            neighbor_delta = -direct_delta * 0.22
        else:
            neighbor_delta = direct_delta * 0.22

        for nid in neighbor_ids:
            if nid not in concept_clusters or nid == concept_id:
                continue
            concept_clusters[nid] = self._ensure_concept_cluster_defaults(concept_clusters[nid])
            n_local = concept_clusters[nid]["word_importance"]
            for wkey in list(n_local.keys()):
                n_local[wkey] = max(0.3, min(2.8, n_local[wkey] + neighbor_delta))
                if wkey in word_importance:
                    word_importance[wkey] = max(0.3, min(2.8,
                                                word_importance[wkey] + neighbor_delta * 0.4))

        # --- Level 4: concept group reward EMA ---
        # Maps feedback 1-6 onto a [-1, +1] reward signal.
        group_reward_map = {1: -1.0, 2: -0.5, 3: -0.1, 4: 0.4, 5: 1.0, 6: -0.6}
        normalized = group_reward_map.get(feedback_rating, 0.0)
        for gid, g in concept_groups.items():
            if concept_id in g.get("concept_ids", []):
                g["reward_ema"] = 0.75 * g.get("reward_ema", 0.0) + 0.25 * normalized
                g["usage_count"] = g.get("usage_count", 0) + 1

        cluster["question_history"] = list(cluster.get("question_history", []))[-23:]
        cluster["question_history"].append({
            "iteration": iter_num,
            "type": question_type,
            "rating": feedback_rating,
        })
        cluster["last_question_type"] = question_type
        cluster["last_question_iter"] = iter_num

    # =========================================================================
    # SIGMA REFINEMENT
    # =========================================================================

    def _ensure_sigma_state_defaults(self, global_adaptive: dict):
        global_adaptive.setdefault("sigma_profile", [0.0] * 32)
        global_adaptive.setdefault("last_applied_sigma_profile", [0.0] * 32)
        global_adaptive.setdefault("sigma_iterations", 0)
        global_adaptive.setdefault("sigma_avg_reward_ema", 0.0)
        global_adaptive.setdefault("sigma_exploration_base", 0.035)
        global_adaptive.setdefault("sigma_history", [])

    def _resolve_sigma_strength(self, strength_mode: str):
        strength_map = {
            "off": 0.0,
            "subtle": 0.18,
            "medium": 0.35,
            "strong": 0.60,
            "max": 1.0,
        }
        return strength_map.get((strength_mode or "subtle").lower(), 0.18)

    def _sigma_resample_profile(self, profile, target_len):
        if target_len <= 0:
            return np.zeros((0,), dtype=np.float32)

        arr = np.asarray(profile, dtype=np.float32)
        if arr.size == 0:
            arr = np.zeros((32,), dtype=np.float32)
        if arr.size == target_len:
            return arr.copy()
        if arr.size == 1:
            return np.full((target_len,), float(arr[0]), dtype=np.float32)

        src = np.linspace(0.0, 1.0, arr.size)
        dst = np.linspace(0.0, 1.0, target_len)
        return np.interp(dst, src, arr).astype(np.float32)

    def _sigma_smooth_noise(self, noise):
        if noise.size <= 2:
            return noise
        kernel = np.array([0.25, 0.5, 0.25], dtype=np.float32)
        smoothed = np.convolve(noise, kernel, mode="same")
        smoothed = np.convolve(smoothed, kernel, mode="same")
        return smoothed.astype(np.float32)

    def _sigma_enforce_monotonic(self, tuned: torch.Tensor, original: torch.Tensor):
        out = tuned.clone()
        count = int(out.shape[0])
        if count <= 2:
            return out

        eps = max(1e-6, float((original[0] - original[-1]).abs().item()) * 1e-6)

        for i in range(1, count - 1):
            max_allowed = out[i - 1] - eps
            if out[i] >= max_allowed:
                out[i] = max_allowed

        out[-1] = original[-1]
        for i in range(count - 2, 0, -1):
            min_allowed = out[i + 1] + eps
            if out[i] <= min_allowed:
                out[i] = min_allowed

        out[0] = original[0]
        out[-1] = original[-1]
        return out

    def _refine_sigma_schedule(self, sigmas, rating: int, global_adaptive: dict, strength_mode: str, seed: int,
                               rating_profile: Optional[dict] = None):
        if not isinstance(sigmas, torch.Tensor):
            return torch.FloatTensor([]), "Sigma refinement inactive."

        original_sigmas = sigmas.detach().clone()
        if original_sigmas.numel() <= 2:
            return original_sigmas, "Sigma refinement skipped: schedule too short."

        self._ensure_sigma_state_defaults(global_adaptive)

        rating_profile = rating_profile or normalize_refiner_rating(rating)
        rating_key = rating_profile.get("key", "")
        reward = float(rating_profile.get("reward", (rating - 5.5) / 4.5))
        sigma_iterations = int(global_adaptive.get("sigma_iterations", 0))

        profile = np.asarray(global_adaptive.get("sigma_profile", [0.0] * 32), dtype=np.float32)
        last_applied = np.asarray(global_adaptive.get("last_applied_sigma_profile", [0.0] * 32), dtype=np.float32)
        sigma_lr = 0.22 / (1.0 + 0.08 * math.sqrt(max(1, sigma_iterations)))
        profile = np.clip(profile + (last_applied * reward * sigma_lr), -1.0, 1.0)
        profile = np.clip(profile - profile.mean(), -1.0, 1.0)

        sigma_avg_reward_ema = float(global_adaptive.get("sigma_avg_reward_ema", 0.0))
        sigma_avg_reward_ema = 0.85 * sigma_avg_reward_ema + 0.15 * reward
        sigma_exploration_base = float(global_adaptive.get("sigma_exploration_base", 0.035))
        sigma_exploration_scale = max(0.18, 1.0 - max(0.0, sigma_avg_reward_ema) * 0.7)

        if seed != 0:
            np.random.seed(seed % (2 ** 32))
        noise = np.random.normal(
            0.0,
            sigma_exploration_base * sigma_exploration_scale,
            size=profile.shape[0]
        ).astype(np.float32)
        applied_profile = np.clip(profile + self._sigma_smooth_noise(noise), -1.0, 1.0)
        applied_profile = np.clip(applied_profile - applied_profile.mean(), -1.0, 1.0)

        middle_profile = self._sigma_resample_profile(applied_profile, max(0, original_sigmas.shape[0] - 2))
        middle_profile_tensor = torch.tensor(middle_profile, dtype=original_sigmas.dtype, device=original_sigmas.device)

        tuned_sigmas = original_sigmas.clone()
        sigma_strength = self._resolve_sigma_strength(strength_mode)
        for idx in range(1, int(original_sigmas.shape[0]) - 1):
            delta = float(middle_profile_tensor[idx - 1].item())
            current = original_sigmas[idx]
            prev_sigma = original_sigmas[idx - 1]
            next_sigma = original_sigmas[idx + 1]
            if delta >= 0.0:
                tuned_sigmas[idx] = current + (prev_sigma - current) * sigma_strength * delta
            else:
                tuned_sigmas[idx] = current + (current - next_sigma) * sigma_strength * delta

        tuned_sigmas = self._sigma_enforce_monotonic(tuned_sigmas, original_sigmas)

        global_adaptive["sigma_profile"] = profile.tolist()
        global_adaptive["last_applied_sigma_profile"] = applied_profile.tolist()
        global_adaptive["sigma_iterations"] = sigma_iterations + 1
        global_adaptive["sigma_avg_reward_ema"] = sigma_avg_reward_ema
        global_adaptive["sigma_exploration_base"] = max(
            0.012,
            min(0.05, sigma_exploration_base * (0.97 if rating_key == "like" else 1.02))
        )
        sigma_history = list(global_adaptive.get("sigma_history", []))[-119:]
        sigma_history.append({
            "iteration": sigma_iterations + 1,
            "rating": int(rating),
            "rating_label": rating_profile.get("label", str(rating)),
            "reward": round(float(reward), 6),
            "lr": round(float(sigma_lr), 6),
        })
        global_adaptive["sigma_history"] = sigma_history

        mean_shift = float((tuned_sigmas[1:-1] - original_sigmas[1:-1]).abs().mean().item()) if tuned_sigmas.numel() > 2 else 0.0
        max_shift = float((tuned_sigmas[1:-1] - original_sigmas[1:-1]).abs().max().item()) if tuned_sigmas.numel() > 2 else 0.0
        sigma_status = (
            f"Sigma: iter {global_adaptive['sigma_iterations']} | strength {strength_mode} ({sigma_strength:.2f}) | "
            f"mean shift {mean_shift:.6f} | max shift {max_shift:.6f} | "
            f"endpoints preserved ({float(original_sigmas[0].item()):.6f} -> {float(original_sigmas[-1].item()):.6f})"
        )
        return tuned_sigmas, sigma_status

    # =========================================================================
    # LATENT REFINEMENT
    # =========================================================================

    def _resize_tensor_like(self, tensor: torch.Tensor, reference: torch.Tensor):
        if not isinstance(tensor, torch.Tensor) or not isinstance(reference, torch.Tensor):
            return None

        out = tensor.to(device=reference.device, dtype=reference.dtype)
        if list(out.shape) == list(reference.shape):
            return out
        if out.dim() != reference.dim() or out.dim() not in {4, 5}:
            return None

        batch = reference.shape[0]
        if out.shape[0] < batch:
            reps = [1] * out.dim()
            reps[0] = math.ceil(batch / max(1, out.shape[0]))
            out = out.repeat(*reps)
        out = out[:batch]

        channels = reference.shape[1]
        if out.shape[1] < channels:
            pad_shape = list(out.shape)
            pad_shape[1] = channels - out.shape[1]
            out = torch.cat([out, torch.zeros(pad_shape, device=out.device, dtype=out.dtype)], dim=1)
        out = out[:, :channels]

        if out.dim() == 4:
            return F.interpolate(out, size=reference.shape[-2:], mode="bilinear", align_corners=False)
        return F.interpolate(out, size=reference.shape[-3:], mode="trilinear", align_corners=False)

    def _load_saved_latent(self, refinement_key, mode):
        path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt")
        if not os.path.exists(path):
            return None, "Latent: no saved latent for this key."

        try:
            data = torch.load(path, map_location="cpu", weights_only=False)
        except Exception as e:
            return None, f"Latent: failed to load saved latent ({e})."

        if not isinstance(data, dict) or not isinstance(data.get("samples"), torch.Tensor):
            return None, "Latent: saved latent file is invalid."

        return data, "Latent: saved latent loaded."

    def _value_links_to_output(self, value, source_id, output_index):
        if isinstance(value, (list, tuple)):
            if len(value) >= 2 and str(value[0]) == source_id:
                try:
                    if int(value[1]) == output_index:
                        return True
                except (TypeError, ValueError):
                    pass
            return any(self._value_links_to_output(item, source_id, output_index) for item in value)

        if isinstance(value, dict):
            return any(self._value_links_to_output(item, source_id, output_index) for item in value.values())

        return False

    def _is_output_connected(self, prompt, unique_id, output_index):
        if prompt is None or unique_id is None:
            return False

        source_id = str(unique_id)
        prompt_nodes = prompt.get("output", prompt) if isinstance(prompt, dict) else {}
        if not isinstance(prompt_nodes, dict):
            return False

        for node in prompt_nodes.values():
            if not isinstance(node, dict):
                continue
            inputs = node.get("inputs", {})
            if isinstance(inputs, dict) and self._value_links_to_output(inputs, source_id, output_index):
                return True

        workflow = prompt.get("workflow") if isinstance(prompt, dict) else None
        links = workflow.get("links", []) if isinstance(workflow, dict) else []
        for link in links:
            if isinstance(link, (list, tuple)) and len(link) >= 3:
                try:
                    if str(link[1]) == source_id and int(link[2]) == output_index:
                        return True
                except (TypeError, ValueError):
                    continue
            elif isinstance(link, dict):
                try:
                    if str(link.get("origin_id")) == source_id and int(link.get("origin_slot")) == output_index:
                        return True
                except (TypeError, ValueError):
                    continue

        return False

    def _latent_refinement_disabled(self, latent):
        return clone_latent(latent), "Latent: disabled because latent refinement input/output path is not fully connected."

    def _raise_wrong_latent(self, latent):
        sample_type = latent_sample_type_name(latent)
        latent_type = latent.get("type", "unspecified") if isinstance(latent, dict) else "missing"
        raise ValueError(f"{self.WRONG_LATENT_ERROR} Received samples={sample_type}, type={latent_type}.")

    def _save_latent_reference(self, latent, refinement_key, mode):
        samples = latent_samples(latent)
        if samples is None:
            return False

        path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt")
        os.makedirs(os.path.dirname(path), exist_ok=True)
        bundle = cpu_tensor_bundle(latent)
        bundle["_meta"] = {
            "refinement_key": refinement_key,
            "mode": (mode or "ltx2").lower(),
            "samples_shape": list(samples.shape),
        }
        torch.save(bundle, path)
        return True

    def _delete_latent_reference(self, refinement_key, mode):
        path = refinement_state_path(refinement_key, mode, prefix="latent", extension="pt")
        try:
            os.remove(path)
            return True
        except FileNotFoundError:
            return False
        except OSError:
            return False

    def _refine_latent(self, latent, refinement_key, mode, rating, reward, global_adaptive,
                       rating_profile: Optional[dict] = None):
        if isinstance(latent, dict) and not latent_is_plain_video_tensor(latent):
            self._raise_wrong_latent(latent)

        current = latent_samples(latent)
        saved_bundle, load_status = self._load_saved_latent(refinement_key, mode)

        if current is None:
            if saved_bundle is None:
                raise ValueError(self.NO_LATENT_REFERENCE_ERROR)

            saved_latent = latent_from_tensor_bundle(saved_bundle)
            saved_samples = latent_samples(saved_latent)
            if saved_samples is None:
                raise ValueError(self.NO_LATENT_REFERENCE_ERROR)

            latent_state = global_adaptive.setdefault("latent_refinement", {})
            latent_state["last_shape"] = list(saved_samples.shape)
            latent_state["saved_shape"] = list(saved_samples.shape)
            latent_state["status"] = self.SAVED_LATENT_ONLY_STATUS
            return saved_latent, latent_state["status"]

        refined_latent = clone_latent(latent)
        if saved_bundle is None:
            self._save_latent_reference(latent, refinement_key, mode)
            global_adaptive.setdefault("latent_refinement", {})["last_shape"] = list(current.shape)
            return refined_latent, f"{load_status} Current latent saved as reference."

        raw_saved_samples = saved_bundle.get("samples")
        if list(raw_saved_samples.shape) != list(current.shape):
            self._save_latent_reference(latent, refinement_key, mode)
            latent_state = global_adaptive.setdefault("latent_refinement", {})
            latent_state.pop("momentum", None)
            latent_state["last_shape"] = list(current.shape)
            latent_state["saved_shape"] = list(raw_saved_samples.shape)
            return refined_latent, "Latent: input shape changed. Reference rewritten, passthrough."

        saved_samples = raw_saved_samples.to(device=current.device, dtype=current.dtype)
        latent_state = global_adaptive.setdefault("latent_refinement", {})
        latent_state["last_shape"] = list(current.shape)
        latent_state["saved_shape"] = list(raw_saved_samples.shape)

        momentum_data = latent_state.get("momentum")
        if isinstance(momentum_data, dict):
            try:
                momentum = serializable_to_tensor(momentum_data).to(device=current.device, dtype=current.dtype)
                momentum = self._resize_tensor_like(momentum, current)
            except Exception:
                momentum = None
        else:
            momentum = None
        if momentum is None:
            momentum = torch.zeros_like(current)

        nonzero_mask = current.ne(0) & saved_samples.ne(0)
        if not bool(nonzero_mask.any()):
            refined_latent["samples"] = current
            latent_state["status"] = "Latent: only zero-valued positions available, passthrough."
            return refined_latent, latent_state["status"]

        rating_profile = rating_profile or normalize_refiner_rating(rating)
        rating_key = rating_profile.get("key", "")
        missing_axes = set(rating_profile.get("missing_axes", []))
        if rating_key == "like":
            lr = 0.035
        elif missing_axes:
            lr = 0.018
            if "details" in missing_axes:
                lr += 0.006
            if "concept" in missing_axes:
                lr += 0.012
            if "quality" in missing_axes:
                lr += 0.035
            lr = min(0.065, lr)
        elif rating_key == "dislike":
            lr = 0.055
        else:
            lr = 0.018
        target_delta = (saved_samples - current) * nonzero_mask.to(dtype=current.dtype)
        update = target_delta * (reward * lr)
        momentum = 0.82 * momentum + 0.18 * update
        max_step = max(0.001, min(0.08, float(current.detach().abs().mean().item()) * 0.18))
        latent_delta = torch.clamp(momentum, min=-max_step, max=max_step)

        refined_samples = current + latent_delta
        refined_samples = torch.where(nonzero_mask, refined_samples, current)
        refined_samples = torch.nan_to_num(refined_samples, nan=0.0)
        range_min = torch.minimum(current, saved_samples)
        range_max = torch.maximum(current, saved_samples)
        refined_samples = torch.maximum(torch.minimum(refined_samples, range_max), range_min)
        refined_latent["samples"] = refined_samples

        latent_state["momentum"] = tensor_to_serializable(momentum.detach().cpu())
        latent_state["last_rating"] = int(rating)
        latent_state["last_rating_label"] = rating_profile.get("label", str(rating))
        latent_state["last_reward"] = round(float(reward), 4)
        latent_state["nonzero_ratio"] = round(float(nonzero_mask.float().mean().item()), 6)
        latent_state["status"] = (
            f"Latent: adjusted shape {tuple(current.shape)} | nonzero {latent_state['nonzero_ratio']:.1%} | "
            f"max step {max_step:.5f} | bounded to input/reference range"
        )
        return refined_latent, latent_state["status"]

    # =========================================================================
    # LORA WEIGHT SUGGESTIONS
    # =========================================================================

    def _lora_state_id(self, lora_name: str, lora_type: str):
        return md5(f"{lora_name}::{lora_type}".encode("utf-8")).hexdigest()[:16]

    def _lora_words(self, text: str):
        return {
            word.strip().lower()
            for word in re.split(r"[\s,;:_\\/\-().\[\]{}]+", text or "")
            if self._is_valuable_token(word.strip())
        }

    def _ensure_lora_memory(self, memory: dict, lora_entry: dict):
        lora_type = lora_entry.get("type", "general")
        lora_id = lora_entry.get("id") or self._lora_state_id(lora_entry.get("name", ""), lora_type)
        state = memory.setdefault(
            lora_id,
            {
                "name": lora_entry.get("name", ""),
                "type": lora_type,
                "offset_ratio": 0.0,
                "stable_offset_ratio": None,
                "reward_ema": 0.0,
                "culprit_score": 0.0,
                "good_streak": 0,
                "bad_streak": 0,
                "culprit_hits": 0,
                "iterations": 0,
            },
        )
        state["name"] = lora_entry.get("name", "")
        state["type"] = lora_type
        state.setdefault("offset_ratio", 0.0)
        state.setdefault("stable_offset_ratio", None)
        state.setdefault("reward_ema", 0.0)
        state.setdefault("culprit_score", 0.0)
        state.setdefault("good_streak", 0)
        state.setdefault("bad_streak", 0)
        state.setdefault("culprit_hits", 0)
        state.setdefault("iterations", 0)
        return lora_id, state

    def _score_lora_prompt_relation(self, lora_entry, ordered_concept_ids, concept_clusters, current_concept_labels):
        lora_type = lora_entry.get("type", "general")
        lora_words = self._lora_words(lora_entry.get("name", ""))

        best_score = 0.35 if lora_type == "general" else 0.0
        best_labels = []
        best_importance = 1.0

        for cid in ordered_concept_ids:
            cluster = concept_clusters.get(cid)
            if not cluster:
                continue
            cluster = self._ensure_concept_cluster_defaults(cluster)
            label = current_concept_labels.get(cid, cluster.get("label", cid))
            label_words = self._lora_words(label)
            anchor_words = set(cluster.get("anchor_words", []))
            prompt_words = label_words | anchor_words
            category = cluster.get("category", "general")

            overlap = len(lora_words & prompt_words) / max(1, len(lora_words)) if lora_words else 0.0
            category_score = 0.62 if lora_type == category else 0.0
            if lora_type in {"action", "concept"} and category in {"concept", "details", "subject", "appearance", "action", "environment"}:
                category_score = max(category_score, 0.58)
            if lora_type == "character" and category in {"character", "subject", "appearance"}:
                category_score = max(category_score, 0.72)
            if lora_type == "quality" and category == "quality":
                category_score = 0.78
            if lora_type == "style" and category in {"style", "camera"}:
                category_score = max(category_score, 0.58)

            score = max(category_score, overlap)
            if score > best_score:
                best_score = score
                best_labels = [label]
                best_importance = self._get_concept_mean_importance(cluster)
            elif score > 0 and abs(score - best_score) < 1e-6:
                best_labels.append(label)

        return min(1.0, best_score), best_labels[:4], best_importance

    def _update_lora_weight_suggestions(self, lora_stack, active, global_adaptive, ordered_concept_ids,
                                        concept_clusters, current_concept_labels, rating, reward,
                                        rating_profile: Optional[dict] = None):
        if not isinstance(lora_stack, dict) or not lora_stack.get("loras"):
            return "LoRA suggestions: no FunPack LoRA stack connected."

        rating_profile = rating_profile or normalize_refiner_rating(rating)
        rating_key = rating_profile.get("key", "")
        missing_axes = set(rating_profile.get("missing_axes", []))
        missing_count = len(missing_axes)
        memory = global_adaptive.setdefault("lora_weight_memory", {})
        suggestions = {}
        status_parts = []
        relation_cache = {}
        top_concept_relation = 0.0
        concept_lora_count = 0

        for entry in lora_stack.get("loras", []):
            lora_type = entry.get("type", "general")
            lora_id = entry.get("id") or self._lora_state_id(entry.get("name", ""), lora_type)
            relation, matched_labels, concept_importance = self._score_lora_prompt_relation(
                entry,
                ordered_concept_ids,
                concept_clusters,
                current_concept_labels,
            )
            relation_cache[lora_id] = (relation, matched_labels, concept_importance)
            if lora_type in {"action", "concept", "character"}:
                concept_lora_count += 1
                top_concept_relation = max(top_concept_relation, relation)

        for entry in lora_stack.get("loras", []):
            lora_type = entry.get("type", "general")
            profile = LORA_REFINER_TYPE_PROFILES.get(lora_type, LORA_REFINER_TYPE_PROFILES["general"])
            lora_id, state = self._ensure_lora_memory(memory, entry)
            relation, matched_labels, concept_importance = relation_cache.get(lora_id, (0.0, [], 1.0))

            state["reward_ema"] = 0.84 * float(state.get("reward_ema", 0.0)) + 0.16 * reward
            if rating_key == "like":
                state["good_streak"] = int(state.get("good_streak", 0)) + 1
                state["bad_streak"] = 0
            elif rating_key == "dislike" or {"concept", "quality"}.issubset(missing_axes):
                state["bad_streak"] = int(state.get("bad_streak", 0)) + 1
                state["good_streak"] = 0
            else:
                state["good_streak"] = 0
                state["bad_streak"] = 0

            offset = float(state.get("offset_ratio", 0.0))
            stable_offset = state.get("stable_offset_ratio")
            culprit_score = float(state.get("culprit_score", 0.0))
            culprit_hits = int(state.get("culprit_hits", 0))
            if stable_offset is not None and (rating_key == "like" or missing_axes == {"details"}):
                offset = 0.72 * offset + 0.28 * float(stable_offset)

            step = profile["step"] * max(0.15, relation)
            max_offset = profile["max_offset"]
            min_offset = profile["min_offset"]
            effective_relation = max(relation, profile.get("culprit_bias", 0.0))
            base_model = float(entry.get("base_model_weight", entry.get("model_weight", 1.0)))
            base_abs = abs(base_model)
            is_concept_lora = lora_type in {"action", "concept"}
            concept_match_strength = 1.0
            if is_concept_lora:
                concept_match_strength += 0.30 if matched_labels else 0.0
                concept_match_strength += max(0.0, relation - 0.30) * 1.35

            is_primary_concept_lora = (
                lora_type in {"action", "concept", "character"} and
                (
                    concept_lora_count == 1 or
                    (top_concept_relation > 0.0 and relation >= max(0.30, top_concept_relation - 1e-6))
                )
            )

            if rating_key == "like":
                value_mult = 0.75 + min(1.4, max(0.5, concept_importance)) * 0.25
                offset += step * (0.45 + max(0.0, reward)) * value_mult
                culprit_score *= 0.72
                if state["good_streak"] >= 3:
                    if stable_offset is None:
                        stable_offset = offset
                    else:
                        stable_offset = 0.78 * float(stable_offset) + 0.22 * offset
                    state["stable_offset_ratio"] = _clamp(stable_offset, min_offset, max_offset)
                    offset = state["stable_offset_ratio"]
            elif missing_axes:
                if "concept" in missing_axes or "quality" in missing_axes:
                    state["stable_offset_ratio"] = None

                culprit_score *= 0.74 if rating_key == "awful" else 0.82
                culprit_hits = max(0, culprit_hits - 1)
                axis_boost = 0.0

                if "details" in missing_axes:
                    if relation >= 0.12 and lora_type in {"action", "concept", "character", "general", "style"}:
                        value_mult = 0.85 + min(1.2, max(0.5, concept_importance)) * 0.20
                        axis_boost += (0.26 + relation * 0.34) * value_mult
                    elif lora_type == "quality":
                        axis_boost += 0.06

                if "concept" in missing_axes:
                    if lora_type in {"action", "concept", "character"}:
                        if is_primary_concept_lora:
                            axis_boost += (0.95 + abs(reward) * 0.45) * concept_match_strength
                        elif relation > 0.08:
                            axis_boost += 0.34 + relation * 0.40
                    elif lora_type == "general" and relation >= 0.25:
                        axis_boost += 0.25 + relation * 0.25

                if "quality" in missing_axes:
                    if lora_type == "quality":
                        axis_boost += 0.90 + abs(reward) * 0.35
                    elif lora_type == "style" and relation >= 0.25:
                        axis_boost += 0.20 + relation * 0.20
                    elif lora_type == "general" and relation >= 0.20:
                        axis_boost += 0.16 + relation * 0.15

                if axis_boost > 0.0:
                    if rating_key == "awful":
                        axis_boost *= 1.25
                    max_offset = max(max_offset, profile["bad_max_offset"] * (0.62 + 0.10 * missing_count))
                    relation_floor = 0.35 if missing_count >= 2 else 0.22
                    boost_step = profile["step"] * max(relation_floor, effective_relation)
                    offset += boost_step * axis_boost
                elif relation <= 0.05:
                    offset *= 0.97
                else:
                    offset *= 0.94
            elif rating_key == "dislike":
                severity = _clamp((5.0 - float(rating)) / 4.0, 0.0, 1.0)
                culprit_signal = max(0.20, effective_relation) * (0.70 + base_abs * 0.30)
                if is_concept_lora:
                    culprit_signal *= concept_match_strength
                culprit_score = _clamp(culprit_score * 0.72 + severity * culprit_signal, 0.0, 2.5)
                culprit_hits = culprit_hits + 1 if culprit_score >= 0.45 else max(0, culprit_hits - 1)
                max_offset = profile["bad_max_offset"] if rating <= 2 else max(max_offset, profile["bad_max_offset"] * 0.72)
                bad_floor_strength = min(1.0, 0.45 + 0.35 * culprit_score + 0.14 * state["bad_streak"])
                if is_concept_lora:
                    bad_floor_strength = min(1.0, bad_floor_strength + 0.18 * concept_match_strength)
                min_offset = min(min_offset, profile["bad_min_offset"] * bad_floor_strength)
                offset -= step * (1.0 + severity * 3.0) * max(0.4, effective_relation) * max(0.8, 0.85 + culprit_score)
                if state["bad_streak"] >= 2:
                    offset -= step * (0.65 + severity * 1.8) * max(0.25, effective_relation)
                if is_concept_lora and state["bad_streak"] >= 2:
                    offset -= step * (0.85 + severity * 2.4) * max(0.50, effective_relation) * concept_match_strength
                if state["bad_streak"] >= 2:
                    state["stable_offset_ratio"] = None
                if is_concept_lora and matched_labels and state["bad_streak"] >= 2 and culprit_score >= 0.70:
                    offset = min(offset, -1.0)
                if state["bad_streak"] >= 3 and culprit_score >= 0.85 and abs(1.0 + offset) < 0.08:
                    offset = -1.0
            else:
                culprit_score *= 0.92
                culprit_hits = max(0, culprit_hits - 1)
                if relation <= 0.05:
                    offset *= 0.94
                elif stable_offset is not None:
                    offset = 0.65 * offset + 0.35 * float(stable_offset)
                else:
                    offset *= 0.90

            offset = _clamp(offset, min_offset, max_offset)
            state["offset_ratio"] = offset
            state["culprit_score"] = culprit_score
            state["culprit_hits"] = culprit_hits
            state["iterations"] = int(state.get("iterations", 0)) + 1

            model_weight = base_model * (1.0 + offset)
            suspect = culprit_score >= 0.65 or model_weight <= 0.0 or culprit_hits >= 2
            action = (
                "invert" if model_weight < 0.0 else
                "mute" if model_weight == 0.0 else
                "hold" if abs(model_weight - base_model) < 1e-6 else
                "reduce" if abs(model_weight) < abs(base_model) else
                "boost"
            )
            suggestions[lora_id] = {
                "name": entry.get("name", ""),
                "type": lora_type,
                "model_weight": model_weight,
                "base_model_weight": base_model,
                "offset_ratio": offset,
                "culprit_score": culprit_score,
                "culprit_hits": culprit_hits,
                "suspect": suspect,
                "action": action,
                "relation": relation,
                "matched_concepts": matched_labels,
                "rating": int(rating),
                "rating_label": rating_profile.get("label", str(rating)),
                "rating_range": rating_profile.get("legacy_range", ""),
                "good_streak": state.get("good_streak", 0),
                "bad_streak": state.get("bad_streak", 0),
                "stable": state.get("stable_offset_ratio") is not None,
            }

            match_text = ",".join(matched_labels) if matched_labels else "none"
            status_parts.append(
                f"{entry.get('name', '?')}[{lora_type}] rel={relation:.2f} "
                f"offset={offset:+.3f} next={model_weight:+.3f} "
                f"sus={culprit_score:.2f}{' !' if suspect else ''} "
                f"match={match_text}"
            )

        active["lora_weight_suggestions"] = suggestions
        active["last_lora_stack"] = lora_stack
        if not status_parts:
            return "LoRA suggestions: stack empty."
        return f"LoRA suggestions ({rating_profile.get('label', str(rating))}): " + " | ".join(status_parts)

    # =========================================================================
    # MAIN REFINE
    # =========================================================================

    def refine(self, positive_conditioning, mode: str, rating: int, refinement_key: str,
               scheduler_mode: str = "original", positive_prompt: str = "",
               clip=None, reset_session: bool = False, unlimited_history: bool = False,
               seed: int = 0, feedback_enabled: bool = False, feedback_rating: int = 3,
               sigmas=None, sigma_strength: str = "subtle", lora_stack=None, latent=None,
               into_the_void: bool = False, im_feeling_lucky: bool = False,
               prompt=None, unique_id=None):

        mode = (mode or "ltx2").lower()
        if mode not in self._tokenizer_sources:
            mode = "ltx2"
        rating_profile = normalize_refiner_rating(rating)
        rating_label = rating_profile.get("label", str(rating))
        rating = int(rating_profile.get("legacy_score", 6))
        reward = float(rating_profile.get("reward", (rating - 5.5) / 4.5))

        if seed != 0:
            torch.manual_seed(seed)
            random.seed(seed)

        refinements_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "refinements")
        os.makedirs(refinements_dir, exist_ok=True)

        json_file = refinement_state_path(refinement_key, mode)
        fallback_loss_graph = render_refinement_loss_graph(
            refinement_key=refinement_key,
            scheduler_mode=scheduler_mode,
            mode=mode,
            total_iterations=0,
            latest_learning_loss=0.0,
            points=[],
        )
        fallback_sigmas = sigmas.detach().clone() if isinstance(sigmas, torch.Tensor) else torch.FloatTensor([])
        latent_output_connected = self._is_output_connected(prompt, unique_id, self.LATENT_OUTPUT_INDEX)
        fallback_latent = clone_latent(latent)
        fallback_latent_status = "Latent: not evaluated before conditioning validation."

        if not positive_conditioning or not isinstance(positive_conditioning, list) or len(positive_conditioning) == 0:
            return (positive_conditioning, "ERROR: Empty positive CONDITIONING input", "", "ERROR: No positive conditioning", fallback_loss_graph, fallback_sigmas, fallback_latent)

        item = positive_conditioning[0]
        if isinstance(item, (list, tuple)) and len(item) >= 2:
            raw_positive = item[0]
            positive_meta = item[1] if isinstance(item[1], dict) else {"pooled_output": None}
        else:
            raw_positive = item if isinstance(item, torch.Tensor) else None
            positive_meta = {"pooled_output": None}

        if not isinstance(raw_positive, torch.Tensor):
            return (positive_conditioning, "ERROR: No positive embedding tensor found", "", "ERROR: Invalid embedding", fallback_loss_graph, fallback_sigmas, fallback_latent)

        if rating_profile.get("skip_learning"):
            pending_cleared = False
            if os.path.exists(json_file):
                try:
                    with open(json_file, "r", encoding="utf-8") as f:
                        data = json.load(f)
                    if data.get("pending_feedback") is not None:
                        data["pending_feedback"] = None
                        with open(json_file, "w", encoding="utf-8") as f:
                            json.dump(data, f, indent=2)
                        pending_cleared = True
                except (json.JSONDecodeError, OSError, ValueError):
                    pending_cleared = False
            pending_status = " Pending feedback queue was cleared." if pending_cleared else ""
            status = (
                f"Feedback ignored | Mode {mode.upper()} | Rating {rating_label}\n"
                "No learning history, LoRA weights, sigma schedule, or latent reference was updated."
                f"{pending_status}"
            )
            training_info = (
                f"Rating: {rating_label}\n"
                "This run was intentionally forgotten. The connected conditioning, sigmas, and latent pass through unchanged."
                f"{pending_status}"
            )
            return (positive_conditioning, status, "", training_info, fallback_loss_graph, fallback_sigmas, fallback_latent)

        analysis_prompt = self._normalize_prompt_for_mode(positive_prompt, mode)
        prompt_key = analysis_prompt if mode == "wan" else positive_prompt
        exact_prompt_key = prompt_key
        if im_feeling_lucky:
            prompt_key = "__lucky_memory__"
        current_prompt_words = self._concept_overlap_words(analysis_prompt)
        prompt_variant_match = None

        # ====================== STATE TEMPLATES ======================
        # Single source of truth for both reset and corrupt-recovery paths.
        def _fresh_global():
            state = {
                "word_importance": {},
                "concept_clusters": {},
                "concept_groups": {},
                "exploration_base": 0.08,
                "momentum": None,
                "avg_reward_ema": 0.0,
                "good_ratio": 0.0,
                "dynamic_sim_threshold": 0.82,
                "last_feedback_concept": None,
                "feedbacked_concepts": [],
                "feedback_memory": {
                    "recent_questions": [],
                    "rating_change_events": [],
                },
                "loss_history": [],
                "lora_weight_memory": {},
                "void_token_bank": {},
                "void_token_pairs": {},
                "lucky_context_memory": {},
                "lucky_phrase_placements": {},
                "mode": mode,
                "scheduler_mode": scheduler_mode,
                "prodigy_d": {},
                "prodigy_lr_base": 1.0,
                "warmup_steps": 8,
                "total_steps_estimate": 150,
                "current_step": 0,
            }
            self._ensure_sigma_state_defaults(state)
            return state

        def _fresh_data():
            return {
                "refinement_key": refinement_key,
                "global_adaptive": _fresh_global(),
                "prompt_histories": {
                    prompt_key: {
                        "canonical_prompt": positive_prompt,
                        "prompt_concept_words": current_prompt_words,
                        "source_prompt_key": exact_prompt_key,
                        "source_conditioning_embeds": tensor_to_serializable(raw_positive),
                        "reference_embeds": tensor_to_serializable(raw_positive),
                        "liked_reference_embeds": None,
                        "liked_reference_count": 0,
                        "history": [],
                        "last_rating": rating,
                        "last_rating_label": rating_label
                    }
                },
                "last_prompt_key": prompt_key,
                "pending_feedback": None
            }

        def _seed_fresh_prompt_discovery(data):
            if not analysis_prompt or not isinstance(data, dict):
                return False
            global_state = data.get("global_adaptive")
            if not isinstance(global_state, dict):
                return False
            seq_len = self._get_conditioning_seq_len(raw_positive)
            token_mask = self._get_conditioning_token_mask(raw_positive) if mode == "wan" else None
            discovery_groups = self._build_word_groups(
                analysis_prompt,
                None,
                seq_len,
                token_mask=token_mask,
            )
            if discovery_groups:
                self._seed_void_token_bank(
                    global_state,
                    discovery_groups,
                    raw_positive,
                    0,
                    token_mask=token_mask,
                )
            self._update_lucky_phrase_placements(global_state, analysis_prompt, "discover", 0)
            return bool(discovery_groups)

        # ====================== RESET / NEW SESSION ======================
        lucky_bootstrap = False
        fresh_discovery_seeded = False
        if reset_session or not os.path.exists(json_file):
            if reset_session:
                self._delete_latent_reference(refinement_key, mode)
            data = _fresh_data()
            fresh_discovery_seeded = _seed_fresh_prompt_discovery(data)
            with open(json_file, "w", encoding="utf-8") as f:
                json.dump(data, f, indent=2)
            if im_feeling_lucky:
                lucky_bootstrap = True
            else:
                if latent_output_connected:
                    fallback_latent, fallback_latent_status = self._refine_latent(
                        latent,
                        refinement_key,
                        mode,
                        rating,
                        reward,
                        {},
                        rating_profile,
                    )
                else:
                    fallback_latent, fallback_latent_status = self._latent_refinement_disabled(latent)
                return (positive_conditioning, "New session started - Reference saved", "", f"New session started. Reference embedding saved.\n{fallback_latent_status}", fallback_loss_graph, fallback_sigmas, fallback_latent)
        else:
            # ====================== SAFE JSON LOAD ======================
            try:
                with open(json_file, "r", encoding="utf-8") as f:
                    data = json.load(f)
            except (json.JSONDecodeError, OSError, ValueError) as e:
                print(f"[FunPackVideoRefiner] Corrupt session file, resetting: {e}")
                try:
                    os.remove(json_file)
                except OSError:
                    pass
                data = _fresh_data()
                fresh_discovery_seeded = _seed_fresh_prompt_discovery(data)
                with open(json_file, "w", encoding="utf-8") as f:
                    json.dump(data, f, indent=2)
                if im_feeling_lucky:
                    lucky_bootstrap = True
                else:
                    if latent_output_connected:
                        fallback_latent, fallback_latent_status = self._refine_latent(
                            latent,
                            refinement_key,
                            mode,
                            rating,
                            reward,
                            {},
                            rating_profile,
                        )
                    else:
                        fallback_latent, fallback_latent_status = self._latent_refinement_disabled(latent)
                    return (positive_conditioning, "Session file was corrupt - Reset and started fresh", "", f"Session reset due to corrupt file\n{fallback_latent_status}", fallback_loss_graph, fallback_sigmas, fallback_latent)

        if lucky_bootstrap:
            rating_profile = dict(RATING_PROFILES["Initial discovery"], label="Initial discovery")
            rating_label = rating_profile["label"]
            rating = int(rating_profile.get("legacy_score", 6))
            reward = float(rating_profile.get("reward", 0.0))

        global_adaptive = data["global_adaptive"]
        previous_prompt_key_for_rating = data.get("last_prompt_key")
        # Migrate sessions created before the multi-level concept system was added
        global_adaptive.setdefault("concept_clusters", {})
        global_adaptive.setdefault("concept_groups", {})
        global_adaptive.setdefault("word_importance", {})
        global_adaptive.setdefault("feedbacked_concepts", [])
        global_adaptive.setdefault("feedback_memory", {"recent_questions": [], "rating_change_events": []})
        global_adaptive.setdefault("loss_history", [])
        global_adaptive.setdefault("lora_weight_memory", {})
        self._ensure_void_token_bank(global_adaptive)
        self._ensure_void_pair_bank(global_adaptive)
        self._ensure_lucky_context_memory(global_adaptive)
        self._ensure_lucky_phrase_placements(global_adaptive)
        global_adaptive.setdefault("mode", mode)
        self._ensure_sigma_state_defaults(global_adaptive)
        for cid in list(global_adaptive["concept_clusters"].keys()):
            global_adaptive["concept_clusters"][cid] = self._ensure_concept_cluster_defaults(
                global_adaptive["concept_clusters"][cid]
            )

        prompt_histories = data.get("prompt_histories", {})
        tokenizer = self._get_tokenizer(mode)

        if prompt_key not in prompt_histories and not im_feeling_lucky:
            matched_prompt_key, prompt_variant_match = self._find_prompt_variant_history(
                exact_prompt_key,
                current_prompt_words,
                raw_positive,
                prompt_histories,
            )
            if matched_prompt_key:
                prompt_key = matched_prompt_key

        # ====================== FEEDBACK STATE MACHINE (CONCEPT-LEVEL) ======================
        # Operates on full concept phrases rather than individual words.
        # Rating a concept updates all its tracked words proportionally and sends a
        # dampened neighbour signal to adjacent phrases. See _apply_concept_feedback
        # for the full multi-level propagation model.
        feedback_question_output = ""
        pending = data.get("pending_feedback")
        if not feedback_enabled:
            if pending is not None:
                data["pending_feedback"] = None
                with open(json_file, "w", encoding="utf-8") as f:
                    json.dump(data, f, indent=2)
            feedback_question_output = "Feedback disabled. Queue cleared."
        else:
            if pending is not None:
                # Discard stale or incompatible pending feedback before we even
                # consider applying it to the current prompt.
                if pending.get("type") != "concept" or pending.get("prompt_key") != prompt_key:
                    data["pending_feedback"] = None
                    pending = None
                    with open(json_file, "w", encoding="utf-8") as f:
                        json.dump(data, f, indent=2)
            else:
                feedback_question_output = "Feedback enabled. A concept question will appear after this generation."

        # ====================== PROMPT HISTORY SETUP ======================
        if prompt_key in prompt_histories:
            active = prompt_histories[prompt_key]
            is_new_prompt = False
        else:
            is_new_prompt = True
            active = {
                "canonical_prompt": positive_prompt,
                "prompt_concept_words": current_prompt_words,
                "source_prompt_key": exact_prompt_key,
                "source_conditioning_embeds": tensor_to_serializable(raw_positive),
                "reference_embeds": tensor_to_serializable(raw_positive),
                "liked_reference_embeds": None,
                "liked_reference_count": 0,
                "history": [],
                "last_rating": rating,
                "last_rating_label": rating_label
            }
            prompt_histories[prompt_key] = active
        self._remember_prompt_variant(
            active,
            exact_prompt_key,
            positive_prompt,
            current_prompt_words,
            prompt_variant_match,
        )

        # Safe source-conditioning loading. This remains the comparison anchor
        # even after liked results become the active refinement reference.
        source_data = active.get("source_conditioning_embeds") or active.get("reference_embeds")
        try:
            source_reference = serializable_to_tensor(source_data)
        except Exception as e:
            print(f"[FunPackVideoRefiner] Failed to load source conditioning: {e}. Resetting for this prompt.")
            source_reference = raw_positive.clone()
            active["source_conditioning_embeds"] = tensor_to_serializable(source_reference)
            active["reference_embeds"] = tensor_to_serializable(source_reference)
            active["liked_reference_embeds"] = None
            active["liked_reference_count"] = 0
            active["history"] = []
            is_new_prompt = True

        device = source_reference.device
        cur_positive = raw_positive.to(device) if raw_positive.device != device else raw_positive
        source_changed = False
        source_change_reason = ""
        source_retarget_status = ""

        # Shape mismatch guard
        if source_reference.shape != cur_positive.shape:
            print(f"[FunPackVideoRefiner] Source conditioning shape {source_reference.shape} != current {cur_positive.shape}. Resetting source reference.")
            source_reference = cur_positive.clone()
            active["source_conditioning_embeds"] = tensor_to_serializable(source_reference)
            active["reference_embeds"] = tensor_to_serializable(source_reference)
            active["liked_reference_embeds"] = None
            active["liked_reference_count"] = 0
            active["history"] = []
            is_new_prompt = True
            source_changed = True
            source_change_reason = "shape"

        seq_len = self._get_conditioning_seq_len(cur_positive)
        active_token_mask = self._get_conditioning_token_mask(cur_positive) if mode == "wan" else None

        source_similarity = self._conditioning_similarity(source_reference, cur_positive, active_token_mask)
        source_prompt_key = active.get("source_prompt_key")
        prompt_source_changed = bool(source_prompt_key and source_prompt_key != exact_prompt_key)
        if exact_prompt_key != prompt_key:
            prompt_source_changed = True
        conditioning_source_changed = (
            not source_changed and not is_new_prompt and source_similarity < 0.985
        )
        if im_feeling_lucky:
            prompt_source_changed = False
            conditioning_source_changed = False
        if prompt_source_changed or conditioning_source_changed:
            can_retarget_variant = (
                prompt_variant_match is not None and
                prompt_source_changed and
                not source_changed and
                list(source_reference.shape) == list(cur_positive.shape)
            )
            if can_retarget_variant:
                migrated = self._retarget_prompt_history_to_source(active, source_reference, cur_positive)
                source_reference = cur_positive.clone()
                active["source_conditioning_embeds"] = tensor_to_serializable(source_reference)
                active["source_prompt_key"] = exact_prompt_key
                is_new_prompt = False
                source_similarity = 1.0
                source_retarget_status = f" | prompt variant retargeted ({migrated} anchors)"
            else:
                source_reference = cur_positive.clone()
                active["source_conditioning_embeds"] = tensor_to_serializable(source_reference)
                active["reference_embeds"] = tensor_to_serializable(source_reference)
                active["liked_reference_embeds"] = None
                active["liked_reference_count"] = 0
                active["history"] = []
                is_new_prompt = True
                source_changed = True
                source_change_reason = "prompt" if prompt_source_changed else "conditioning"
                source_similarity = 1.0

        active["source_prompt_key"] = exact_prompt_key

        # ====================== WORD GROUPING (Level 2) ======================
        word_groups = self._build_word_groups(
            analysis_prompt,
            tokenizer,
            seq_len,
            token_mask=active_token_mask
        )

        # ====================== CONCEPT CLUSTER + GROUP SETUP (Levels 3 & 4) ======================
        concept_clusters = global_adaptive["concept_clusters"]
        if analysis_prompt:
            word_to_concept, ordered_concept_ids, current_concept_labels = self._build_word_concept_map(
                analysis_prompt, concept_clusters
            )
        else:
            word_to_concept, ordered_concept_ids, current_concept_labels = {}, [], {}

        if feedback_enabled and not ordered_concept_ids and analysis_prompt:
            fallback_cid = self._build_prompt_fallback_concept(analysis_prompt, concept_clusters)
            if fallback_cid:
                ordered_concept_ids = [fallback_cid]
                current_concept_labels[fallback_cid] = concept_clusters[fallback_cid].get("label", analysis_prompt[:64])
                for w in concept_clusters[fallback_cid].get("anchor_words", []):
                    word_to_concept.setdefault(w, fallback_cid)

        if feedback_enabled and pending is not None and pending.get("concept_id") not in set(ordered_concept_ids):
            data["pending_feedback"] = None
            pending = None
            with open(json_file, "w", encoding="utf-8") as f:
                json.dump(data, f, indent=2)

        # Build or update concept groups from the current ordered concept list
        concept_groups = self._build_concept_groups(
            ordered_concept_ids, concept_clusters,
            global_adaptive["concept_groups"], current_concept_labels, window=3
        )
        global_adaptive["concept_groups"] = concept_groups

        # Track which groups are active in this iteration
        active_group_ids = {
            gid for gid, g in concept_groups.items()
            if any(cid in ordered_concept_ids for cid in g.get("concept_ids", []))
        }

        # ====================== SCHEDULER SETUP ======================
        global_adaptive["scheduler_mode"] = scheduler_mode
        history = active.get("history", [])
        iter_num = len(history) + 1
        total_iters = sum(len(p.get("history", [])) for p in prompt_histories.values())
        feedback_memory = global_adaptive.setdefault("feedback_memory", {"recent_questions": [], "rating_change_events": []})
        rating_key = rating_profile.get("key", "")
        rated_positive = None
        if history and not source_changed:
            rated_positive = self._history_modified_conditioning(history[-1], source_reference, device)

        bank_rated_prompt = analysis_prompt
        bank_rated_lucky_prompt = ""
        bank_rated_word_groups = word_groups
        bank_rated_positive = rated_positive
        bank_rated_token_mask = active_token_mask
        if history and isinstance(history[-1].get("lucky"), dict):
            bank_rated_lucky_prompt = str(history[-1].get("lucky", {}).get("prompt", "")).strip()

        if previous_prompt_key_for_rating in prompt_histories:
            previous_active = prompt_histories.get(previous_prompt_key_for_rating)
            previous_history = previous_active.get("history", []) if isinstance(previous_active, dict) else []
            if previous_history:
                try:
                    previous_source = serializable_to_tensor(
                        previous_active.get("source_conditioning_embeds") or previous_active.get("reference_embeds")
                    ).to(device)
                except Exception:
                    previous_source = None
                previous_positive = self._history_modified_conditioning_any_shape(previous_history[-1], device, dtype=cur_positive.dtype)
                if previous_positive is not None:
                    previous_prompt = (
                        previous_history[-1].get("analysis_prompt") or
                        previous_history[-1].get("prompt_full") or
                        previous_history[-1].get("prompt") or
                        previous_active.get("canonical_prompt", "")
                    )
                    previous_seq_len = self._get_conditioning_seq_len(previous_positive)
                    previous_token_mask = self._get_conditioning_token_mask(previous_positive) if mode == "wan" else None
                    previous_word_groups = self._build_word_groups(
                        previous_prompt,
                        tokenizer,
                        previous_seq_len,
                        token_mask=previous_token_mask,
                    )
                    previous_word_groups = previous_word_groups + self._lucky_groups_from_history(
                        previous_history[-1],
                        previous_seq_len,
                    )
                    if previous_word_groups:
                        bank_rated_prompt = previous_prompt
                        if isinstance(previous_history[-1].get("lucky"), dict):
                            bank_rated_lucky_prompt = str(previous_history[-1].get("lucky", {}).get("prompt", "")).strip()
                        bank_rated_word_groups = previous_word_groups
                        bank_rated_positive = previous_positive
                        bank_rated_token_mask = previous_token_mask

        if bank_rated_positive is not None:
            self._update_void_token_bank(
                global_adaptive,
                bank_rated_word_groups,
                bank_rated_positive,
                rating_profile,
                iter_num,
                token_mask=bank_rated_token_mask,
            )
            self._update_lucky_phrase_placements(global_adaptive, bank_rated_prompt, rating_key, iter_num)
            if bank_rated_lucky_prompt:
                self._update_lucky_phrase_placements(global_adaptive, bank_rated_lucky_prompt, rating_key, iter_num)

        should_seed_current_prompt_after_lucky = (
            bool(analysis_prompt) and
            not (lucky_bootstrap and fresh_discovery_seeded) and
            (
                analysis_prompt != bank_rated_prompt or
                bank_rated_positive is None
            )
        )

        if feedback_enabled and pending is not None:
            self._apply_concept_feedback(
                pending["concept_id"],
                feedback_rating,
                pending.get("question_type", "presence"),
                global_adaptive["concept_clusters"],
                pending.get("neighbor_ids", []),
                global_adaptive["word_importance"],
                global_adaptive["concept_groups"],
                pending.get("iteration", 0)
            )
            global_adaptive["last_feedback_concept"] = pending.get("concept_label", "")
            data["pending_feedback"] = None
            pending = None
            with open(json_file, "w", encoding="utf-8") as f:
                json.dump(data, f, indent=2)

        liked_reference = None
        liked_reference_count = int(active.get("liked_reference_count", 0) or 0)
        liked_data = active.get("liked_reference_embeds")
        if liked_reference_count > 0 and liked_data is not None:
            try:
                candidate = serializable_to_tensor(liked_data).to(device)
                if list(candidate.shape) == list(source_reference.shape):
                    liked_reference = candidate
                else:
                    liked_reference_count = 0
                    active["liked_reference_embeds"] = None
                    active["liked_reference_count"] = 0
            except Exception:
                liked_reference_count = 0
                active["liked_reference_embeds"] = None
                active["liked_reference_count"] = 0

        liked_reference_updated = False
        if rating_key == "like" and not source_changed:
            rated_reference = rated_positive
            if rated_reference is None:
                rated_reference = source_reference.clone()
            if liked_reference is None or liked_reference_count <= 0:
                liked_reference = rated_reference.clone()
                liked_reference_count = 1
            else:
                liked_reference = (
                    liked_reference * liked_reference_count + rated_reference
                ) / float(liked_reference_count + 1)
                liked_reference_count += 1
            active["liked_reference_embeds"] = tensor_to_serializable(liked_reference)
            active["liked_reference_count"] = liked_reference_count
            active["liked_reference_last_iteration"] = iter_num
            liked_reference_updated = True

        if liked_reference is not None and liked_reference_count > 0:
            reference = liked_reference.clone()
            reference_mode = "liked average"
        else:
            reference = source_reference.clone()
            reference_mode = "source"

        rollback_reference_found = False
        rollback_rating_label = ""
        rollback_requested = bool(rating_profile.get("rollback_on_failure")) or rating_key == "dislike"
        if rollback_requested and not source_changed and len(history) > 1:
            current_level = int(rating_profile.get("level", 1))
            for entry in reversed(history[:-1]):
                entry_profile = normalize_refiner_rating(entry.get("rating_label", entry.get("rating", 0)))
                if int(entry_profile.get("level", 0)) <= current_level:
                    continue
                mod_data = entry.get("modified_embeds")
                if mod_data is None:
                    continue
                try:
                    candidate = serializable_to_tensor(mod_data).to(device)
                    if list(candidate.shape) == list(source_reference.shape):
                        reference = candidate.clone()
                        reference_mode = "rollback"
                        rollback_reference_found = True
                        rollback_rating_label = entry_profile.get("label", str(entry.get("rating", "")))
                        break
                except Exception:
                    continue
        if (
            rollback_requested and
            not source_changed and
            not rollback_reference_found and
            liked_reference is not None and
            liked_reference_count > 0
        ):
            reference = liked_reference.clone()
            reference_mode = "rollback"
            rollback_reference_found = True
            rollback_rating_label = "liked average"

        active["reference_embeds"] = tensor_to_serializable(reference)
        similarity = self._conditioning_similarity(reference, cur_positive, active_token_mask)

        last_rating = active.get("last_rating", 5)
        last_rating_profile = normalize_refiner_rating(active.get("last_rating_label", last_rating))
        if "last_rating_label" not in active:
            last_rating_profile = normalize_refiner_rating(last_rating)
        last_reward = float(last_rating_profile.get("reward", (last_rating - 5.5) / 4.5))
        rating_shift = abs(rating - last_rating)
        feedback_memory["rating_change_events"] = list(feedback_memory.get("rating_change_events", []))[-15:]
        feedback_memory["rating_change_events"].append({
            "iteration": iter_num,
            "rating": rating,
            "rating_label": rating_label,
            "shift": rating_shift,
            "prompt_key": prompt_key,
        })

        word_importance = global_adaptive["word_importance"]
        missing_axis_status = self._apply_missing_axis_prompt_pressure(
            word_groups,
            word_to_concept,
            rating_profile,
            concept_clusters,
            word_importance,
            iter_num,
        )
        prompt_delta_status = self._apply_prompt_delta_attribution(
            history,
            current_prompt_words,
            rating_profile,
            concept_clusters,
            word_importance,
            iter_num,
        )
        prompt_emphasis = float(rating_profile.get("prompt_emphasis", reward))

        lr_scale, confidence, exploration_mult, word_lr_mult = self._get_scheduler_factors(
            scheduler_mode, rating, reward, similarity, iter_num, total_iters,
            word_importance, word_groups, global_adaptive, device
        )

        expl = global_adaptive["exploration_base"] * exploration_mult

        # ====================== WORD IMPORTANCE UPDATE (concept-aware, Levels 2-3) ======================
        for start, end, full_word, word_token_list in word_groups:
            if not self._is_valuable_token(full_word):
                continue
            wkey = full_word.lower()

            base_lr = 0.22 / (1 + 0.07 * (len(history) ** 0.5))
            group_delta = prompt_emphasis * base_lr * lr_scale * confidence
            if wkey in word_lr_mult:
                group_delta *= word_lr_mult[wkey]

            # Level 3: update per-concept importance (primary, context-isolated signal)
            cid = word_to_concept.get(wkey)
            if cid and cid in concept_clusters:
                concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
                local_imp = concept_clusters[cid]["word_importance"]
                profile_gain = (
                    concept_clusters[cid]["priority_weight"] *
                    (0.82 + 0.18 * concept_clusters[cid]["user_affinity"]) *
                    (0.80 + 0.20 * concept_clusters[cid]["presence_target"])
                )
                group_delta *= max(0.55, min(1.85, profile_gain))
                if wkey not in local_imp:
                    local_imp[wkey] = 1.0
                local_imp[wkey] = max(0.35, min(2.8, local_imp[wkey] + group_delta))
                concept_clusters[cid]["usage_count"] = concept_clusters[cid].get("usage_count", 0) + 1
                concept_clusters[cid]["last_seen_iter"] = iter_num

            # Level 2: dampened global fallback so the scheduler/prodigy system
            # retains a signal without cross-context word-meaning contamination
            if wkey not in word_importance:
                word_importance[wkey] = 1.0
            word_importance[wkey] = max(0.35, min(2.8, word_importance[wkey] + group_delta * 0.4))

        # Update active group usage counters (Level 4)
        for gid in active_group_ids:
            if gid in concept_groups:
                concept_groups[gid]["last_seen_iter"] = iter_num
                concept_groups[gid]["usage_count"] = concept_groups[gid].get("usage_count", 0) + 1

        # ====================== CORE REFINEMENT ======================
        momentum = global_adaptive.get("momentum")
        if momentum is None or not isinstance(momentum, dict):
            momentum = torch.zeros_like(reference)
        else:
            momentum = serializable_to_tensor(momentum).to(device)
            if list(momentum.shape) != list(reference.shape):
                momentum = torch.zeros_like(reference)

        avg_reward_ema = global_adaptive["avg_reward_ema"]
        avg_reward_ema = 0.85 * avg_reward_ema + 0.15 * reward
        global_adaptive["avg_reward_ema"] = avg_reward_ema

        good_ratio = global_adaptive["good_ratio"]
        if rating_key == "like":
            good_ratio = 0.9 * good_ratio + 0.1 * 1.0
            expl = max(0.015, expl * 0.96)
        elif set(rating_profile.get("missing_axes", [])) == {"details"}:
            good_ratio = 0.9 * good_ratio + 0.1 * 0.55
            expl = min(0.12, expl * 1.02)
        else:
            good_ratio = 0.9 * good_ratio + 0.1 * 0.0
            expl = min(0.12, expl * 1.08)
        global_adaptive["good_ratio"] = good_ratio

        sim_threshold = global_adaptive["dynamic_sim_threshold"]
        is_close = (not is_new_prompt) and (similarity >= sim_threshold)

        if (is_close and history) or (rollback_requested and rollback_reference_found and history):
            last_entry = history[-1]
            mod_data = last_entry.get("modified_embeds")
            if mod_data is not None:
                try:
                    prev_modified = serializable_to_tensor(mod_data).to(device)
                    if list(prev_modified.shape) != list(reference.shape):
                        prev_modified = torch.zeros_like(reference)
                except Exception:
                    prev_modified = torch.zeros_like(reference)
            else:
                prev_modified = torch.zeros_like(reference)
            prev_delta = prev_modified - reference
            noise_scale = expl * (1.0 - avg_reward_ema * 0.7)
            if reference_mode == "liked average" and rating_key == "like":
                noise_scale = 0.0
            noise = torch.randn_like(reference) * noise_scale
            if rollback_requested and rollback_reference_found:
                new_delta = (-prev_delta * 0.9) + (momentum * 0.25)
            else:
                multiplier = max(0.05, 1.0 + reward * 1.45)
                if rating == last_rating and rating_key == "like":
                    multiplier += 0.35
                new_delta = (prev_delta * multiplier) + noise + (momentum * 0.6)
        else:
            good_deltas = []
            for entry in history:
                entry_profile = normalize_refiner_rating(entry.get("rating_label", entry.get("rating", 0)))
                if entry_profile.get("key") not in {"like", "missing_details"}:
                    continue
                mod_data = entry.get("modified_embeds")
                if mod_data is None:
                    continue
                try:
                    mod = serializable_to_tensor(mod_data).to(device)
                    if list(mod.shape) == list(reference.shape):
                        good_deltas.append(mod - reference)
                except Exception:
                    continue
            if good_deltas:
                new_delta = torch.stack(good_deltas).mean(dim=0) * (0.7 + reward * 0.4)
            elif reference_mode == "liked average" and rating_key == "like":
                new_delta = torch.zeros_like(reference)
            else:
                new_delta = torch.randn_like(reference) * expl * 0.45

        # Apply concept-aware importance to delta (Levels 2-3)
        # Each token position is scaled by the concept-local importance of the word
        # occupying that position, with global importance as fallback.
        if cur_positive.dim() > 1:
            seq_len = cur_positive.shape[1] if cur_positive.dim() == 3 else cur_positive.shape[0]
            importance_tensor = torch.ones(seq_len, device=device)
            profile_tensor = torch.ones(seq_len, device=device)
            if active_token_mask is not None:
                importance_tensor = importance_tensor * active_token_mask.to(device=device, dtype=importance_tensor.dtype)
                profile_tensor = profile_tensor * active_token_mask.to(device=device, dtype=profile_tensor.dtype)
            for start, end, full_word, _ in word_groups:
                wkey = full_word.lower()
                cid = word_to_concept.get(wkey)
                if cid and cid in concept_clusters:
                    concept_clusters[cid] = self._ensure_concept_cluster_defaults(concept_clusters[cid])
                    imp = concept_clusters[cid]["word_importance"].get(
                        wkey, word_importance.get(wkey, 1.0)
                    )
                    profile_mult = (
                        concept_clusters[cid]["priority_weight"] *
                        (0.84 + 0.16 * concept_clusters[cid]["user_affinity"]) *
                        (0.86 + 0.14 * concept_clusters[cid]["presence_target"])
                    )
                    profile_mult *= (
                        1.0 -
                        0.10 * (concept_clusters[cid]["stability_weight"] - 1.0) -
                        0.06 * (concept_clusters[cid]["semantic_fidelity"] - 1.0)
                    )
                    profile_mult *= (
                        1.0 -
                        0.10 * max(0.0, concept_clusters[cid]["overrep_sensitivity"] - 1.0)
                    )
                    profile_mult = max(0.55, min(1.80, profile_mult))
                else:
                    imp = word_importance.get(wkey, 1.0)
                    profile_mult = 1.0
                importance_tensor[start:end] = imp
                profile_tensor[start:end] = profile_mult
            new_delta = new_delta * importance_tensor.unsqueeze(-1)
            new_delta = new_delta * profile_tensor.unsqueeze(-1)

        token_mask_nd = self._mask_to_embedding_dims(active_token_mask, reference)
        if token_mask_nd is not None:
            new_delta = new_delta * token_mask_nd

        # Final safety guard
        if new_delta.shape != reference.shape:
            new_delta = torch.zeros_like(reference)

        new_positive = reference + new_delta
        new_positive = torch.clamp(new_positive, min=-60.0, max=60.0)
        norm_factor = reference.norm(dim=-1, keepdim=True) + 1e-8
        new_positive = new_positive / (new_positive.norm(dim=-1, keepdim=True) + 1e-8) * norm_factor
        lucky_canvas, lucky_canvas_label = (None, "")
        lucky_prompt_text = ""
        lucky_prompt_tokens = []
        lucky_encoded_meta = None
        lucky_prompt_sequences = []
        lucky_phrase_sequences = []
        if im_feeling_lucky:
            lucky_prompt_sequences = self._lucky_prompt_sequences(prompt_histories)
            lucky_phrase_sequences = self._lucky_prompt_phrase_sequences(prompt_histories)
            if clip is not None:
                lucky_prompt_text, lucky_prompt_info = self._compose_lucky_prompt_text(
                    global_adaptive,
                    word_groups,
                    int(cur_positive.shape[-1]),
                    prompt_sequences=lucky_prompt_sequences,
                    phrase_sequences=lucky_phrase_sequences,
                    target_count=self._get_conditioning_seq_len(cur_positive),
                )
                lucky_prompt_tokens = list(lucky_prompt_info.get("tokens", [])) if isinstance(lucky_prompt_info, dict) else []
                encoded_canvas, encoded_meta, encoded_status = self._encode_lucky_prompt_conditioning(
                    clip,
                    lucky_prompt_text,
                    cur_positive,
                    device,
                    cur_positive.dtype,
                )
                if encoded_canvas is not None:
                    encoded_delta = self._resize_conditioning_sequence_like(new_delta, encoded_canvas)
                    if encoded_delta is not None:
                        refined_encoded_canvas = encoded_canvas + encoded_delta
                        refined_encoded_canvas = torch.clamp(refined_encoded_canvas, min=-60.0, max=60.0)
                        encoded_norm = encoded_canvas.norm(dim=-1, keepdim=True).clamp_min(1e-8)
                        refined_encoded_canvas = (
                            refined_encoded_canvas /
                            refined_encoded_canvas.norm(dim=-1, keepdim=True).clamp_min(1e-8) *
                            encoded_norm
                        )
                    else:
                        refined_encoded_canvas = encoded_canvas
                    lucky_canvas = refined_encoded_canvas
                    lucky_canvas_label = encoded_status
                    lucky_encoded_meta = encoded_meta
                elif encoded_status:
                    print(f"[FunPackVideoRefiner] Lucky CLIP/Gemma canvas unavailable: {encoded_status}")

            if lucky_canvas is None:
                lucky_canvas, lucky_canvas_label = self._select_lucky_memory_canvas(
                    prompt_histories,
                    cur_positive,
                    device,
                    cur_positive.dtype,
                )
        new_positive, void_status, lucky_metadata = self._apply_into_the_void(
            new_positive,
            reference,
            global_adaptive,
            word_groups,
            word_importance,
            into_the_void=into_the_void,
            im_feeling_lucky=im_feeling_lucky,
            token_mask=active_token_mask,
            prompt_sequences=lucky_prompt_sequences,
            lucky_canvas=lucky_canvas,
            lucky_canvas_label=lucky_canvas_label,
            lucky_prompt_text=lucky_prompt_text,
            lucky_prompt_tokens=lucky_prompt_tokens if lucky_encoded_meta is not None else [],
        )
        if should_seed_current_prompt_after_lucky:
            self._seed_void_token_bank(
                global_adaptive,
                word_groups,
                cur_positive,
                iter_num,
                token_mask=active_token_mask,
            )
            self._update_lucky_phrase_placements(global_adaptive, analysis_prompt, "discover", iter_num)

        # Update momentum
        momentum = 0.75 * momentum + 0.25 * (new_delta * reward)
        global_adaptive["momentum"] = tensor_to_serializable(momentum)

        if avg_reward_ema > 0.3:
            sim_threshold = max(0.75, sim_threshold - 0.002)
        global_adaptive["dynamic_sim_threshold"] = sim_threshold
        global_adaptive["exploration_base"] = expl

        # Prune stale concept clusters and groups to prevent unbounded JSON growth
        if len(concept_clusters) > 64:
            concept_clusters = {
                cid: c for cid, c in concept_clusters.items()
                if iter_num - c.get("last_seen_iter", 0) < 500
            }
            global_adaptive["concept_clusters"] = concept_clusters

        if len(concept_groups) > 32:
            concept_groups = {
                gid: g for gid, g in concept_groups.items()
                if iter_num - g.get("last_seen_iter", 0) < 500
            }
            global_adaptive["concept_groups"] = concept_groups

        # ====================== HISTORY ENTRY ======================
        history_entry = {
            "iteration": iter_num,
            "rating": rating,
            "rating_label": rating_label,
            "rating_range": rating_profile.get("legacy_range", ""),
            "reward": round(reward, 3),
            "modified_embeds": tensor_to_serializable(new_positive),
            "similarity": round(similarity, 4),
            "source_similarity": round(source_similarity, 4),
            "reference_mode": reference_mode,
            "liked_reference_count": int(liked_reference_count),
            "liked_reference_updated": bool(liked_reference_updated),
            "rollback_from_rating": rollback_rating_label,
            "prompt": positive_prompt[:180],
            "prompt_full": positive_prompt,
            "analysis_prompt": analysis_prompt,
            "prompt_words": current_prompt_words,
            "exact_prompt_key": exact_prompt_key,
            "into_the_void": bool(into_the_void),
            "im_feeling_lucky": bool(im_feeling_lucky),
            "void_status": void_status,
            "lucky": lucky_metadata,
        }
        history.append(history_entry)

        if not unlimited_history and len(history) > 200:
            sorted_hist = sorted(history, key=lambda x: x.get("rating", 0), reverse=True)
            top = sorted_hist[:40]
            recent = history[-120:]
            seen_iters = {e["iteration"] for e in top}
            history = top + [e for e in recent if e["iteration"] not in seen_iters]

        active["history"] = history
        active["last_rating"] = rating
        active["last_rating_label"] = rating_label
        active["last_positive_prompt"] = positive_prompt
        active["last_exact_prompt_words"] = current_prompt_words
        data["last_prompt_key"] = prompt_key
        data["prompt_histories"] = prompt_histories
        data["global_adaptive"] = global_adaptive

        refined_sigmas, sigma_status = self._refine_sigma_schedule(
            sigmas,
            rating,
            global_adaptive,
            sigma_strength,
            seed,
            rating_profile,
        )
        if latent_output_connected:
            refined_latent, latent_status = self._refine_latent(
                latent,
                refinement_key,
                mode,
                rating,
                reward,
                global_adaptive,
                rating_profile,
            )
        else:
            refined_latent, latent_status = self._latent_refinement_disabled(latent)

        # ====================== INTELLIGENT CONCEPT FEEDBACK SELECTION ======================
        # Chooses one concept/question pair per run using category-aware scoring.
        # Different question types learn different aspects of user preference:
        # presence, priority, balance, fidelity, stability, and preference.
        if feedback_enabled and data.get("pending_feedback") is None and ordered_concept_ids:
            selected_question = self._select_feedback_question(
                ordered_concept_ids,
                concept_clusters,
                concept_groups,
                current_concept_labels,
                last_rating,
                rating,
                similarity,
                iter_num
            )

            if selected_question is None:
                selected_question = self._force_feedback_fallback(
                    ordered_concept_ids,
                    concept_clusters,
                    concept_groups,
                    current_concept_labels,
                    rating_shift,
                    similarity
                )

            if selected_question:
                data["pending_feedback"] = {
                    "type": "concept",
                    "concept_id": selected_question["concept_id"],
                    "concept_label": selected_question["concept_label"],
                    "question_type": selected_question["question_type"],
                    "neighbor_ids": selected_question["neighbor_ids"],
                    "group_id": selected_question["group_id"],
                    "prompt_key": prompt_key,
                    "iteration": iter_num
                }
                feedback_memory["recent_questions"] = list(feedback_memory.get("recent_questions", []))[-15:]
                feedback_memory["recent_questions"].append({
                    "iteration": iter_num,
                    "concept_id": selected_question["concept_id"],
                    "question_type": selected_question["question_type"],
                    "prompt_key": prompt_key,
                })
                global_adaptive["last_feedback_concept"] = selected_question["concept_label"]
                feedback_question_output = self._format_feedback_question(
                    selected_question["concept_label"],
                    selected_question["question_type"]
                )
            else:
                feedback_question_output = (
                    "There is enough information collected on current concepts. "
                    "Node will ask you again in case if rating changes significantly."
                )
        elif feedback_enabled and data.get("pending_feedback") is None:
            feedback_question_output = (
                "There is enough information collected on current concepts. "
                "Node will ask you again in case if rating changes significantly."
            )

        # ====================== TRAINING INFO ======================
        current_top = self._get_top_tokens(word_importance, tokenizer, 10)

        # Per-concept phrase summaries (Level 3)
        active_concept_parts = []
        for cid in ordered_concept_ids:
            if cid not in concept_clusters:
                continue
            c = concept_clusters[cid]
            top_local = self._get_top_tokens(c["word_importance"], tokenizer, 4)
            active_concept_parts.append(
                f"[{current_concept_labels.get(cid, c['label'])}/{c.get('category', 'general')}: "
                f"top={top_local} | p={c.get('presence_target', 1.0):.2f} "
                f"prio={c.get('priority_weight', 1.0):.2f} "
                f"stab={c.get('stability_weight', 1.0):.2f} "
                f"cat={c.get('category_source', 'auto')}]"
            )
        concept_line = "; ".join(active_concept_parts) if active_concept_parts else "none"

        # Concept group health summaries (Level 4)
        group_parts = []
        for gid in active_group_ids:
            g = concept_groups.get(gid)
            if g:
                ema = g.get("reward_ema", 0.0)
                health_icon = "✅" if ema > 0.3 else "⚠️" if ema > -0.3 else "🔄"
                group_parts.append(f"{health_icon} {g['label']} (ema={ema:+.2f})")
        group_line = "; ".join(group_parts) if group_parts else "none"

        # Dominant concept: the phrase the model currently weights most heavily
        dom_cid, dom_score, dom_label = self._get_dominant_concept(
            ordered_concept_ids, concept_clusters
        )
        dom_label = current_concept_labels.get(dom_cid, dom_label) if dom_cid else dom_label
        dominant_line = f"'{dom_label}' (avg_imp={dom_score:.2f})" if dom_cid else "undetermined"

        lora_suggestion_status = self._update_lora_weight_suggestions(
            lora_stack,
            active,
            global_adaptive,
            ordered_concept_ids,
            concept_clusters,
            current_concept_labels,
            rating,
            reward,
            rating_profile,
        )
        if im_feeling_lucky:
            learned_prompt_memories = sum(
                1 for key, item in prompt_histories.items()
                if key != "__lucky_memory__" and isinstance(item, dict) and item.get("history")
            )
            prompt_history_status = (
                "Prompt history: Lucky memory stream "
                f"({len(history)} Lucky updates, {learned_prompt_memories} learned prompt memories)."
            )
        elif prompt_variant_match:
            prompt_history_status = (
                "Prompt history: reused similar enhanced prompt "
                f"(overlap {prompt_variant_match.get('overlap', 0.0):.0%}, "
                f"coverage {prompt_variant_match.get('coverage', 0.0):.0%}, "
                f"conditioning {prompt_variant_match.get('conditioning_similarity', 0.0):.2f})."
            )
        elif exact_prompt_key != prompt_key:
            prompt_history_status = "Prompt history: reused compatible prompt variant."
        else:
            prompt_history_status = "Prompt history: exact prompt."

        normalized_reward = max(0.0, min(1.0, (avg_reward_ema + 1.0) / 2.0))
        stability_factor = max(0.0, 1.0 - similarity)
        raw_loss = (1.0 - normalized_reward) * 0.7 + stability_factor * 0.3
        learning_loss = max(0.02, raw_loss * (1.0 - min(0.95, good_ratio * 0.8)))
        reference_status = (
            f"Reference: {reference_mode} | liked anchors {liked_reference_count} | "
            f"source similarity {source_similarity:.4f}"
        )
        if liked_reference_updated:
            reference_status += " | liked average updated"
        if rollback_reference_found:
            reference_status += f" | rollback from {rollback_rating_label or 'higher rating'}"
        if source_changed:
            reference_status += f" | source refreshed ({source_change_reason or 'changed'})"
        if source_retarget_status:
            reference_status += source_retarget_status
        missing_axis_line = f"{missing_axis_status}\n" if missing_axis_status else ""
        prompt_delta_line = f"{prompt_delta_status}\n" if prompt_delta_status else ""

        training_info = (
            f"Mode: {mode.upper()} | Scheduler: {scheduler_mode.upper()} | Step: {global_adaptive['current_step']} | "
            f"Rating: {rating_label} ({rating_profile.get('legacy_range', rating)}) | "
            f"EMA Reward: {avg_reward_ema:+.3f} | Confidence: {confidence:.2f} | LR Scale: {lr_scale:.3f}\n"
            f"Exploration: {expl:.3f} | Similarity: {similarity:.4f} | Good Ratio: {good_ratio:.1%} | Prompt Emphasis: {prompt_emphasis:+.2f}\n"
            f"**Learning Loss: {learning_loss:.4f}** (lower is better)\n"
            f"{reference_status}\n"
            f"Dominant concept: {dominant_line}\n"
            f"{void_status}\n"
            f"Concept phrases ({len(concept_clusters)} total): {concept_line}\n"
            f"Concept groups ({len(concept_groups)} total): {group_line}\n"
            f"Global top words: {current_top}\n"
            f"{missing_axis_line}"
            f"{prompt_delta_line}"
            f"{prompt_history_status}\n"
            f"{lora_suggestion_status}\n"
            f"{sigma_status}\n"
            f"{latent_status}"
        )

        if scheduler_mode == "accurate":
            training_info += "\n[Accurate] Conservative • Prodigy + Cosine"
        elif scheduler_mode == "aggressive":
            training_info += "\n[Aggressive] Fast style locking"

        global_total_iterations = sum(len(p.get("history", [])) for p in prompt_histories.values())
        loss_history = list(global_adaptive.get("loss_history", []))[-511:]
        loss_history.append({
            "total_iteration": global_total_iterations,
            "learning_loss": round(float(learning_loss), 6),
            "rating": int(rating),
            "rating_label": rating_label,
            "similarity": round(float(similarity), 6),
            "scheduler_mode": scheduler_mode,
            "mode": mode,
        })
        global_adaptive["loss_history"] = loss_history

        loss_graph = render_refinement_loss_graph(
            refinement_key=refinement_key,
            scheduler_mode=scheduler_mode,
            mode=mode,
            total_iterations=global_total_iterations,
            latest_learning_loss=float(learning_loss),
            points=loss_history[-256:],
        )

        # ====================== STATUS ======================
        trend = "↑" if reward > last_reward else "↓" if reward < last_reward else "→"
        health = (
            "🚀 Strong convergence" if avg_reward_ema > 0.6 else
            "✅ Learning well" if avg_reward_ema > 0.3 else
            "⚠️ Still exploring" if avg_reward_ema > -0.2 else
            "🔄 Heavy correction"
        )
        pending_feedback = data.get("pending_feedback")
        if not feedback_enabled:
            feedback_state = "Feedback off"
        elif pending_feedback is not None:
            feedback_state = f"Feedback queued for '{pending_feedback.get('concept_label', 'concept')}'"
        else:
            feedback_state = "Feedback ready"

        suggestions_snapshot = active.get("lora_weight_suggestions", {})
        suspect_count = sum(1 for item in suggestions_snapshot.values() if item.get("suspect"))
        if isinstance(lora_stack, dict) and lora_stack.get("loras"):
            lora_state = f"LoRA active ({len(lora_stack.get('loras', []))} loaded"
            if suspect_count > 0:
                lora_state += f", {suspect_count} suspect"
            lora_state += ")"
        else:
            lora_state = "LoRA idle"

        sigma_state = "Sigma active" if isinstance(sigmas, torch.Tensor) and sigma_strength != "off" else (
            "Sigma connected (off)" if isinstance(sigmas, torch.Tensor) else "Sigma idle"
        )

        latent_state = (
            "Latent active" if latent_output_connected and refined_latent is not None else
            "Latent armed" if latent_output_connected else
            "Latent idle"
        )

        if im_feeling_lucky:
            learned_prompt_memories = sum(
                1 for key, item in prompt_histories.items()
                if key != "__lucky_memory__" and isinstance(item, dict) and item.get("history")
            )
            lucky_bank_size = len(global_adaptive.get("void_token_bank", {}))
            lucky_phrase_size = len(global_adaptive.get("lucky_phrase_placements", {}))
            session_line = (
                f"Session: Lucky memory {len(history)} update(s), {global_total_iterations} total update(s), "
                f"{learned_prompt_memories} learned prompt memory(s), "
                f"{lucky_bank_size} token(s), {lucky_phrase_size} phrase placement(s)"
            )
        else:
            session_line = (
                f"Session: {len(prompt_histories)} prompt(s), {global_total_iterations} total update(s), "
                f"{len(history)} history item(s) on this prompt"
            )

        status = (
            f"{health} | Mode {mode.upper()} | Rating {rating_label} ({rating_profile.get('legacy_range', rating)}) {trend} | Iter {iter_num}\n"
            f"{session_line}\n"
            f"Reference: {reference_mode} ({liked_reference_count} liked) | Focus: {dominant_line} | {feedback_state}\n"
            f"Systems: {lora_state} | {sigma_state} | {latent_state} | {void_status}"
        )

        # ====================== FINAL SAVE ======================
        with open(json_file, "w", encoding="utf-8") as f:
            json.dump(data, f, indent=2)

        # ====================== RETURN ======================
        output_positive_meta = lucky_encoded_meta if isinstance(lucky_encoded_meta, dict) else positive_meta
        modified_positive = [(new_positive, output_positive_meta)]

        return (modified_positive, status, feedback_question_output, training_info, loss_graph, refined_sigmas, refined_latent)


FunPackGemmaEmbeddingRefiner = FunPackVideoRefiner


V2_RATING_LABELS = [
    "-Just forget it-",
    "Perfect",
    "Missing details",
    "Missing action",
    "Missing quality",
    "Missing details + action",
    "Wrong details",
    "Wrong action",
    "Wrong details + action",
    "Wrong appearance",
    "Missing details + quality",
    "Missing action + quality",
    "Awful",
]

V2_RATING_PROFILES = {
    "-Just forget it-": {"key": "forget", "reward": 0.0, "level": 0, "missing_axes": [], "skip_learning": True},
    "Initial discovery": {"key": "discover", "reward": 0.0, "level": 4, "missing_axes": []},
    "Perfect": {"key": "like", "reward": 1.0, "level": 8, "missing_axes": []},
    "Missing details": {"key": "missing_details", "reward": 0.35, "level": 6, "missing_axes": ["details"]},
    "Missing action": {"key": "missing_action", "reward": 0.05, "level": 5, "missing_axes": ["action"]},
    "Missing quality": {"key": "missing_quality", "reward": -0.30, "level": 4, "missing_axes": ["quality"]},
    "Missing details + action": {"key": "missing_details_action", "reward": -0.10, "level": 3, "missing_axes": ["details", "action"]},
    "Wrong details": {"key": "wrong_details", "reward": 0.20, "level": 5, "missing_axes": ["details"], "wrong_axes": ["details"]},
    "Wrong action": {"key": "wrong_action", "reward": 0.10, "level": 4, "missing_axes": ["action"], "wrong_axes": ["action"]},
    "Wrong details + action": {"key": "wrong_details_action", "reward": 0.00, "level": 3, "missing_axes": ["details", "action"], "wrong_axes": ["details", "action"]},
    "Wrong appearance": {
        "key": "wrong_appearance",
        "reward": 0.0,
        "level": 4,
        "missing_axes": [],
        "wrong_categories": ["appearance", "subject", "environment"],
    },
    "Missing details + quality": {"key": "missing_details_quality", "reward": -0.40, "level": 2, "missing_axes": ["details", "quality"]},
    "Missing action + quality": {"key": "missing_action_quality", "reward": -0.55, "level": 1, "missing_axes": ["action", "quality"]},
    "Awful": {"key": "awful", "reward": -0.90, "level": 0, "missing_axes": ["details", "action", "quality"]},
}

V2_FEEDBACK_AXES = ("details", "action", "quality")
V2_ADVISOR_MODES = ["Off", "Diagnostics", "Repair prompt"]

V2_PROMPT_ADVISOR_SYSTEM_PROMPT = """You are FunPack Refiner V2 Prompt Advisor. You help a video prompt refiner decide whether a prompt needs a small, safe repair before it is encoded.

Rules:
- Preserve the user's current request and original intent.
- Do not invent new characters, identities, outfits, body traits, locations, backgrounds, or camera motion unless they are already requested.
- Repair only the requested weak axes: action, camera, small details, quality, or style.
- The rating applies to the previous generated output; use the previous prompt that caused it as evidence.
- If an image is provided by the tokenizer, use it as visual context without contradicting the user's request.
- Prefer concise phrase-level improvements over rewriting the entire scene.
- If no safe repair is needed, return the current prompt unchanged.
- Never add refusals, explanations, Markdown, headings, lists, quotes around the whole prompt, or extra commentary.
- Output exactly two lines:
DIAGNOSTIC: one short sentence explaining the useful repair or why no repair is safe.
REPAIRED_PROMPT: one continuous positive prompt paragraph."""

V2_RATING_ALIASES = {
    "I like it": "Perfect",
    "I don't like it": "Awful",
    "Missing concept": "Missing action",
    "Missing concepts": "Missing action",
    "Missing details + concept": "Missing details + action",
    "Missing details + concepts": "Missing details + action",
    "Missing concept + details": "Missing details + action",
    "Missing concepts + details": "Missing details + action",
    "Missing concept + quality": "Missing action + quality",
    "Missing concepts + quality": "Missing action + quality",
    "Missing quality + concept": "Missing action + quality",
    "Missing quality + concepts": "Missing action + quality",
    "Missing quality + action": "Missing action + quality",
    "Missing action + details": "Missing details + action",
    "Wrong action + details": "Wrong details + action",
    "Wrong intent": "Wrong details + action",
    "Wrong concept": "Wrong action",
    "Wrong concepts": "Wrong action",
    "Wrong movement": "Wrong action",
    "Wrong micro-movements": "Wrong details",
    "Wrong character": "Wrong appearance",
    "Wrong characters": "Wrong appearance",
    "Wrong outfit": "Wrong appearance",
    "Wrong clothing": "Wrong appearance",
    "Wrong clothes": "Wrong appearance",
    "Missing everything": "Awful",
}


def normalize_refiner_v2_rating(value):
    if isinstance(value, str):
        cleaned = V2_RATING_ALIASES.get(value.strip(), value.strip())
        if cleaned in V2_RATING_PROFILES:
            return dict(V2_RATING_PROFILES[cleaned], label=cleaned)
        try:
            value = int(float(cleaned))
        except (TypeError, ValueError):
            return dict(V2_RATING_PROFILES["Missing action"], label="Missing action")

    try:
        legacy_score = int(value)
    except (TypeError, ValueError):
        legacy_score = 6

    legacy_score = int(_clamp(legacy_score, 1, 10))
    if legacy_score >= 9:
        label = "Perfect"
    elif legacy_score >= 7:
        label = "Missing details"
    elif legacy_score >= 5:
        label = "Missing action"
    elif legacy_score >= 3:
        label = "Missing quality"
    else:
        label = "Awful"

    profile = dict(V2_RATING_PROFILES[label], label=label)
    profile["legacy_score"] = legacy_score
    return profile


class FunPackVideoRefinerV2(FunPackVideoRefiner):
    CATEGORY = "FunPack/Refinement"
    RETURN_TYPES = ("CONDITIONING", "STRING", "STRING", "IMAGE", "CONDITIONING")
    RETURN_NAMES = ("modified_positive", "status", "training_info", "loss_graph", "modified_negative")
    FUNCTION = "refine_v2"
    DESCRIPTION = "Prompt-owned Video Refiner V2. Encodes through the connected CLIP, learns from ratings, and writes LoRA suggestions without sigma/latent/feedback systems."

    V2_STATE_PREFIX = "refine_v2"
    ACTION_LORA_TYPES = {"action", "concept"}
    AUTO_INJECT_BLOCKED_CATEGORIES = {"appearance", "subject", "environment"}
    AUTO_INJECT_ALLOWED_CATEGORIES = {"action", "camera", "details", "quality", "style"}
    CATEGORY_DESCRIPTIONS = {
        "action": "physical actions, body movement, animation, motion, gestures, moving subjects",
        "camera": "camera motion, framing, zoom, pan, dolly, close-up, wide shot, focus behavior",
        "subject": "main person, object, creature, vehicle, character, or scene subject",
        "appearance": "character appearance, clothing, face, hair, pose, anatomy, visible traits",
        "environment": "place, location, background, weather, room, landscape, setting",
        "style": "visual style, art direction, lighting style, color grading, cinematic look",
        "quality": "sharpness, detail, realism, clean image quality, low noise, high resolution",
        "details": "small objects, props, reflections, texture, secondary prompt details",
    }
    CATEGORY_KEYWORDS = {
        "action": {
            "walk", "walking", "run", "running", "turn", "turning", "dance", "dancing", "jump",
            "jumping", "fly", "flying", "move", "moving", "motion", "gesture", "gesturing",
            "hold", "holding", "reach", "reaching", "look", "looking", "blink", "blinking",
            "smile", "smiling", "sit", "sitting", "stand", "standing", "kneel", "kneeling",
            "sway", "swaying", "spin", "spinning", "fall", "falling", "grab", "grabbing",
            "throw", "throwing", "catch", "catching", "climb", "climbing", "fight", "fighting",
            "talk", "talking", "speak", "speaking", "wave", "waving", "crawl", "crawling",
            "drive", "driving", "ride", "riding", "open", "opening", "close", "closing",
        },
        "camera": {
            "camera", "shot", "closeup", "close-up", "wide", "angle", "zoom", "pan", "panning",
            "dolly", "tracking", "handheld", "focus", "framing", "push", "push-in", "pull",
            "pull-back", "tilt", "orbit", "orbiting", "crane", "rack", "lens", "viewpoint",
            "perspective", "overhead", "low-angle", "high-angle", "close-up", "macro",
        },
        "subject": {
            "woman", "man", "girl", "boy", "person", "character", "robot", "creature", "dragon",
            "animal", "dog", "cat", "car", "vehicle", "object", "sculpture", "monster",
            "child", "baby", "adult", "crowd", "dancer", "runner", "warrior", "knight",
        },
        "appearance": {
            "hair", "eyes", "face", "skin", "dress", "jacket", "armor", "outfit", "clothing",
            "hands", "pose", "expression", "beard", "makeup", "body", "anatomy", "wearing",
            "wears", "dressed", "clothed", "costume", "shirt", "coat", "robe", "boots",
            "hat", "helmet", "gloves", "mask", "tattoo", "freckles", "scar", "silhouette",
            "pants", "skirt", "tights", "stockings", "socks", "shoes", "heels", "uniform",
            "flowing", "curly", "straight", "long", "short", "blonde", "brunette",
        },
        "environment": {
            "forest", "city", "street", "room", "kitchen", "beach", "mountain", "temple",
            "sunset", "night", "rain", "snow", "sky", "background", "studio", "tabletop",
            "setting", "environment", "landscape", "interior", "exterior", "alley", "road",
            "field", "desert", "ocean", "sea", "lake", "river", "waterfall", "clouds",
            "weather", "fog", "mist", "storm", "snowfall", "backdrop", "horizon",
        },
        "style": {
            "anime", "cinematic", "photorealistic", "painterly", "illustration", "stylized",
            "realistic", "film", "noir", "vintage", "dramatic", "soft", "lighting",
            "moody", "surreal", "documentary", "editorial", "watercolor", "sketch",
            "render", "rendered", "monochrome", "pastel", "neon", "gothic",
        },
        "quality": {
            "masterpiece", "best", "quality", "detailed", "sharp", "highres", "high-res",
            "ultra", "perfect", "clean", "realism", "realistic", "smooth", "crisp",
            "high-detail", "highly-detailed", "ultra-detailed", "noise-free", "polished",
            "refined", "clear", "high-resolution", "8k", "4k",
        },
        "details": {
            "reflection", "reflections", "texture", "textures", "shadow", "shadows", "smoke",
            "dust", "particles", "small", "tiny", "prop", "props", "fabric", "glass",
            "sparkles", "embers", "debris", "scratches", "cracks", "drops", "droplets",
            "pattern", "patterns", "grain", "details", "ornament", "ornaments",
        },
    }
    CATEGORY_PRIORITY = {
        "action": 0.018,
        "camera": 0.014,
        "subject": 0.008,
        "appearance": 0.012,
        "environment": 0.012,
        "style": 0.006,
        "quality": 0.004,
        "details": 0.002,
    }
    ACTION_SUFFIX_STEMS = {
        "walk", "run", "turn", "danc", "jump", "fly", "mov", "gestur", "hold", "reach",
        "look", "blink", "smil", "sit", "stand", "kneel", "sway", "spin", "fall",
        "grab", "throw", "catch", "climb", "fight", "talk", "speak", "wav", "crawl",
        "driv", "rid", "open", "clos",
    }
    APPEARANCE_CONTEXT_WORDS = {
        "hair", "eyes", "face", "skin", "dress", "jacket", "armor", "outfit", "clothing",
        "shirt", "coat", "robe", "boots", "hat", "helmet", "gloves", "mask", "body",
        "anatomy", "beard", "makeup", "costume", "pants", "skirt", "tights", "stockings",
        "socks", "shoes", "heels", "uniform",
    }
    ENVIRONMENT_CONTEXT_WORDS = {
        "background", "setting", "environment", "landscape", "room", "street", "forest",
        "city", "beach", "mountain", "temple", "sky", "studio", "interior", "exterior",
        "backdrop", "horizon", "weather",
    }
    QUALITY_CONTEXT_WORDS = {
        "quality", "detail", "details", "detailed", "sharp", "clean", "crisp", "realism",
        "resolution", "highres", "high-resolution", "smooth", "polished",
    }

    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "positive_prompt": ("STRING", {"multiline": True, "default": "", "placeholder": "Positive prompt"}),
                "rating": (V2_RATING_LABELS, {"default": "Missing action", "label": "Rating"}),
                "mode": (["Refine", "Learning"], {
                    "default": "Refine",
                    "label": "Mode",
                    "tooltip": "Refine applies learned prompt and conditioning changes. Learning records observations and ratings while passing prompts and conditioning through unchanged.",
                }),
                "advisor_mode": (V2_ADVISOR_MODES, {
                    "default": "Off",
                    "label": "Advisor",
                    "tooltip": "Optional CLIP text-generation advisor. Diagnostics only reports advice; Repair prompt can apply a validated generated prompt repair before encoding.",
                }),
            },
            "optional": {
                "clip": ("CLIP", {"tooltip": "Optional text encoder. When connected, V2 encodes the prompt itself."}),
                "advisor_clip": ("CLIP", {
                    "tooltip": "Optional separate CLIP/Gemma text generator for Advisor. If disconnected, Advisor falls back to the main CLIP.",
                }),
                "positive_conditioning": ("CONDITIONING", {
                    "tooltip": "Optional pre-encoded Gemma3/LTX2 conditioning. Used only when CLIP is not connected.",
                }),
                "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "label": "Exploration Seed"}),
                "reset_session": ("BOOLEAN", {"default": False, "label": "Reset V2 Session"}),
                "lora_stack": ("FUNPACK_LORA_STACK", {"tooltip": "Optional stack from FunPack LoRA Loader. V2 writes prompt-specific suggested weights."}),
                "clip_vision_output": ("CLIP_VISION_OUTPUT", {
                    "tooltip": "Optional CLIP Vision output describing the source image. Stored as advisory context; it is not blended into positive conditioning.",
                }),
                "source_image": ("IMAGE", {
                    "tooltip": "Optional original/source image or frame batch. V2 stores size, aspect ratio, and a simple fingerprint to notice changed inputs.",
                }),
                "negative_prompt": ("STRING", {
                    "multiline": True,
                    "default": "",
                    "tooltip": "Optional negative prompt. V2 can persistently add poorly rated tags and return repaired negative conditioning.",
                }),
                "refinement_key_input": ("STRING", {
                    "default": "",
                    "multiline": False,
                    "forceInput": True,
                    "tooltip": "Optional linked refinement key, for example from FunPack Refinement Key Loader. Overrides the refinement_key widget when connected.",
                }),
                "user_intent_prompt": ("STRING", {
                    "multiline": True,
                    "default": "",
                    "tooltip": "Optional raw user/request prompt. V2 can use this as an intent source to repair missing action/detail/quality phrases if an enhancer omits them.",
                }),
                "im_feeling_lucky": ("BOOLEAN", {
                    "default": False,
                    "label": "I'm Feeling Lucky",
                    "tooltip": "Compose a learned prompt from V2 phrase memory, then encode it through the connected CLIP.",
                }),
                "advisor_thinking": ("BOOLEAN", {
                    "default": True,
                    "label": "Advisor Thinking",
                    "tooltip": "Let compatible Gemma CLIP text generators use thinking mode for advisor diagnostics and prompt repair.",
                }),
            },
        }

    @classmethod
    def IS_CHANGED(cls, **kwargs):
        return float("nan")

    def _v2_state_path(self, refinement_key):
        return refinement_state_path(refinement_key, "clip", prefix=self.V2_STATE_PREFIX)

    def _v2_execution_mode(self, mode):
        clean = str(mode or "Refine").strip().lower()
        if clean in {"learning", "learn", "observe", "observer", "observation"}:
            return "Learning"
        return "Refine"

    def _v2_advisor_mode(self, mode):
        clean = str(mode or "Off").strip().lower().replace("_", " ").replace("-", " ")
        if clean in {"diagnostic", "diagnostics", "observe", "analysis"}:
            return "Diagnostics"
        if clean in {"repair", "repair prompt", "prompt repair", "advisor repair"}:
            return "Repair prompt"
        return "Off"

    def _v2_empty_state(self, refinement_key):
        return {
            "version": 2,
            "refinement_key": refinement_key,
            "state_namespace": "clip",
            "global": {
                "total_iterations": 0,
                "avg_reward_ema": 0.0,
                "good_streak": 0,
                "bad_streak": 0,
                "last_rating_label": "Initial discovery",
                "last_missing_axes": [],
                "phrase_memory": {},
                "axis_conditioning_memory": {},
                "negative_prompt_memory": {},
                "lora_weight_memory": {},
                "preferred_context_memory": {},
                "intent_alignment_memory": {},
                "intent_family_memory": {},
                "perfect_anchors": {},
                "variant_evidence": {},
                "intent_preference_phrases": {},
                "conditioning_deltas": {},
                "active_repair_axes": [],
                "vision_memory": {},
                "loss_history": [],
            },
            "prompt_histories": {},
            "last_run": None,
        }

    def _v2_load_state(self, refinement_key, reset_session=False):
        path = self._v2_state_path(refinement_key)
        if reset_session or not os.path.exists(path):
            preserved_scene_builder = None
            if reset_session and os.path.exists(path):
                try:
                    with open(path, "r", encoding="utf-8") as file:
                        previous = json.load(file)
                    if isinstance(previous, dict) and isinstance(previous.get("scene_builder"), dict):
                        preserved_scene_builder = previous["scene_builder"]
                except (json.JSONDecodeError, OSError, ValueError):
                    preserved_scene_builder = None
            state = self._v2_empty_state(refinement_key)
            if preserved_scene_builder is not None:
                state["scene_builder"] = preserved_scene_builder
            return state, "fresh"
        try:
            with open(path, "r", encoding="utf-8") as file:
                data = json.load(file)
            if not isinstance(data, dict) or int(data.get("version", 0)) != 2:
                return self._v2_empty_state(refinement_key), "reset invalid"
            data.setdefault("global", {})
            data.setdefault("prompt_histories", {})
            data.setdefault("last_run", None)
            data["global"].setdefault("axis_conditioning_memory", {})
            data["global"].setdefault("negative_prompt_memory", {})
            data["global"].setdefault("phrase_memory", {})
            data["global"].setdefault("lora_weight_memory", {})
            data["global"].setdefault("preferred_context_memory", {})
            data["global"].setdefault("intent_alignment_memory", {})
            data["global"].setdefault("intent_family_memory", {})
            data["global"].setdefault("perfect_anchors", {})
            data["global"].setdefault("variant_evidence", {})
            data["global"].setdefault("intent_preference_phrases", {})
            data["global"].setdefault("conditioning_deltas", {})
            data["global"].setdefault("active_repair_axes", [])
            data["global"].setdefault("vision_memory", {})
            data["global"].setdefault("loss_history", [])
            return data, "loaded"
        except (json.JSONDecodeError, OSError, ValueError):
            return self._v2_empty_state(refinement_key), "reset unreadable"

    def _v2_save_state(self, data, refinement_key):
        path = self._v2_state_path(refinement_key)
        os.makedirs(os.path.dirname(path), exist_ok=True)
        with open(path, "w", encoding="utf-8") as file:
            json.dump(self._v2_json_safe(data), file, indent=2)

    def _v2_json_safe(self, value):
        if isinstance(value, dict):
            return {str(key): self._v2_json_safe(item) for key, item in value.items()}
        if isinstance(value, (list, tuple)):
            return [self._v2_json_safe(item) for item in value]
        if isinstance(value, set):
            return [self._v2_json_safe(item) for item in sorted(value)]
        return value

    def _v2_prompt_key(self, prompt):
        return re.sub(r"\s+", " ", str(prompt or "").strip())

    def _v2_extract_conditioning(self, encoded):
        if not isinstance(encoded, list) or not encoded:
            return None, {"pooled_output": None}
        item = encoded[0]
        if isinstance(item, (list, tuple)) and len(item) >= 2:
            cond = item[0]
            meta = item[1] if isinstance(item[1], dict) else {"pooled_output": None}
        else:
            cond = item if isinstance(item, torch.Tensor) else None
            meta = {"pooled_output": None}
        return cond, meta

    def _v2_encode_prompt(self, clip, prompt_text, encode_cache=None):
        if clip is None:
            return None, {"pooled_output": None}, "CLIP missing"
        prompt_text = str(prompt_text or "").strip()
        if not prompt_text:
            return None, {"pooled_output": None}, "prompt empty"
        cache_key = None
        if isinstance(encode_cache, dict):
            cache_key = (id(clip), prompt_text)
            cached = encode_cache.get(cache_key)
            if cached is not None:
                return cached
        try:
            encoded = clip.encode_from_tokens_scheduled(clip.tokenize(prompt_text))
        except Exception as error:
            return None, {"pooled_output": None}, f"encode failed: {error}"
        cond, meta = self._v2_extract_conditioning(encoded)
        if not isinstance(cond, torch.Tensor):
            return None, {"pooled_output": None}, "encode returned invalid conditioning"
        result = (cond, meta, f"encoded {self._get_conditioning_seq_len(cond)} positions")
        if cache_key is not None:
            encode_cache[cache_key] = result
        return result

    def _v2_gemma3_tokenizer_status(self):
        tokenizer = self._get_tokenizer("ltx2")
        if tokenizer is None:
            return "Gemma3 tokenizer unavailable"
        source = str(getattr(tokenizer, "name_or_path", "") or "").strip()
        if source:
            return f"Gemma3 tokenizer loaded: {source}"
        return "Gemma3 tokenizer loaded"

    def _v2_conditioning_source(self, clip, prompt_text, positive_conditioning, encode_cache=None):
        if clip is not None:
            cond, meta, encode_status = self._v2_encode_prompt(clip, prompt_text, encode_cache=encode_cache)
            return cond, meta, encode_status, "CLIP-owned"

        if positive_conditioning is None:
            return (
                None,
                {"pooled_output": None},
                "CLIP missing and no positive CONDITIONING connected",
                "CONDITIONING-owned",
            )

        cond, meta = self._v2_extract_conditioning(positive_conditioning)
        if not isinstance(cond, torch.Tensor):
            return (
                None,
                {"pooled_output": None},
                "connected positive CONDITIONING is invalid",
                "CONDITIONING-owned",
            )

        tokenizer_status = self._v2_gemma3_tokenizer_status()
        encode_status = (
            f"accepted connected positive CONDITIONING "
            f"({self._get_conditioning_seq_len(cond)} positions); {tokenizer_status}"
        )
        return cond, meta, encode_status, "CONDITIONING-owned"

    def _v2_conditioning_vector(self, conditioning):
        if not isinstance(conditioning, torch.Tensor) or conditioning.dim() <= 1:
            return None
        if conditioning.dim() == 3:
            return conditioning.detach().float().mean(dim=(0, 1))
        return conditioning.detach().float().mean(dim=0)

    def _v2_phrase_words(self, text):
        return [
            word.strip().lower()
            for word in re.findall(r"[\w'’.-]+", str(text or ""), flags=re.UNICODE)
            if self._is_valuable_token(word.strip())
        ]

    def _v2_has_action_suffix(self, words):
        for word in words:
            if word.endswith("ing"):
                stem = word[:-3]
            elif word.endswith("ed"):
                stem = word[:-2]
            else:
                continue
            if stem in self.ACTION_SUFFIX_STEMS:
                return True
            if stem.endswith("n") and stem[:-1] in self.ACTION_SUFFIX_STEMS:
                return True
            if stem.endswith("p") and stem[:-1] in self.ACTION_SUFFIX_STEMS:
                return True
        return False

    def _v2_rebalance_anchored_scores(self, scores, words):
        words = set(words or [])
        has_action = bool(words & self.CATEGORY_KEYWORDS["action"]) or self._v2_has_action_suffix(words)
        if words & self.APPEARANCE_CONTEXT_WORDS:
            scores["appearance"] = max(scores["appearance"], 0.66)
            if not has_action:
                scores["action"] = min(scores["action"], 0.40)
        if words & self.ENVIRONMENT_CONTEXT_WORDS:
            scores["environment"] = max(scores["environment"], 0.66)
            if not has_action:
                scores["action"] = min(scores["action"], 0.36)
        if words & self.QUALITY_CONTEXT_WORDS:
            scores["quality"] = max(scores["quality"], 0.64)
            if not has_action:
                scores["action"] = min(scores["action"], 0.34)
        if has_action:
            scores["action"] = max(scores["action"], 0.66)
        return scores

    def _v2_heuristic_scores(self, phrase):
        words = set(self._v2_phrase_words(phrase))
        scores = {category: 0.0 for category in self.CATEGORY_DESCRIPTIONS}
        for category, keywords in self.CATEGORY_KEYWORDS.items():
            hits = len(words & keywords)
            if hits:
                scores[category] += min(0.92, 0.48 + hits * 0.18)

        text = str(phrase or "").lower()
        if self._v2_has_action_suffix(words):
            scores["action"] = max(scores["action"], 0.52)
        if "camera" in text and re.search(r"\b(move|moves|moving|push|pull|pan|zoom|track|dolly|tilt)", text):
            scores["camera"] = max(scores["camera"], 0.78)
            scores["action"] = max(scores["action"], 0.48)
        if re.search(r"\b(slowly|quickly|gentle|smooth|fast)\b", text) and scores["action"] > 0.0:
            scores["action"] = min(0.92, scores["action"] + 0.12)
        scores = self._v2_rebalance_anchored_scores(scores, words)
        return scores

    def _v2_scores_primary(self, scores):
        if not scores:
            return "details", 0.0
        primary, confidence = max(
            scores.items(),
            key=lambda item: (item[1], self.CATEGORY_PRIORITY.get(item[0], 0.0)),
        )
        return primary, float(confidence)

    def _v2_primary_category_for_text(self, text):
        return self._v2_scores_primary(self._v2_heuristic_scores(text))[0]

    def _v2_item_primary_category(self, item):
        if not isinstance(item, dict):
            return self._v2_primary_category_for_text(item)
        scores = (
            item.get("effective_category_scores") or
            item.get("category_scores") or
            item.get("clip_heuristic_scores") or
            self._v2_heuristic_scores(item.get("text", ""))
        )
        primary, _ = self._v2_scores_primary(scores)
        return str(item.get("primary") or primary or "details").lower()

    def _v2_item_has_blocked_auto_category(self, item):
        if isinstance(item, dict):
            categories = {
                str(item.get("primary", "")).lower(),
                str(item.get("machine_primary", "")).lower(),
                self._v2_item_primary_category(item),
            }
            scores = item.get("effective_category_scores", item.get("category_scores", {}))
            if isinstance(scores, dict):
                primary, confidence = self._v2_scores_primary(scores)
                categories.add(primary)
                for category, score in scores.items():
                    if float(score or 0.0) >= max(0.42, float(confidence) * 0.82):
                        categories.add(str(category).lower())
            return bool(categories & self.AUTO_INJECT_BLOCKED_CATEGORIES)
        return self._v2_primary_category_for_text(item) in self.AUTO_INJECT_BLOCKED_CATEGORIES

    def _v2_scene_key(self, text):
        text = re.sub(r"[^\w'’]+", " ", str(text or "").strip().lower(), flags=re.UNICODE)
        return re.sub(r"\s+", " ", text).strip()

    def _v2_now_iso(self):
        return datetime.now(timezone.utc).replace(microsecond=0).isoformat()

    def _v2_scene_builder_db(self, state):
        if not isinstance(state, dict):
            return {}, "Scene Builder: unavailable."
        scene_db = state.get("scene_builder")
        if not isinstance(scene_db, dict):
            return {}, "Scene Builder: unavailable."
        memory = scene_db.get("universal_memory", {})
        scenes = scene_db.get("scenes", {})
        memory_count = len(memory) if isinstance(memory, dict) else 0
        scene_count = len(scenes) if isinstance(scenes, dict) else 0
        return scene_db, f"Scene Builder: available ({memory_count} database phrase(s), {scene_count} saved scene(s))."

    def _v2_scene_memory_item(self, scene_db, text):
        if not isinstance(scene_db, dict):
            return None
        memory = scene_db.get("universal_memory", {})
        if not isinstance(memory, dict):
            return None
        key = self._v2_scene_key(text)
        if key and isinstance(memory.get(key), dict):
            return memory[key]
        clean = self._v2_clean_phrase_text(text)
        for item in memory.values():
            if not isinstance(item, dict):
                continue
            item_text = self._v2_clean_phrase_text(item.get("text", ""))
            if item_text and clean and self._v2_phrase_texts_match(clean, item_text):
                return item
        return None

    def _v2_apply_scene_builder_authority(self, item, scene_db):
        scene_item = self._v2_scene_memory_item(scene_db, item.get("text", "") if isinstance(item, dict) else item)
        if not isinstance(item, dict) or not isinstance(scene_item, dict) or not bool(scene_item.get("category_locked")):
            return False
        category = str(scene_item.get("category") or "").strip().lower()
        item["scene_category_locked"] = True
        item["scene_category_source"] = "user"
        item["scene_category"] = category
        if category not in self.CATEGORY_DESCRIPTIONS:
            item["source"] = "scene_builder_user"
            return True
        scores = self._v2_category_template(0.0)
        scores[category] = 1.0
        item["category_scores"] = dict(scores)
        item["clip_heuristic_scores"] = dict(scores)
        item["machine_primary"] = category
        item["machine_confidence"] = 1.0
        item["category_weights"] = self._v2_category_template(0.0)
        item["category_evidence_count"] = 0
        item["effective_category_scores"] = dict(scores)
        item["primary"] = category
        item["confidence"] = 1.0
        item["source"] = "scene_builder_user"
        return True

    def _v2_prompt_repair_text_allowed(self, text):
        if not self._v2_clean_phrase_text(text):
            return False
        if self._v2_item_has_blocked_auto_category(text):
            return False
        return self._v2_primary_category_for_text(text) in self.AUTO_INJECT_ALLOWED_CATEGORIES

    def _v2_auto_inject_entry_allowed(self, entry, prompt=""):
        text = str(entry.get("text", "") if isinstance(entry, dict) else entry).strip()
        if not text:
            return False
        explicit = self._v2_prompt_contains_text(prompt, text)
        if isinstance(entry, dict) and bool(entry.get("auto_inject_suppressed")) and not explicit:
            return False
        if self._v2_item_has_blocked_auto_category(entry) and not explicit:
            return False
        return True

    def _v2_clip_similarity_scores(self, clip, phrase, category_vectors, encode_cache=None):
        phrase_cond, _, _ = self._v2_encode_prompt(clip, phrase, encode_cache=encode_cache)
        phrase_vector = self._v2_conditioning_vector(phrase_cond)
        if phrase_vector is None:
            return {}
        scores = {}
        for category, vector in category_vectors.items():
            if not isinstance(vector, torch.Tensor) or vector.shape != phrase_vector.shape:
                continue
            sim = F.cosine_similarity(phrase_vector.unsqueeze(0), vector.to(phrase_vector.device).unsqueeze(0), dim=-1)
            scores[category] = float(((sim.item() + 1.0) * 0.5))
        return scores

    def _v2_category_vectors(self, clip, encode_cache=None):
        if isinstance(encode_cache, dict):
            cache_key = ("category_vectors", id(clip))
            cached = encode_cache.get(cache_key)
            if cached is not None:
                return cached
        vectors = {}
        for category, description in self.CATEGORY_DESCRIPTIONS.items():
            cond, _, _ = self._v2_encode_prompt(clip, description, encode_cache=encode_cache)
            vector = self._v2_conditioning_vector(cond)
            if vector is not None:
                vectors[category] = vector.cpu()
        if isinstance(encode_cache, dict):
            encode_cache[cache_key] = vectors
        return vectors

    def _v2_merge_clip_category_scores(self, heuristic_scores, clip_scores):
        if not clip_scores:
            return heuristic_scores
        merged = dict(heuristic_scores)
        heuristic_primary, heuristic_confidence = self._v2_scores_primary(heuristic_scores)
        ranked_clip = sorted(clip_scores.items(), key=lambda item: item[1], reverse=True)
        if not ranked_clip:
            return merged
        clip_primary, clip_confidence = ranked_clip[0]
        clip_runner_up = ranked_clip[1][1] if len(ranked_clip) > 1 else 0.0
        clip_margin = float(clip_confidence) - float(clip_runner_up)

        if heuristic_confidence >= 0.60:
            return merged
        if heuristic_confidence >= 0.44 and clip_primary != heuristic_primary and clip_margin < 0.075:
            return merged

        mean_clip = sum(float(value) for value in clip_scores.values()) / max(1, len(clip_scores))
        for category, score in clip_scores.items():
            relative = max(0.0, float(score) - mean_clip)
            merged[category] = max(float(merged.get(category, 0.0)), min(0.56, relative * 2.40))
        if clip_margin >= 0.045:
            merged[clip_primary] = max(float(merged.get(clip_primary, 0.0)), min(0.68, 0.48 + clip_margin * 2.20))
        return merged

    def _v2_category_template(self, value=0.0):
        return {category: float(value) for category in self.CATEGORY_DESCRIPTIONS}

    def _v2_clean_category_scores(self, scores, default=0.0):
        cleaned = self._v2_category_template(default)
        if isinstance(scores, dict):
            for category in cleaned:
                try:
                    cleaned[category] = float(scores.get(category, cleaned[category]))
                except (TypeError, ValueError):
                    cleaned[category] = float(default)
        return cleaned

    def _v2_axis_categories(self, axis):
        if axis == "action":
            return {"action", "camera"}
        if axis == "details":
            return {"details", "appearance", "environment", "camera", "action"}
        if axis == "quality":
            return {"quality", "style"}
        return set()

    def _v2_axis_category_weight(self, axis, category, scores=None):
        if category not in self._v2_axis_categories(axis):
            return 0.0
        if axis == "details" and category == "details":
            return 1.18
        if axis == "details" and category == "action":
            return 0.58
        if axis == "details" and category in {"appearance", "environment"}:
            return 0.72
        if axis == "action" and category == "camera":
            return 0.58
        if axis == "details" and category == "camera":
            return 0.64
        scores = scores or {}
        primary, _ = self._v2_scores_primary(scores)
        return 1.0 if category == primary else 0.78

    def _v2_effective_category_scores(self, machine_scores, category_weights, evidence_count=0):
        machine = self._v2_clean_category_scores(machine_scores)
        learned = self._v2_clean_category_scores(category_weights)
        evidence_count = max(0, int(evidence_count or 0))
        learned_power = min(1.85, 0.30 + evidence_count * 0.18)
        effective = {}
        for category in machine:
            value = float(machine.get(category, 0.0)) + float(learned.get(category, 0.0)) * learned_power
            effective[category] = round(max(-1.0, min(8.0, value)), 6)
        return effective

    def _v2_position_bucket(self, index, total):
        total = max(1, int(total or 1))
        index = max(0, int(index or 0))
        if total <= 2:
            return "solo" if total == 1 else ("early" if index == 0 else "late")
        ratio = index / float(max(1, total - 1))
        if ratio < 0.34:
            return "early"
        if ratio > 0.66:
            return "late"
        return "mid"

    def _v2_context_for_phrase(self, phrases, index, text, window=3):
        phrases = phrases or []
        index = max(0, min(len(phrases) - 1, int(index or 0))) if phrases else 0
        anchor_words = set(self._v2_phrase_words(text))
        context_words = []
        for offset in range(max(0, index - window), min(len(phrases), index + window + 1)):
            if offset == index:
                continue
            item = phrases[offset]
            item_text = item.get("text", "") if isinstance(item, dict) else str(item)
            for word in self._v2_phrase_words(item_text):
                if word in anchor_words or word in context_words:
                    continue
                context_words.append(word)
        bucket = self._v2_position_bucket(index, len(phrases))
        return {
            "anchor_words": sorted(anchor_words),
            "context_words": context_words[:10],
            "position_bucket": bucket,
            "window": int(window),
        }

    def _v2_context_signature(self, context):
        if not isinstance(context, dict):
            return ""
        words = sorted({
            word
            for word in context.get("context_words", [])
            if self._is_valuable_token(str(word).strip())
        })[:8]
        bucket = str(context.get("position_bucket", "mid"))
        return f"{bucket}|{','.join(words)}" if words else bucket

    def _v2_context_similarity(self, left, right):
        if not isinstance(left, dict) or not isinstance(right, dict):
            return 0.0
        left_words = {
            word for word in left.get("context_words", [])
            if self._is_valuable_token(str(word).strip())
        }
        right_words = {
            word for word in right.get("context_words", [])
            if self._is_valuable_token(str(word).strip())
        }
        if not left_words and not right_words:
            word_score = 0.0
        else:
            word_score = len(left_words & right_words) / float(max(1, len(left_words | right_words)))
        bucket_bonus = 0.12 if left.get("position_bucket") == right.get("position_bucket") else 0.0
        return max(0.0, min(1.0, word_score + bucket_bonus))

    def _v2_ensure_context_sense(self, entry, context):
        signature = self._v2_context_signature(context)
        if not signature:
            return None
        senses = entry.setdefault("context_senses", {})
        initial_weights = self._v2_clean_category_scores(entry.get("category_weights", {}))
        sense = senses.setdefault(signature, {
            "signature": signature,
            "context": context,
            "category_weights": {
                category: round(float(value) * 0.35, 6)
                for category, value in initial_weights.items()
            },
            "effective_category_scores": self._v2_clean_category_scores(entry.get("effective_category_scores", {})),
            "category_evidence_count": 0,
            "occurrence_count": 0,
            "last_seen_iter": 0,
            "rating_evidence": {},
        })
        sense["context"] = context
        sense["category_weights"] = self._v2_clean_category_scores(sense.get("category_weights", {}))
        sense["effective_category_scores"] = self._v2_clean_category_scores(sense.get("effective_category_scores", {}))
        sense.setdefault("category_evidence_count", 0)
        sense.setdefault("occurrence_count", 0)
        sense.setdefault("last_seen_iter", 0)
        sense.setdefault("rating_evidence", {})
        return sense

    def _v2_prune_context_senses(self, entry, limit=24):
        senses = entry.get("context_senses", {})
        if not isinstance(senses, dict) or len(senses) <= limit:
            return

        def rank(item):
            _, sense = item
            if not isinstance(sense, dict):
                return (-1, -1, -1)
            return (
                int(sense.get("category_evidence_count", 0)),
                int(sense.get("occurrence_count", 0)),
                int(sense.get("last_seen_iter", 0)),
            )

        kept = dict(sorted(senses.items(), key=rank, reverse=True)[:limit])
        entry["context_senses"] = kept

    def _v2_best_context_sense(self, entry, context):
        if not isinstance(entry, dict) or not isinstance(context, dict):
            return None, 0.0
        senses = entry.get("context_senses", {})
        if not isinstance(senses, dict):
            return None, 0.0
        signature = self._v2_context_signature(context)
        if signature and isinstance(senses.get(signature), dict):
            return senses[signature], 1.0
        best_sense = None
        best_score = 0.0
        for sense in senses.values():
            if not isinstance(sense, dict):
                continue
            score = self._v2_context_similarity(context, sense.get("context", {}))
            if score > best_score:
                best_sense = sense
                best_score = score
        if best_score < 0.22:
            return None, 0.0
        return best_sense, best_score

    def _v2_blend_category_weights(self, target, source, scale=1.0):
        source = self._v2_clean_category_scores(source)
        for category, value in source.items():
            target[category] = float(target.get(category, 0.0)) + float(value) * float(scale)
        return target

    def _v2_ensure_phrase_memory_entry(self, memory, text, phrase=None):
        text = str(text or "").strip().lower()
        phrase = phrase or {}
        primary = phrase.get("primary") or phrase.get("machine_primary") or "details"
        entry = memory.setdefault(text, {
            "text": text,
            "tokens": phrase.get("tokens", []),
            "primary": primary,
            "machine_primary": primary,
            "category_scores": phrase.get("category_scores", {}),
            "clip_heuristic_scores": phrase.get("clip_heuristic_scores", phrase.get("category_scores", {})),
            "category_weights": self._v2_category_template(0.0),
            "effective_category_scores": phrase.get("effective_category_scores", phrase.get("category_scores", {})),
            "category_evidence_count": 0,
            "rating_evidence": {},
            "context_senses": {},
            "occurrence_count": 0,
            "score": 0.0,
            "liked_count": 0,
            "missing_count": 0,
            "satisfied_count": 0,
            "resolved_count": 0,
            "regressed_count": 0,
            "wrong_count": 0,
            "wrong_appearance_count": 0,
            "auto_inject_blocked_count": 0,
            "auto_inject_suppressed": False,
            "bad_count": 0,
            "wanted_axes": {},
            "satisfied_axes": {},
            "resolved_axes": {},
            "wrong_axes": {},
            "positions": {},
        })
        entry.setdefault("text", text)
        entry.setdefault("tokens", phrase.get("tokens", []))
        entry.setdefault("primary", primary)
        entry.setdefault("machine_primary", phrase.get("machine_primary", entry.get("primary", primary)))
        entry["category_scores"] = self._v2_clean_category_scores(entry.get("category_scores", phrase.get("category_scores", {})))
        entry["clip_heuristic_scores"] = self._v2_clean_category_scores(
            entry.get("clip_heuristic_scores", entry.get("category_scores", {}))
        )
        entry["category_weights"] = self._v2_clean_category_scores(entry.get("category_weights", {}))
        entry["effective_category_scores"] = self._v2_clean_category_scores(
            entry.get("effective_category_scores", entry.get("category_scores", {}))
        )
        entry.setdefault("category_evidence_count", 0)
        entry.setdefault("rating_evidence", {})
        entry.setdefault("context_senses", {})
        entry.setdefault("occurrence_count", 0)
        entry.setdefault("score", 0.0)
        entry.setdefault("liked_count", 0)
        entry.setdefault("missing_count", 0)
        entry.setdefault("satisfied_count", 0)
        entry.setdefault("resolved_count", 0)
        entry.setdefault("regressed_count", 0)
        entry.setdefault("wrong_count", 0)
        entry.setdefault("wrong_appearance_count", 0)
        entry.setdefault("auto_inject_blocked_count", 0)
        entry.setdefault("auto_inject_suppressed", False)
        entry.setdefault("bad_count", 0)
        entry.setdefault("wanted_axes", {})
        entry.setdefault("satisfied_axes", {})
        entry.setdefault("resolved_axes", {})
        entry.setdefault("wrong_axes", {})
        entry.setdefault("positions", {})
        return entry

    def _v2_apply_learned_category_scores(self, item, phrase_memory):
        if not isinstance(phrase_memory, dict):
            return item
        text = str(item.get("text", "")).strip().lower()
        context = item.get("context", {})
        entry = phrase_memory.get(text)
        machine_scores = self._v2_clean_category_scores(item.get("category_scores", {}))
        weights = self._v2_category_template(0.0)
        evidence_count = 0
        context_status = "none"
        if isinstance(entry, dict):
            sense, sense_similarity = self._v2_best_context_sense(entry, context)
            entry_scale = 0.35 if isinstance(sense, dict) else 1.0
            self._v2_blend_category_weights(weights, entry.get("category_weights", {}), entry_scale)
            evidence_count += int(int(entry.get("category_evidence_count", 0)) * entry_scale)
            if isinstance(sense, dict):
                self._v2_blend_category_weights(
                    weights,
                    sense.get("category_weights", {}),
                    1.20 + sense_similarity * 0.65,
                )
                evidence_count += int(int(sense.get("category_evidence_count", 0)) * max(0.35, sense_similarity))
                context_status = "exact" if sense_similarity >= 0.99 else f"near:{sense_similarity:.2f}"

        for word in self._v2_phrase_words(text):
            token_entry = phrase_memory.get(word)
            if not isinstance(token_entry, dict) or token_entry is entry:
                continue
            self._v2_blend_category_weights(weights, token_entry.get("category_weights", {}), 0.32)
            evidence_count += int(int(token_entry.get("category_evidence_count", 0)) * 0.22)
            sense, sense_similarity = self._v2_best_context_sense(token_entry, context)
            if isinstance(sense, dict):
                self._v2_blend_category_weights(
                    weights,
                    sense.get("category_weights", {}),
                    0.65 + sense_similarity * 0.55,
                )
                evidence_count += int(int(sense.get("category_evidence_count", 0)) * max(0.42, sense_similarity))
                context_status = "exact" if sense_similarity >= 0.99 else f"near:{sense_similarity:.2f}"

        effective = self._v2_effective_category_scores(machine_scores, weights, evidence_count)
        primary, confidence = self._v2_scores_primary(effective)
        machine_primary, machine_confidence = self._v2_scores_primary(machine_scores)
        item["clip_heuristic_scores"] = machine_scores
        item["machine_primary"] = machine_primary
        item["machine_confidence"] = round(float(machine_confidence), 4)
        item["category_weights"] = weights
        item["category_evidence_count"] = evidence_count
        item["effective_category_scores"] = effective
        item["category_scores"] = effective
        item["primary"] = primary
        item["confidence"] = round(float(confidence), 4)
        item["source"] = "rating_weighted" if evidence_count > 0 else item.get("source", "heuristic")
        item["context_signature"] = self._v2_context_signature(context)
        item["context_source"] = context_status
        return item

    def _v2_concept_units_for_run(self, last_run):
        units = []
        seen = set()

        def add(text, kind, phrase=None, position=0, context=None):
            clean = str(text or "").strip().lower()
            context = context or {}
            seen_key = f"{clean}:{self._v2_context_signature(context)}"
            if len(clean) < 3 or seen_key in seen or not any(c.isalpha() for c in clean):
                return
            words = self._v2_phrase_words(clean)
            if not words:
                return
            seen.add(seen_key)
            source = phrase or {}
            scores = source.get("clip_heuristic_scores", source.get("category_scores", self._v2_heuristic_scores(clean)))
            primary, confidence = self._v2_scores_primary(scores)
            units.append({
                "text": clean,
                "kind": kind,
                "tokens": source.get("tokens", words),
                "position": int(position),
                "context": context,
                "context_signature": self._v2_context_signature(context),
                "category_scores": self._v2_clean_category_scores(scores),
                "clip_heuristic_scores": self._v2_clean_category_scores(scores),
                "effective_category_scores": self._v2_clean_category_scores(source.get("effective_category_scores", scores)),
                "primary": source.get("primary", primary),
                "machine_primary": source.get("machine_primary", primary),
                "confidence": source.get("confidence", round(float(confidence), 4)),
                "scene_category_locked": bool(source.get("scene_category_locked")),
                "scene_category_source": source.get("scene_category_source", ""),
                "scene_category": source.get("scene_category", ""),
            })

        for index, phrase in enumerate(last_run.get("phrases", []) or []):
            if not isinstance(phrase, dict):
                continue
            text = str(phrase.get("text", "")).strip().lower()
            phrase_context = phrase.get("context") or self._v2_context_for_phrase(last_run.get("phrases", []), index, text)
            add(text, "phrase", phrase, phrase.get("position", index), phrase_context)
            words = self._v2_phrase_words(text)
            for word_index, word in enumerate(words):
                token_context = self._v2_context_for_phrase(last_run.get("phrases", []), index, word)
                add(word, "token", phrase, phrase.get("position", index) + word_index, token_context)
            for size in (2, 3):
                for start in range(0, max(0, len(words) - size + 1)):
                    add(" ".join(words[start:start + size]), "ngram", phrase, phrase.get("position", index) + start, phrase_context)

        prompt = str(last_run.get("prompt", "") or "")
        encoded_prompt = str(last_run.get("encoded_prompt", "") or "")

        def add_prompt_segments(source_prompt, kind, offset=0):
            prompt_segments = [
                {"text": segment.strip()}
                for segment in re.split(r"[,.;\n]+", source_prompt)
                if segment.strip()
            ]
            for position, segment in enumerate(prompt_segments):
                add(
                    segment.get("text", ""),
                    kind,
                    None,
                    offset + position,
                    self._v2_context_for_phrase(prompt_segments, position, segment.get("text", "")),
                )

        add_prompt_segments(prompt, "prompt_phrase")
        if encoded_prompt and encoded_prompt != prompt:
            add_prompt_segments(encoded_prompt, "auto_phrase", offset=128)

        for position, candidate in enumerate(last_run.get("repair_candidates", []) or []):
            if not isinstance(candidate, dict):
                continue
            add(candidate.get("text", ""), "repair_candidate", candidate, 256 + position, {})
        return units[:96]

    def _v2_classify_phrases(self, clip, phrases, global_state=None, encode_cache=None, scene_db=None):
        phrase_items = []
        uncertain = []
        phrase_memory = (global_state or {}).get("phrase_memory", {}) if isinstance(global_state, dict) else {}
        phrases = phrases or []
        for index, phrase in enumerate(phrases):
            text = str(phrase.get("text", "")).strip().lower()
            if not text:
                continue
            scores = self._v2_heuristic_scores(text)
            primary, confidence = self._v2_scores_primary(scores)
            context = self._v2_context_for_phrase(phrases, index, text)
            item = {
                "text": text,
                "tokens": list(phrase.get("tokens", [])),
                "position": index,
                "context": context,
                "context_signature": self._v2_context_signature(context),
                "category_scores": scores,
                "clip_heuristic_scores": dict(scores),
                "machine_primary": primary,
                "machine_confidence": round(float(confidence), 4),
                "category_weights": self._v2_category_template(0.0),
                "category_evidence_count": 0,
                "effective_category_scores": dict(scores),
                "primary": primary,
                "confidence": round(float(confidence), 4),
                "source": "heuristic",
            }
            if confidence < 0.62:
                uncertain.append(item)
            phrase_items.append(item)

        if uncertain and clip is not None:
            category_vectors = self._v2_category_vectors(clip, encode_cache=encode_cache)
            for item in uncertain:
                clip_scores = self._v2_clip_similarity_scores(clip, item["text"], category_vectors, encode_cache=encode_cache)
                if not clip_scores:
                    continue
                merged = self._v2_merge_clip_category_scores(item["category_scores"], clip_scores)
                primary, confidence = self._v2_scores_primary(merged)
                item["category_scores"] = merged
                item["clip_heuristic_scores"] = dict(merged)
                item["machine_primary"] = primary
                item["machine_confidence"] = round(float(confidence), 4)
                item["effective_category_scores"] = dict(merged)
                item["primary"] = primary
                item["confidence"] = round(float(confidence), 4)
                item["source"] = "clip_similarity"

        for item in phrase_items:
            if self._v2_apply_scene_builder_authority(item, scene_db):
                continue
            self._v2_apply_learned_category_scores(item, phrase_memory)

        return phrase_items

    def _v2_axis_matches_category(self, axis, category):
        category = (category or "details").lower()
        return category in self._v2_axis_categories(axis)

    def _v2_order_axes(self, axes):
        axis_set = {axis for axis in axes or [] if axis in V2_FEEDBACK_AXES}
        return [axis for axis in V2_FEEDBACK_AXES if axis in axis_set]

    def _v2_axes_for_category(self, category):
        return {
            axis
            for axis in V2_FEEDBACK_AXES
            if self._v2_axis_matches_category(axis, category)
        }

    def _v2_axes_for_scores(self, scores):
        primary, confidence = self._v2_scores_primary(scores)
        axes = set(self._v2_axes_for_category(primary))
        for category, score in (scores or {}).items():
            if float(score or 0.0) >= max(0.28, float(confidence) * 0.72):
                axes |= self._v2_axes_for_category(category)
        return axes

    def _v2_axis_feedback(self, rating_profile, previous_missing_axes=None):
        key = rating_profile.get("key", "")
        if rating_profile.get("skip_learning") or key == "discover":
            return {
                "missing_axes": [],
                "satisfied_axes": [],
                "resolved_axes": [],
                "regressed_axes": [],
                "wrong_axes": [],
            }

        all_axes = set(V2_FEEDBACK_AXES)
        missing_axes = set(rating_profile.get("missing_axes", [])) & all_axes
        wrong_axes = set(rating_profile.get("wrong_axes", [])) & all_axes
        if key == "awful":
            missing_axes = set(all_axes)
        elif key == "like":
            missing_axes = set()
            wrong_axes = set()

        has_previous_axis_signal = previous_missing_axes is not None
        previous_missing = set(previous_missing_axes or []) & all_axes
        satisfied_axes = all_axes - missing_axes
        resolved_axes = previous_missing - missing_axes
        regressed_axes = missing_axes - previous_missing if has_previous_axis_signal else set()

        return {
            "missing_axes": self._v2_order_axes(missing_axes),
            "satisfied_axes": self._v2_order_axes(satisfied_axes),
            "resolved_axes": self._v2_order_axes(resolved_axes),
            "regressed_axes": self._v2_order_axes(regressed_axes),
            "wrong_axes": self._v2_order_axes(wrong_axes),
        }

    def _v2_axis_feedback_status(self, axis_feedback):
        if not isinstance(axis_feedback, dict):
            return "Axis feedback: none."

        def fmt(name):
            values = self._v2_order_axes(axis_feedback.get(name, []))
            return ", ".join(values) if values else "none"

        return (
            f"Axis feedback: missing {fmt('missing_axes')} | "
            f"satisfied {fmt('satisfied_axes')} | "
            f"resolved {fmt('resolved_axes')} | "
            f"regressed {fmt('regressed_axes')} | "
            f"wrong {fmt('wrong_axes')}."
        )

    def _v2_active_repair_feedback(self, global_state, axis_feedback, learning_profile):
        if not isinstance(global_state, dict):
            return axis_feedback, "Repair persistence: unavailable."
        active = set(global_state.get("active_repair_axes", []) or []) & set(V2_FEEDBACK_AXES)
        key = learning_profile.get("key", "") if isinstance(learning_profile, dict) else ""
        if key == "like":
            active = set()
            global_state["active_repair_axes"] = []
            feedback = dict(axis_feedback or {})
            feedback["missing_axes"] = []
            return feedback, "Repair persistence: cleared by Perfect."
        if not learning_profile.get("skip_learning") and key != "discover":
            active |= set(axis_feedback.get("missing_axes", [])) & set(V2_FEEDBACK_AXES)
            active |= set(axis_feedback.get("wrong_axes", [])) & set(V2_FEEDBACK_AXES)
        ordered = self._v2_order_axes(active)
        global_state["active_repair_axes"] = ordered
        feedback = dict(axis_feedback or {})
        feedback["missing_axes"] = ordered
        if ordered:
            return feedback, f"Repair persistence: active until Perfect ({', '.join(ordered)})."
        return feedback, "Repair persistence: no active repairs."

    def _v2_top_category_summary(self, scores, limit=3):
        cleaned = self._v2_clean_category_scores(scores)
        ranked = [
            (category, value)
            for category, value in sorted(cleaned.items(), key=lambda item: item[1], reverse=True)
            if abs(float(value)) > 0.0001
        ]
        if not ranked:
            return "neutral"
        return ", ".join(f"{category}:{value:.2f}" for category, value in ranked[:limit])

    def _v2_training_guidance(self, has_previous_run, learning_profile, axis_feedback, phrases, memory_status, lora_status):
        if not has_previous_run:
            return (
                "Guidance: first V2 run only seeds the current prompt. "
                "Set the rating after reviewing this output, then run again with the same refinement key to train."
            )
        if learning_profile.get("skip_learning"):
            return (
                "Guidance: '-Just forget it-' skipped learning. Use it for workflow/model/seed failures; "
                "use a missing-axis rating when the prompt should teach the node."
            )
        if learning_profile.get("key") == "wrong_appearance":
            return (
                "Guidance: wrong-appearance rating suppresses auto-injected character, clothing, "
                "subject, and background memory without training unrelated action/detail/quality ratings."
            )
        if learning_profile.get("wrong_axes"):
            return (
                "Guidance: wrong-intent rating preserved satisfied axes but marked the current "
                "action/detail context as wrong for this request. The next run will repair from intent "
                "and preferred context clusters."
            )

        missing_axes = set(axis_feedback.get("missing_axes", [])) if isinstance(axis_feedback, dict) else set()
        resolved_axes = set(axis_feedback.get("resolved_axes", [])) if isinstance(axis_feedback, dict) else set()
        regressed_axes = set(axis_feedback.get("regressed_axes", [])) if isinstance(axis_feedback, dict) else set()
        phrase_axes = set()
        context_sources = set()
        weak_contexts = 0
        for phrase in phrases or []:
            phrase_axes |= self._v2_axes_for_scores(
                phrase.get("effective_category_scores", phrase.get("category_scores", {})) or
                {phrase.get("primary", "details"): 1.0}
            )
            context_source = str(phrase.get("context_source", "none"))
            context_sources.add(context_source)
            if context_source == "none":
                weak_contexts += 1

        notes = []
        if missing_axes:
            if not (missing_axes & phrase_axes):
                notes.append(
                    "missing axes did not strongly match current prompt concepts; make the missing action/detail/quality words more explicit"
                )
            else:
                notes.append(
                    "missing axes matched prompt concepts; repeated consistent ratings will strengthen those category weights"
                )
        if resolved_axes:
            notes.append("resolved axes were preserved as positive evidence")
        if regressed_axes:
            notes.append("regressed axes were marked as needing repair")
        if weak_contexts and phrases:
            notes.append("some concepts have no learned context yet; repeat similar phrasing if you want stable sense learning")
        if "exact" in context_sources:
            notes.append("at least one concept matched an exact learned context")
        if lora_status.startswith("LoRA suggestions: no"):
            notes.append("no LoRA stack connected, so only prompt/conditioning memory trained")
        if "no prompt phrases stored" in memory_status:
            notes.append("no valuable prompt concepts were found; add concrete descriptive words")
        return "Guidance: " + "; ".join(notes[:4]) + "." if notes else "Guidance: training signal looks usable."

    def _v2_category_diagnostics(self, phrases, limit=8):
        parts = []
        for item in (phrases or [])[:limit]:
            text = item.get("text", "")
            machine = item.get("machine_primary", item.get("primary", "details"))
            effective = item.get("primary", "details")
            context = item.get("context_source", "none")
            evidence = int(item.get("category_evidence_count", 0))
            top = self._v2_top_category_summary(item.get("effective_category_scores", item.get("category_scores", {})), limit=2)
            parts.append(f"- {text}: {machine}->{effective}, ctx={context}, evidence={evidence}, top={top}")
        return "Category diagnostics:\n" + ("\n".join(parts) if parts else "none")

    def _v2_phrase_axes(self, phrase):
        if not isinstance(phrase, dict):
            return set()
        scores = phrase.get("effective_category_scores", phrase.get("category_scores", {}))
        axes = self._v2_axes_for_scores(scores) if isinstance(scores, dict) else set()
        primary = phrase.get("primary")
        if primary:
            axes |= self._v2_axes_for_category(primary)
        return axes

    def _v2_clean_phrase_text(self, text):
        text = re.sub(r"\s+", " ", str(text or "").strip().lower()).strip(" ,;:.")
        if len(text) < 3 or not any(char.isalpha() for char in text):
            return ""
        if not self._v2_phrase_words(text):
            return ""
        return text

    def _v2_phrase_root_set(self, text):
        return self._v2_repair_intent_roots(text)

    def _v2_phrase_texts_match(self, left, right):
        left_clean = self._v2_clean_phrase_text(left)
        right_clean = self._v2_clean_phrase_text(right)
        if not left_clean or not right_clean:
            return False
        if self._v2_prompt_contains_text(left_clean, right_clean) or self._v2_prompt_contains_text(right_clean, left_clean):
            return True
        left_roots = self._v2_phrase_root_set(left_clean)
        right_roots = self._v2_phrase_root_set(right_clean)
        if not left_roots or not right_roots:
            return False
        overlap = left_roots & right_roots
        if not overlap:
            return False
        if min(len(left_roots), len(right_roots)) == 1:
            return True
        needed = min(2, len(left_roots))
        return len(overlap) >= needed or (len(overlap) / max(1, len(left_roots))) >= 0.58

    def _v2_phrase_represented_by(self, text, phrases):
        return any(
            self._v2_phrase_texts_match(text, phrase.get("text", "") if isinstance(phrase, dict) else phrase)
            for phrase in phrases or []
        )

    def _v2_alignment_token_units(self, text):
        words = self._v2_phrase_words(text)
        units = [{"token": word, "kind": "word", "words": [word]} for word in words]
        for index in range(0, max(0, len(words) - 1)):
            pair = " ".join(words[index:index + 2])
            units.append({"token": pair, "kind": "pair", "words": words[index:index + 2]})
        return units

    def _v2_intent_alignment_key(self, intent_prompt):
        clean = self._v2_prompt_key(intent_prompt).lower()
        return md5(clean.encode("utf-8")).hexdigest()

    def _v2_intent_source_prompt(self, prompt, intent_prompt="", intent_is_vague=False):
        intent_prompt = self._v2_prompt_key(intent_prompt)
        if intent_prompt and not intent_is_vague and not self._v2_user_intent_prompt_is_vague(intent_prompt):
            return intent_prompt
        return self._v2_prompt_key(prompt)

    def _v2_prompt_body_similarity(self, left, right):
        left_roots = self._v2_repair_intent_roots(left)
        right_roots = self._v2_repair_intent_roots(right)
        if not left_roots or not right_roots:
            return 0.0
        overlap = left_roots & right_roots
        if not overlap:
            return 0.0
        jaccard = len(overlap) / float(max(1, len(left_roots | right_roots)))
        containment = len(overlap) / float(max(1, min(len(left_roots), len(right_roots))))
        return max(jaccard, containment * 0.88)

    def _v2_intent_family_slot(self, global_state, intent_prompt, create=True):
        intent_prompt = self._v2_prompt_key(intent_prompt)
        if not isinstance(global_state, dict) or not intent_prompt:
            return "", None, 0.0
        memory = global_state.setdefault("intent_family_memory", {})
        if not isinstance(memory, dict):
            memory = {}
            global_state["intent_family_memory"] = memory
        best_key = ""
        best_slot = None
        best_similarity = 0.0
        for key, slot in memory.items():
            if not isinstance(slot, dict):
                continue
            candidates = [slot.get("intent_prompt", "")]
            candidates.extend(slot.get("intent_prompts", []) if isinstance(slot.get("intent_prompts"), list) else [])
            for candidate in candidates:
                similarity = self._v2_prompt_body_similarity(intent_prompt, candidate)
                if similarity > best_similarity:
                    best_key = key
                    best_slot = slot
                    best_similarity = similarity
        if best_slot is not None and best_similarity >= 0.58:
            return best_key, best_slot, best_similarity
        if not create:
            return "", None, best_similarity
        family_key = self._v2_intent_alignment_key(intent_prompt)
        slot = memory.setdefault(family_key, {
            "family_key": family_key,
            "intent_prompt": intent_prompt,
            "intent_prompts": [],
            "total_seen": 0,
            "perfect_anchors": {},
            "loved_variants": {},
            "variant_evidence": {},
            "intent_preference_phrases": {},
            "conditioning_deltas": {},
            "negative_tags": {},
            "last_seen_iter": 0,
        })
        return family_key, slot, 1.0

    def _v2_sync_intent_family_aliases(self, global_state, family_key, slot):
        if not family_key or not isinstance(slot, dict) or not isinstance(global_state, dict):
            return
        global_state.setdefault("perfect_anchors", {})[family_key] = slot.get("perfect_anchors", {})
        global_state.setdefault("variant_evidence", {})[family_key] = slot.get("variant_evidence", {})
        global_state.setdefault("intent_preference_phrases", {})[family_key] = slot.get("intent_preference_phrases", {})
        global_state.setdefault("conditioning_deltas", {})[family_key] = slot.get("conditioning_deltas", {})

    def _v2_intent_texts_for_family(self, intent_prompt, intent_phrases, family_slot=None):
        texts = []
        intent_prompt = self._v2_clean_phrase_text(intent_prompt)
        if intent_prompt:
            texts.append(intent_prompt)
        for phrase in intent_phrases or []:
            text = self._v2_clean_phrase_text(phrase.get("text", "") if isinstance(phrase, dict) else phrase)
            if text and text not in texts:
                texts.append(text)
        if isinstance(family_slot, dict):
            for entry in family_slot.get("intent_preference_phrases", {}).values():
                if not isinstance(entry, dict) or int(entry.get("seen_count", 0)) < 2:
                    continue
                text = self._v2_clean_phrase_text(entry.get("text", ""))
                if text and text not in texts:
                    texts.append(text)
        return texts

    def _v2_text_represented_by_intent(self, text, intent_prompt="", intent_phrases=None, family_slot=None):
        clean = self._v2_clean_phrase_text(text)
        if not clean:
            return False
        for candidate in self._v2_intent_texts_for_family(intent_prompt, intent_phrases or [], family_slot):
            if self._v2_phrase_texts_match(clean, candidate):
                return True
        return False

    def _v2_text_axes(self, text):
        scores = self._v2_heuristic_scores(text)
        axes = self._v2_axes_for_scores(scores)
        primary = self._v2_primary_category_for_text(text)
        if primary:
            axes |= self._v2_axes_for_category(primary)
        return axes

    def _v2_text_semantic_similarity(self, clip, left, right, encode_cache=None):
        if clip is None:
            return 0.0
        left = self._v2_clean_phrase_text(left)
        right = self._v2_clean_phrase_text(right)
        if not left or not right:
            return 0.0
        left_cond, _, _ = self._v2_encode_prompt(clip, left, encode_cache=encode_cache)
        right_cond, _, _ = self._v2_encode_prompt(clip, right, encode_cache=encode_cache)
        left_vector = self._v2_conditioning_vector(left_cond)
        right_vector = self._v2_conditioning_vector(right_cond)
        if left_vector is None or right_vector is None or left_vector.shape != right_vector.shape:
            return 0.0
        try:
            sim = F.cosine_similarity(left_vector.unsqueeze(0), right_vector.unsqueeze(0), dim=-1)
            return float((sim.item() + 1.0) * 0.5)
        except Exception:
            return 0.0

    def _v2_short_intent_expansions(self, intent_prompt):
        key = self._v2_clean_phrase_text(intent_prompt)
        expansions = {
            "defenestration": [
                "thrown out of window",
                "throwing a person out of a window",
                "human thrown through a window",
            ],
            "defenstration": [
                "defenestration",
                "thrown out of window",
                "throwing a person out of a window",
                "human thrown through a window",
            ],
        }
        return expansions.get(key, [])

    def _v2_mark_semantic_intent_locks(self, clip, phrases, intent_prompt="", intent_phrases=None, encode_cache=None):
        intent_prompt = self._v2_clean_phrase_text(intent_prompt)
        if not phrases or not intent_prompt:
            return phrases
        intent_words = self._v2_phrase_words(intent_prompt)
        if len(intent_words) > 3:
            return phrases
        semantic_sources = [intent_prompt]
        for expansion in self._v2_short_intent_expansions(intent_prompt):
            clean_expansion = self._v2_clean_phrase_text(expansion)
            if clean_expansion and clean_expansion not in semantic_sources:
                semantic_sources.append(clean_expansion)
        for phrase in intent_phrases or []:
            text = self._v2_clean_phrase_text(phrase.get("text", "") if isinstance(phrase, dict) else phrase)
            if text and text not in semantic_sources:
                semantic_sources.append(text)
        for phrase in phrases:
            if not isinstance(phrase, dict) or phrase.get("intent_locked"):
                continue
            text = self._v2_clean_phrase_text(phrase.get("text", ""))
            if not text:
                continue
            lexical_match = any(self._v2_phrase_texts_match(text, source) for source in semantic_sources)
            if lexical_match:
                phrase["intent_locked"] = True
                phrase["semantic_intent_locked"] = True
                phrase["intent_similarity"] = 1.0
                continue
            best = max(
                (
                    self._v2_text_semantic_similarity(clip, source, text, encode_cache=encode_cache)
                    for source in semantic_sources
                ),
                default=0.0,
            )
            if best >= 0.62:
                phrase["intent_locked"] = True
                phrase["semantic_intent_locked"] = True
                phrase["intent_similarity"] = round(float(best), 4)
        return phrases

    def _v2_requested_negative_block_texts(self, current_prompt="", intent_prompt="", intent_phrases=None, family_slot=None):
        texts = []
        for source in (current_prompt, intent_prompt):
            clean = self._v2_clean_phrase_text(source)
            if clean and clean not in texts:
                texts.append(clean)
            for phrase in self._ordered_prompt_phrases(source):
                text = self._v2_clean_phrase_text(phrase.get("text", ""))
                if text and text not in texts:
                    texts.append(text)
        for text in self._v2_intent_texts_for_family(intent_prompt, intent_phrases or [], family_slot):
            if text and text not in texts:
                texts.append(text)
        return texts

    def _v2_negative_text_conflicts_with_request(self, text, current_prompt="", intent_prompt="", intent_phrases=None, family_slot=None):
        clean = self._v2_clean_phrase_text(text)
        if not clean:
            return False
        clean_roots = self._v2_repair_intent_roots(clean)
        clean_axes = self._v2_text_axes(clean)
        for requested in self._v2_requested_negative_block_texts(current_prompt, intent_prompt, intent_phrases, family_slot):
            if (
                self._v2_prompt_contains_text(current_prompt, clean) or
                self._v2_prompt_contains_text(requested, clean) or
                self._v2_prompt_contains_text(clean, requested) or
                self._v2_phrase_texts_match(clean, requested)
            ):
                return True
            requested_roots = self._v2_repair_intent_roots(requested)
            overlap = clean_roots & requested_roots
            if not overlap:
                continue
            requested_axes = self._v2_text_axes(requested)
            if "action" in clean_axes and "action" in requested_axes:
                return True
            if len(overlap) >= 2 and (
                len(overlap) / float(max(1, min(len(clean_roots), len(requested_roots)))) >= 0.45
            ):
                return True
        return False

    def _v2_mark_intent_locks(self, phrases, intent_prompt="", intent_phrases=None, family_slot=None):
        if not phrases:
            return phrases
        for phrase in phrases:
            if not isinstance(phrase, dict):
                continue
            text = phrase.get("text", "")
            locked = self._v2_text_represented_by_intent(text, intent_prompt, intent_phrases, None)
            preference_locked = self._v2_text_represented_by_intent(text, "", [], family_slot) and not locked
            phrase["intent_locked"] = bool(locked)
            phrase["preference_locked"] = bool(preference_locked)
        return phrases

    def _v2_serialized_delta(self, target_payload, source_payload):
        if not isinstance(target_payload, dict) or not isinstance(source_payload, dict):
            return None
        try:
            target = serializable_to_tensor(target_payload).detach().float()
            source = serializable_to_tensor(source_payload).detach().float()
            if list(target.shape) != list(source.shape):
                return None
            return tensor_to_serializable((target - source).cpu())
        except Exception:
            return None

    def _v2_store_conditioning_delta_average(self, slot, delta_payload):
        if not isinstance(slot, dict) or not isinstance(delta_payload, dict):
            return False
        count = int(slot.get("count", 0))
        existing = slot.get("delta")
        if count <= 0 or not isinstance(existing, dict):
            slot["delta"] = delta_payload
            slot["count"] = 1
            return True
        try:
            previous = serializable_to_tensor(existing)
            current = serializable_to_tensor(delta_payload).to(previous.device)
            if list(previous.shape) != list(current.shape):
                slot["delta"] = delta_payload
                slot["count"] = 1
                return True
            averaged = (previous * count + current) / float(count + 1)
            slot["delta"] = tensor_to_serializable(averaged.cpu())
            slot["count"] = count + 1
            return True
        except Exception:
            slot["delta"] = delta_payload
            slot["count"] = 1
            return True

    def _v2_update_intent_family_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
        if not isinstance(global_state, dict) or not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
            return "Intent family: no learning update.", ""
        source_intent = self._v2_intent_source_prompt(
            last_run.get("prompt", ""),
            last_run.get("intent_prompt", ""),
            bool(last_run.get("intent_prompt_is_vague")),
        )
        if not source_intent:
            return "Intent family: no prompt body.", ""
        family_key, slot, similarity = self._v2_intent_family_slot(global_state, source_intent, create=True)
        if not isinstance(slot, dict):
            return "Intent family: unavailable.", ""

        slot["family_key"] = family_key
        slot["intent_prompt"] = slot.get("intent_prompt") or source_intent
        prompts = list(slot.get("intent_prompts", [])) if isinstance(slot.get("intent_prompts"), list) else []
        if source_intent not in prompts:
            prompts.append(source_intent)
        slot["intent_prompts"] = prompts[-12:]
        slot["total_seen"] = int(slot.get("total_seen", 0)) + 1
        slot["last_seen_iter"] = int(iter_num)

        intent_phrases = [phrase for phrase in last_run.get("intent_phrases", []) or [] if isinstance(phrase, dict)]
        if not intent_phrases:
            intent_phrases = self._v2_classify_phrases(None, self._ordered_prompt_phrases(source_intent), global_state)
        positive_phrases = [phrase for phrase in last_run.get("phrases", []) or [] if isinstance(phrase, dict)]
        self._v2_mark_intent_locks(positive_phrases, source_intent, intent_phrases, slot)

        preferences = slot.setdefault("intent_preference_phrases", {})
        preference_updates = 0
        for phrase in intent_phrases:
            payload = self._v2_intent_alignment_phrase_payload(phrase)
            if not payload:
                continue
            entry = preferences.setdefault(payload["text"], {
                **payload,
                "seen_count": 0,
                "repair_count": 0,
                "score": 0.0,
                "last_seen_iter": 0,
            })
            entry.update(payload)
            entry["seen_count"] = int(entry.get("seen_count", 0)) + 1
            entry["last_seen_iter"] = int(iter_num)
            if int(entry.get("seen_count", 0)) >= 2:
                entry["score"] = round(min(4.0, float(entry.get("score", 0.0)) + 0.36), 4)
                preference_updates += 1
            else:
                entry["score"] = round(min(4.0, float(entry.get("score", 0.0)) + 0.12), 4)

        has_semantic_intent_lock = any(
            bool(phrase.get("semantic_intent_locked"))
            for phrase in positive_phrases
            if isinstance(phrase, dict)
        )
        missing_intent = [
            phrase for phrase in intent_phrases
            if (
                not has_semantic_intent_lock and
                not self._v2_phrase_represented_by(phrase.get("text", ""), positive_phrases)
            )
        ]
        extra_positive = [
            phrase for phrase in positive_phrases
            if (
                not phrase.get("intent_locked") and
                not phrase.get("semantic_intent_locked") and
                not self._v2_text_represented_by_intent(phrase.get("text", ""), source_intent, intent_phrases, None)
            )
        ]
        repairs = [
            item for item in last_run.get("repair_candidates", []) or []
            if isinstance(item, dict) and self._v2_clean_phrase_text(item.get("text", ""))
        ]

        variant_key = md5(self._v2_prompt_key(last_run.get("prompt", "")).encode("utf-8")).hexdigest()
        variants = slot.setdefault("variant_evidence", {})
        variant = variants.setdefault(variant_key, {
            "positive_prompt": self._v2_prompt_key(last_run.get("prompt", "")),
            "encoded_prompt": self._v2_prompt_key(last_run.get("encoded_prompt", "")),
            "rating_count": 0,
            "avg_reward": 0.0,
            "accepted_count": 0,
            "rejected_count": 0,
            "missing_intent": [],
            "extra_positive": [],
            "repair_candidates": [],
            "last_rating_label": "",
            "last_seen_iter": 0,
        })
        count = int(variant.get("rating_count", 0))
        reward = float(rating_profile.get("reward", 0.0))
        variant["avg_reward"] = round((float(variant.get("avg_reward", 0.0)) * count + reward) / float(count + 1), 6)
        variant["rating_count"] = count + 1
        variant["last_rating_label"] = rating_profile.get("label", "")
        variant["last_seen_iter"] = int(iter_num)
        variant["missing_intent"] = [item.get("text", "") for item in missing_intent]
        variant["extra_positive"] = [item.get("text", "") for item in extra_positive]
        variant["repair_candidates"] = [item.get("text", "") for item in repairs]
        if rating_profile.get("key") == "like":
            variant["accepted_count"] = int(variant.get("accepted_count", 0)) + 1
        elif reward < 0.0 or rating_profile.get("wrong_axes") or rating_profile.get("key") == "awful":
            variant["rejected_count"] = int(variant.get("rejected_count", 0)) + 1

        anchor_count = 0
        delta_updates = 0
        if axis_feedback and set(axis_feedback.get("wrong_axes", [])):
            rejected_repairs = slot.setdefault("rejected_repairs", {})
            for repair in repairs:
                text = self._v2_clean_phrase_text(repair.get("text", ""))
                if not text or self._v2_phrase_represented_by(text, intent_phrases):
                    continue
                entry = rejected_repairs.setdefault(text, {"text": text, "count": 0, "last_seen_iter": 0})
                entry["count"] = int(entry.get("count", 0)) + 1
                entry["last_seen_iter"] = int(iter_num)
        if rating_profile.get("key") == "like" and isinstance(last_run.get("conditioning"), dict):
            anchor_payload = {
                "intent_prompt": source_intent,
                "positive_prompt": self._v2_prompt_key(last_run.get("prompt", "")),
                "encoded_prompt": self._v2_prompt_key(last_run.get("encoded_prompt", "")),
                "conditioning": last_run.get("conditioning"),
                "source_conditioning": last_run.get("source_conditioning"),
                "repair_candidates": [item.get("text", "") for item in repairs],
                "last_seen_iter": int(iter_num),
            }
            anchors = slot.setdefault("perfect_anchors", {})
            if "base" not in anchors:
                anchors["base"] = dict(anchor_payload, anchor_role="base")
            else:
                loved = slot.setdefault("loved_variants", {})
                loved[variant_key] = dict(anchor_payload, anchor_role="variant")
                if len(loved) > 16:
                    slot["loved_variants"] = dict(sorted(
                        loved.items(),
                        key=lambda item: int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
                        reverse=True,
                    )[:16])
            anchor_count = 1 + len(slot.get("loved_variants", {}))
            delta = self._v2_serialized_delta(last_run.get("conditioning"), last_run.get("source_conditioning"))
            if isinstance(delta, dict):
                deltas = slot.setdefault("conditioning_deltas", {})
                positive_delta = deltas.setdefault("positive", {})
                if self._v2_store_conditioning_delta_average(positive_delta, delta):
                    positive_delta["last_seen_iter"] = int(iter_num)
                    delta_updates += 1
            perfect_repairs = slot.setdefault("perfect_repairs", {})
            for repair in repairs:
                text = self._v2_clean_phrase_text(repair.get("text", ""))
                if not text:
                    continue
                entry = perfect_repairs.setdefault(text, {
                    "text": text,
                    "axes": {},
                    "count": 0,
                    "last_seen_iter": 0,
                })
                entry["count"] = int(entry.get("count", 0)) + 1
                entry["last_seen_iter"] = int(iter_num)
                axes = entry.setdefault("axes", {})
                for axis in self._v2_order_axes(set(repair.get("axes", []))):
                    axes[axis] = int(axes.get(axis, 0)) + 1

        if len(variants) > 40:
            slot["variant_evidence"] = dict(sorted(
                variants.items(),
                key=lambda item: (
                    float(item[1].get("avg_reward", 0.0)) if isinstance(item[1], dict) else -99.0,
                    int(item[1].get("rating_count", 0)) if isinstance(item[1], dict) else 0,
                    int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
                ),
                reverse=True,
            )[:40])

        self._v2_sync_intent_family_aliases(global_state, family_key, slot)
        return (
            "Intent family learned: "
            f"family={family_key[:8]} sim={similarity:.2f}, repeated preference(s)={preference_updates}, "
            f"anchors={anchor_count or len(slot.get('perfect_anchors', {})) + len(slot.get('loved_variants', {}))}, "
            f"variant reward {variant['avg_reward']:+.2f}, delta update(s)={delta_updates}."
        ), family_key

    def _v2_apply_intent_family_delta(self, conditioning, family_slot, strength):
        if not isinstance(conditioning, torch.Tensor) or not isinstance(family_slot, dict):
            return conditioning, "intent-family idle"
        positive_delta = family_slot.get("conditioning_deltas", {}).get("positive", {})
        payload = positive_delta.get("delta") if isinstance(positive_delta, dict) else None
        if not self._v2_shape_compatible(payload, conditioning):
            return conditioning, "intent-family idle"
        try:
            delta = serializable_to_tensor(payload).to(device=conditioning.device, dtype=conditioning.dtype)
            if list(delta.shape) != list(conditioning.shape):
                return conditioning, "intent-family idle"
            original_norm = conditioning.norm(dim=-1, keepdim=True).clamp_min(1e-8)
            max_delta = original_norm * min(0.030, max(0.006, float(strength) * 0.35))
            delta_norm = delta.norm(dim=-1, keepdim=True).clamp_min(1e-8)
            scale = torch.minimum(torch.ones_like(delta_norm), max_delta / delta_norm)
            return conditioning + delta * scale, f"intent-family positive delta x{int(positive_delta.get('count', 0))}"
        except Exception:
            return conditioning, "intent-family delta failed"

    def _v2_intent_alignment_phrase_payload(self, phrase):
        text = self._v2_clean_phrase_text(phrase.get("text", "") if isinstance(phrase, dict) else phrase)
        if not text:
            return None
        source = phrase if isinstance(phrase, dict) else {}
        scores = source.get("effective_category_scores", source.get("category_scores", self._v2_heuristic_scores(text)))
        axes = self._v2_order_axes(self._v2_axes_for_scores(scores))
        return {
            "text": text,
            "axes": axes,
            "primary": source.get("primary", self._v2_primary_category_for_text(text)),
            "effective_category_scores": self._v2_clean_category_scores(scores),
        }

    def _v2_update_intent_alignment_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
        if not isinstance(global_state, dict) or not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
            return "Intent alignment: no learning update."
        intent_prompt = self._v2_prompt_key(last_run.get("intent_prompt", ""))
        if not intent_prompt or last_run.get("intent_prompt_is_vague") or self._v2_user_intent_prompt_is_vague(intent_prompt):
            return "Intent alignment: no explicit original intent."

        positive_prompt = self._v2_prompt_key(last_run.get("prompt", ""))
        positive_phrases = [phrase for phrase in last_run.get("phrases", []) or [] if isinstance(phrase, dict)]
        intent_phrases = [phrase for phrase in last_run.get("intent_phrases", []) or [] if isinstance(phrase, dict)]
        if not intent_phrases:
            intent_phrases = self._v2_classify_phrases(None, self._ordered_prompt_phrases(intent_prompt), global_state)
        if not positive_phrases or not intent_phrases:
            return "Intent alignment: no comparable prompt phrases."

        has_semantic_intent_lock = any(
            bool(phrase.get("semantic_intent_locked"))
            for phrase in positive_phrases
            if isinstance(phrase, dict)
        )
        missing_intent = [
            phrase for phrase in intent_phrases
            if (
                not has_semantic_intent_lock and
                not self._v2_phrase_represented_by(phrase.get("text", ""), positive_phrases)
            )
        ]
        extra_positive = [
            phrase for phrase in positive_phrases
            if (
                not phrase.get("intent_locked") and
                not phrase.get("semantic_intent_locked") and
                not self._v2_phrase_represented_by(phrase.get("text", ""), intent_phrases) and
                not self._v2_prompt_contains_text(intent_prompt, phrase.get("text", ""))
            )
        ]

        memory = global_state.setdefault("intent_alignment_memory", {})
        intent_key = self._v2_intent_alignment_key(intent_prompt)
        slot = memory.setdefault(intent_key, {
            "intent_prompt": intent_prompt,
            "variants": {},
            "intent_enhance_pairs": {},
            "intent_tokens": {},
            "provided_tokens": {},
            "bad_tokens": {},
            "missing_intent_phrases": {},
            "extra_positive_phrases": {},
            "last_seen_iter": 0,
        })
        slot["intent_prompt"] = intent_prompt
        slot["last_seen_iter"] = int(iter_num)
        _, family_slot, _ = self._v2_intent_family_slot(global_state, intent_prompt, create=False)
        has_perfect_anchor = bool(
            isinstance(family_slot, dict) and (
                family_slot.get("perfect_anchors") or family_slot.get("loved_variants")
            )
        )

        reward = float(rating_profile.get("reward", 0.0))
        rating_key = rating_profile.get("key", "")
        axis_feedback = axis_feedback or self._v2_axis_feedback(
            rating_profile,
            global_state.get("last_missing_axes", []),
        )
        missing_axes = set(axis_feedback.get("missing_axes", []))
        wrong_axes = set(axis_feedback.get("wrong_axes", []))

        variant_key = md5(positive_prompt.encode("utf-8")).hexdigest()
        variants = slot.setdefault("variants", {})
        variant = variants.setdefault(variant_key, {
            "positive_prompt": positive_prompt,
            "rating_count": 0,
            "avg_reward": 0.0,
            "missing_intent_count": 0,
            "extra_positive_count": 0,
            "last_rating_label": "",
            "last_seen_iter": 0,
        })
        count = int(variant.get("rating_count", 0))
        variant["avg_reward"] = round((float(variant.get("avg_reward", 0.0)) * count + reward) / float(count + 1), 6)
        variant["rating_count"] = count + 1
        variant["missing_intent_count"] = len(missing_intent)
        variant["extra_positive_count"] = len(extra_positive)
        variant["last_rating_label"] = rating_profile.get("label", "")
        variant["last_seen_iter"] = int(iter_num)
        pairs = slot.setdefault("intent_enhance_pairs", {})
        pairs[variant_key] = {
            "intent_prompt": intent_prompt,
            "positive_prompt": positive_prompt,
            "rating_count": int(variant["rating_count"]),
            "avg_reward": float(variant["avg_reward"]),
            "missing_intent_count": int(variant["missing_intent_count"]),
            "extra_positive_count": int(variant["extra_positive_count"]),
            "last_rating_label": variant["last_rating_label"],
            "last_seen_iter": int(iter_num),
        }

        intent_token_memory = slot.setdefault("intent_tokens", {})
        intent_token_set = set()
        for unit in self._v2_alignment_token_units(intent_prompt):
            token = unit["token"]
            intent_token_set.add(token)
            entry = intent_token_memory.setdefault(token, {
                "token": token,
                "kind": unit["kind"],
                "words": list(unit["words"]),
                "seen_count": 0,
                "last_seen_iter": 0,
            })
            entry["kind"] = unit["kind"]
            entry["words"] = list(unit["words"])
            entry["seen_count"] = int(entry.get("seen_count", 0)) + 1
            entry["last_seen_iter"] = int(iter_num)

        provided_token_memory = slot.setdefault("provided_tokens", {})
        bad_token_memory = slot.setdefault("bad_tokens", {})
        token_updates = 0
        for phrase in positive_phrases:
            text = self._v2_clean_phrase_text(phrase.get("text", ""))
            if not text:
                continue
            phrase_axes = self._v2_phrase_axes(phrase)
            phrase_is_blocked_extra = (
                not phrase.get("intent_locked") and
                not phrase.get("semantic_intent_locked") and
                not self._v2_phrase_represented_by(text, intent_phrases) and
                not self._v2_prompt_contains_text(intent_prompt, text)
            )
            phrase_is_bad_extra = (
                rating_key == "awful" or
                (rating_key == "wrong_appearance" and self._v2_item_has_blocked_auto_category(phrase)) or
                bool(wrong_axes & phrase_axes) or
                (float(reward) < -0.35 and phrase_is_blocked_extra)
            )
            for unit in self._v2_alignment_token_units(text):
                token = unit["token"]
                entry = provided_token_memory.setdefault(token, {
                    "token": token,
                    "kind": unit["kind"],
                    "words": list(unit["words"]),
                    "score": 0.0,
                    "seen_count": 0,
                    "accepted_count": 0,
                    "rejected_count": 0,
                    "intent_count": 0,
                    "enhancer_only_count": 0,
                    "phrases": [],
                    "variants": {},
                    "omit": False,
                    "last_seen_iter": 0,
                })
                entry["kind"] = unit["kind"]
                entry["words"] = list(unit["words"])
                entry["seen_count"] = int(entry.get("seen_count", 0)) + 1
                entry["last_seen_iter"] = int(iter_num)
                entry.setdefault("variants", {})[variant_key] = int(entry.setdefault("variants", {}).get(variant_key, 0)) + 1
                phrases_for_token = list(entry.get("phrases", []))
                if text not in phrases_for_token:
                    phrases_for_token.append(text)
                entry["phrases"] = phrases_for_token[-12:]
                if token in intent_token_set:
                    entry["intent_count"] = int(entry.get("intent_count", 0)) + 1
                    delta = 0.06 if rating_key == "like" else 0.0
                else:
                    entry["enhancer_only_count"] = int(entry.get("enhancer_only_count", 0)) + 1
                    if rating_key == "like":
                        delta = 0.12
                        entry["accepted_count"] = int(entry.get("accepted_count", 0)) + 1
                    elif phrase_is_bad_extra:
                        delta = -0.60 if rating_key == "wrong_appearance" else -0.36
                        entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
                    else:
                        delta = max(-0.06, min(0.08, reward * 0.06))
                entry["score"] = round(max(-6.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
                entry["omit"] = (
                    token not in intent_token_set and
                    float(entry.get("score", 0.0)) <= -0.50 and
                    int(entry.get("rejected_count", 0)) > int(entry.get("accepted_count", 0))
                )
                if entry["omit"]:
                    bad_token_memory[token] = {
                        "token": token,
                        "kind": unit["kind"],
                        "words": list(unit["words"]),
                        "score": float(entry.get("score", 0.0)),
                        "rejected_count": int(entry.get("rejected_count", 0)),
                        "accepted_count": int(entry.get("accepted_count", 0)),
                        "phrases": list(entry.get("phrases", []))[-8:],
                        "last_seen_iter": int(iter_num),
                    }
                else:
                    bad_token_memory.pop(token, None)
                token_updates += 1

        missing_updates = 0
        for phrase in missing_intent:
            payload = self._v2_intent_alignment_phrase_payload(phrase)
            if not payload:
                continue
            axes = set(payload.get("axes", []))
            entry = slot.setdefault("missing_intent_phrases", {}).setdefault(payload["text"], {
                **payload,
                "score": 0.0,
                "missing_count": 0,
                "forgiven_count": 0,
                "last_seen_iter": 0,
            })
            if rating_key == "like":
                delta = -0.18
                entry["forgiven_count"] = int(entry.get("forgiven_count", 0)) + 1
            elif rating_key == "awful":
                delta = 0.70 if has_perfect_anchor else 0.44
                entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
            elif missing_axes:
                delta = 0.95 if axes & missing_axes else 0.28
                if not has_perfect_anchor:
                    delta = min(delta, 0.58)
                entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
            elif wrong_axes:
                delta = 0.42 if axes & wrong_axes else 0.16
                if not has_perfect_anchor:
                    delta = min(delta, 0.30)
                entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
            else:
                delta = -0.10 if reward > 0.35 else 0.18 if reward < -0.20 else 0.0
            entry.update(payload)
            entry["score"] = round(max(-2.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
            entry["last_seen_iter"] = int(iter_num)
            missing_updates += 1

        extra_updates = 0
        for phrase in extra_positive:
            payload = self._v2_intent_alignment_phrase_payload(phrase)
            if not payload:
                continue
            axes = set(payload.get("axes", []))
            entry = slot.setdefault("extra_positive_phrases", {}).setdefault(payload["text"], {
                **payload,
                "score": 0.0,
                "accepted_count": 0,
                "rejected_count": 0,
                "last_seen_iter": 0,
            })
            if rating_key == "like":
                delta = 0.42
                entry["accepted_count"] = int(entry.get("accepted_count", 0)) + 1
            elif rating_key == "wrong_appearance" and self._v2_item_has_blocked_auto_category(payload):
                delta = -0.95
                entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
            elif rating_key == "awful":
                delta = -0.78
                entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
            elif wrong_axes:
                delta = -0.70 if axes & wrong_axes else -0.25
                entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
            elif reward < -0.35:
                delta = -0.46
                entry["rejected_count"] = int(entry.get("rejected_count", 0)) + 1
            else:
                delta = max(-0.08, min(0.12, reward * 0.10))
            entry.update(payload)
            entry["score"] = round(max(-6.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
            entry["last_seen_iter"] = int(iter_num)
            extra_updates += 1

        if len(variants) > 40:
            slot["variants"] = dict(sorted(
                variants.items(),
                key=lambda item: (
                    float(item[1].get("avg_reward", 0.0)) if isinstance(item[1], dict) else -99.0,
                    int(item[1].get("rating_count", 0)) if isinstance(item[1], dict) else 0,
                    int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
                ),
                reverse=True,
            )[:40])

        if memory and len(memory) > 80:
            global_state["intent_alignment_memory"] = dict(sorted(
                memory.items(),
                key=lambda item: int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
                reverse=True,
            )[:80])

        return (
            "Intent alignment learned: "
            f"{missing_updates} omitted original phrase(s), {extra_updates} enhancer-only phrase(s), "
            f"{token_updates} provided token(s), {len(bad_token_memory)} omit token(s), "
            f"variant reward {variant['avg_reward']:+.2f}."
        )

    def _v2_apply_intent_alignment_memory(self, prompt, current_phrases, intent_prompt, intent_phrases, global_state):
        intent_prompt = self._v2_prompt_key(intent_prompt)
        if not intent_prompt or self._v2_user_intent_prompt_is_vague(intent_prompt):
            return prompt, "Intent alignment: no explicit original intent.", []
        memory = global_state.get("intent_alignment_memory", {}) if isinstance(global_state, dict) else {}
        slot = memory.get(self._v2_intent_alignment_key(intent_prompt)) if isinstance(memory, dict) else None
        if not isinstance(slot, dict):
            return prompt, "Intent alignment: no learned enhancer variants for this original prompt.", []

        intent_phrases = [phrase for phrase in intent_phrases or [] if isinstance(phrase, dict)]
        current_phrases = [phrase for phrase in current_phrases or [] if isinstance(phrase, dict)]
        additions = []
        for entry in sorted(
            slot.get("missing_intent_phrases", {}).values(),
            key=lambda item: (
                float(item.get("score", 0.0)) if isinstance(item, dict) else -99.0,
                int(item.get("missing_count", 0)) if isinstance(item, dict) else 0,
            ),
            reverse=True,
        ):
            if not isinstance(entry, dict):
                continue
            text = self._v2_clean_phrase_text(entry.get("text", ""))
            if (
                not text or
                float(entry.get("score", 0.0)) < 0.55 or
                self._v2_prompt_contains_text(prompt, text) or
                self._v2_phrase_represented_by(text, current_phrases) or
                not self._v2_phrase_represented_by(text, intent_phrases)
            ):
                continue
            additions.append(text)
            if len(additions) >= 4:
                break

        bad_extra_texts = []
        for entry in slot.get("extra_positive_phrases", {}).values():
            if not isinstance(entry, dict):
                continue
            text = self._v2_clean_phrase_text(entry.get("text", ""))
            if (
                text and
                float(entry.get("score", 0.0)) <= -0.55 and
                int(entry.get("rejected_count", 0)) > int(entry.get("accepted_count", 0)) and
                not self._v2_phrase_represented_by(text, intent_phrases)
            ):
                bad_extra_texts.append(text)
        bad_extra_tokens = {
            str(token)
            for token, entry in slot.get("bad_tokens", {}).items()
            if isinstance(entry, dict) and bool(entry.get("token"))
        }
        intent_token_set = {unit["token"] for unit in self._v2_alignment_token_units(intent_prompt)}

        removed = []
        segments = [segment.strip() for segment in re.split(r"[,;\n]+", str(prompt or "")) if segment.strip()]
        if segments and (bad_extra_texts or bad_extra_tokens):
            kept = []
            for segment in segments:
                segment_is_intent = self._v2_phrase_represented_by(segment, intent_phrases)
                segment_is_bad_extra = any(
                    self._v2_phrase_texts_match(segment, extra_text) or self._v2_prompt_contains_text(segment, extra_text)
                    for extra_text in bad_extra_texts
                )
                segment_tokens = {unit["token"] for unit in self._v2_alignment_token_units(segment)}
                segment_has_bad_token = bool((segment_tokens - intent_token_set) & bad_extra_tokens)
                if (segment_is_bad_extra or segment_has_bad_token) and not segment_is_intent:
                    removed.append(segment)
                    continue
                kept.append(segment)
            prompt = ", ".join(kept) if kept else str(prompt or "").strip()

        if additions:
            prompt = f"{prompt}, {', '.join(additions)}" if str(prompt or "").strip() else ", ".join(additions)

        if not additions and not removed:
            return prompt, "Intent alignment: learned variants present, no prompt changes needed.", []

        adjustments = [
            {"text": text, "source": "intent_missing", "action": "added"}
            for text in additions
        ] + [
            {"text": text, "source": "enhancer_extra", "action": "removed"}
            for text in removed
        ]
        return (
            prompt,
            f"Intent alignment: restored {len(additions)} original phrase(s), removed {len(removed)} rejected enhancer-only phrase(s).",
            adjustments,
        )

    def _v2_context_cluster_key(self, text, context):
        signature = self._v2_context_signature(context)
        key = f"{text}|{signature}" if signature else text
        return md5(key.encode("utf-8")).hexdigest()

    def _v2_memory_kind_scale(self, kind):
        if kind in {"prompt_phrase", "phrase"}:
            return 1.0
        if kind == "auto_phrase":
            return 0.72
        if kind == "repair_candidate":
            return 0.64
        if kind == "ngram":
            return 0.62
        if kind == "token":
            return 0.24
        return 0.50

    def _v2_update_preferred_context_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
        if not isinstance(global_state, dict) or not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
            return "Preferred context: no update."
        if rating_profile.get("key") == "wrong_appearance":
            return "Preferred context: skipped for wrong appearance."
        phrases = [phrase for phrase in last_run.get("phrases", []) or [] if isinstance(phrase, dict)]
        if not phrases:
            return "Preferred context: no prompt phrases."

        axis_feedback = axis_feedback or self._v2_axis_feedback(
            rating_profile,
            global_state.get("last_missing_axes", []),
        )
        positive_axes = set()
        if rating_profile.get("key") == "like":
            positive_axes = {"action", "details"}
            base_delta = 1.0
        else:
            positive_axes = (set(axis_feedback.get("satisfied_axes", [])) | set(axis_feedback.get("resolved_axes", []))) & {"action", "details"}
            base_delta = 0.38
        if not positive_axes:
            return "Preferred context: no liked action/detail axes."

        memory = global_state.setdefault("preferred_context_memory", {})
        touched = []
        for index, phrase in enumerate(phrases):
            text = self._v2_clean_phrase_text(phrase.get("text", ""))
            axes = self._v2_phrase_axes(phrase) & positive_axes
            if not text or not axes:
                continue

            start = max(0, index - 3)
            end = min(len(phrases), index + 4)
            cluster = []
            for pos in range(start, end):
                item_text = self._v2_clean_phrase_text(phrases[pos].get("text", ""))
                if item_text and item_text not in cluster:
                    cluster.append(item_text)
            if text not in cluster:
                cluster.insert(min(index - start, len(cluster)), text)

            context = phrase.get("context") or self._v2_context_for_phrase(phrases, index, text, window=3)
            key = self._v2_context_cluster_key(text, context)
            entry = memory.setdefault(key, {
                "anchor": text,
                "axes": {},
                "context": context,
                "phrases": [],
                "neighbors": [],
                "score": 0.0,
                "liked_count": 0,
                "satisfied_count": 0,
                "last_seen_iter": 0,
            })
            entry["anchor"] = text
            entry["context"] = context
            entry["score"] = round(max(-2.0, min(12.0, float(entry.get("score", 0.0)) + base_delta)), 4)
            if rating_profile.get("key") == "like":
                entry["liked_count"] = int(entry.get("liked_count", 0)) + 1
            else:
                entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
            entry["last_seen_iter"] = int(iter_num)

            axis_counts = entry.setdefault("axes", {})
            for axis in self._v2_order_axes(axes):
                axis_counts[axis] = int(axis_counts.get(axis, 0)) + 1

            existing_phrases = list(entry.get("phrases", []))
            for item_text in cluster:
                if item_text not in existing_phrases:
                    existing_phrases.append(item_text)
            entry["phrases"] = existing_phrases[:12]
            entry["neighbors"] = [item for item in entry["phrases"] if item != text][:10]
            memory[key] = entry
            touched.append(text)

        if len(memory) > 160:
            memory = dict(sorted(
                memory.items(),
                key=lambda item: (
                    float(item[1].get("score", 0.0)) if isinstance(item[1], dict) else -99.0,
                    int(item[1].get("liked_count", 0)) if isinstance(item[1], dict) else 0,
                    int(item[1].get("last_seen_iter", 0)) if isinstance(item[1], dict) else 0,
                ),
                reverse=True,
            )[:160])
            global_state["preferred_context_memory"] = memory

        if not touched:
            return "Preferred context: no action/detail clusters matched."
        return f"Preferred context stored: {len(touched)} action/detail cluster(s): {', '.join(touched[:5])}{'...' if len(touched) > 5 else ''}."

    def _v2_update_phrase_memory(self, global_state, last_run, rating_profile, iter_num, axis_feedback=None):
        if not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
            return "Lucky memory: no learning update."
        memory = global_state.setdefault("phrase_memory", {})
        concepts = self._v2_concept_units_for_run(last_run)
        axis_feedback = axis_feedback or self._v2_axis_feedback(
            rating_profile,
            global_state.get("last_missing_axes", []),
        )
        missing_axes = set(axis_feedback.get("missing_axes", []))
        satisfied_axes = set(axis_feedback.get("satisfied_axes", []))
        resolved_axes = set(axis_feedback.get("resolved_axes", []))
        regressed_axes = set(axis_feedback.get("regressed_axes", []))
        wrong_axes = set(axis_feedback.get("wrong_axes", []))
        reward = float(rating_profile.get("reward", 0.0))
        wrong_appearance = rating_profile.get("key") == "wrong_appearance"
        original_prompt = str(last_run.get("prompt", "") or "")
        preferred_context_status = self._v2_update_preferred_context_memory(
            global_state,
            last_run,
            rating_profile,
            iter_num,
            axis_feedback,
        )
        touched = []
        trained = []
        kind_counts = {}
        suppressed = []
        for phrase in concepts:
            text = str(phrase.get("text", "")).strip().lower()
            if not text:
                continue
            entry = self._v2_ensure_phrase_memory_entry(memory, text, phrase)
            entry["kind"] = phrase.get("kind", entry.get("kind", "phrase"))
            kind_counts[entry["kind"]] = int(kind_counts.get(entry["kind"], 0)) + 1
            entry["machine_primary"] = phrase.get("machine_primary", entry.get("machine_primary", entry.get("primary", "details")))
            entry["clip_heuristic_scores"] = self._v2_clean_category_scores(
                phrase.get("clip_heuristic_scores", phrase.get("category_scores", entry.get("clip_heuristic_scores", {})))
            )
            context = phrase.get("context", {})
            context_sense = self._v2_ensure_context_sense(entry, context)
            if isinstance(context_sense, dict):
                context_sense["occurrence_count"] = int(context_sense.get("occurrence_count", 0)) + 1
                context_sense["last_seen_iter"] = int(iter_num)
            entry["tokens"] = phrase.get("tokens", entry.get("tokens", []))
            entry["occurrence_count"] = int(entry.get("occurrence_count", 0)) + 1
            entry.setdefault("first_seen_iter", int(iter_num))
            entry.setdefault("positions", {})[str(int(phrase.get("position", 0)))] = (
                int(entry.setdefault("positions", {}).get(str(int(phrase.get("position", 0))), 0)) + 1
            )
            if phrase.get("scene_category_locked"):
                locked_category = str(phrase.get("scene_category") or phrase.get("primary") or "details").lower()
                if locked_category in self.CATEGORY_DESCRIPTIONS:
                    locked_scores = self._v2_category_template(0.0)
                    locked_scores[locked_category] = 1.0
                    entry["primary"] = locked_category
                    entry["machine_primary"] = locked_category
                    entry["category_scores"] = dict(locked_scores)
                    entry["clip_heuristic_scores"] = dict(locked_scores)
                    entry["effective_category_scores"] = dict(locked_scores)
                    entry["confidence"] = 1.0
                entry["category_source"] = "user"
                entry["category_locked"] = True
                entry["last_seen_iter"] = int(iter_num)
                memory[text] = entry
                touched.append(text)
                if len(trained) < 8:
                    trained.append(f"{text}[scene_builder:user locked]")
                continue
            kind_scale = self._v2_memory_kind_scale(entry.get("kind", "phrase"))

            machine_scores = self._v2_clean_category_scores(entry.get("clip_heuristic_scores", {}))
            weights = self._v2_clean_category_scores(entry.get("category_weights", {}))
            effective_before = self._v2_effective_category_scores(
                machine_scores,
                weights,
                int(entry.get("category_evidence_count", 0)),
            )
            phrase_axes = self._v2_axes_for_scores(effective_before)
            matched_missing = phrase_axes & missing_axes
            matched_satisfied = phrase_axes & satisfied_axes
            matched_resolved = phrase_axes & resolved_axes
            matched_regressed = phrase_axes & regressed_axes
            matched_wrong = phrase_axes & wrong_axes
            rating_evidence = entry.setdefault("rating_evidence", {})
            weights_before = dict(weights)
            count_category_evidence = rating_profile.get("key") != "discover"

            def bump_evidence(name, axes):
                axis_counts = rating_evidence.setdefault(name, {})
                for axis in self._v2_order_axes(axes):
                    axis_counts[axis] = int(axis_counts.get(axis, 0)) + 1

            def train_axis(axis, amount, prefer_current=True):
                for category in self._v2_axis_categories(axis):
                    relation = self._v2_axis_category_weight(axis, category, effective_before if prefer_current else machine_scores)
                    weights[category] = float(weights.get(category, 0.0)) + amount * relation

            if wrong_appearance:
                explicit = self._v2_prompt_contains_text(original_prompt, text)
                auto_source = phrase.get("kind") in {"auto_phrase", "repair_candidate"}
                should_suppress = self._v2_item_has_blocked_auto_category(phrase) and (auto_source or not explicit)
                if should_suppress:
                    delta = -0.95 * kind_scale
                    entry["wrong_appearance_count"] = int(entry.get("wrong_appearance_count", 0)) + 1
                    entry["auto_inject_blocked_count"] = int(entry.get("auto_inject_blocked_count", 0)) + 1
                    entry["auto_inject_suppressed"] = True
                    rating_evidence["wrong_appearance"] = int(rating_evidence.get("wrong_appearance", 0)) + 1
                    for category in self.AUTO_INJECT_BLOCKED_CATEGORIES:
                        weights[category] = float(weights.get(category, 0.0)) - 0.30 * kind_scale
                    suppressed.append(text)
                else:
                    delta = 0.0
                    count_category_evidence = False
            elif rating_profile.get("key") == "like":
                delta = 0.55 * kind_scale
                entry["liked_count"] = int(entry.get("liked_count", 0)) + 1
                rating_evidence["liked"] = int(rating_evidence.get("liked", 0)) + 1
                entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
                satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
                for axis in V2_FEEDBACK_AXES:
                    satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
                bump_evidence("satisfied_axes", V2_FEEDBACK_AXES)
                for category, score in effective_before.items():
                    if float(score) >= 0.28:
                        weights[category] = float(weights.get(category, 0.0)) + 0.13 * kind_scale
                for axis in V2_FEEDBACK_AXES:
                    train_axis(axis, 0.035 * kind_scale)
            elif rating_profile.get("key") == "awful":
                delta = -0.90 * kind_scale
                entry["bad_count"] = int(entry.get("bad_count", 0)) + 1
                rating_evidence["bad"] = int(rating_evidence.get("bad", 0)) + 1
                primary_before, _ = self._v2_scores_primary(effective_before)
                weights[primary_before] = float(weights.get(primary_before, 0.0)) - 0.045 * kind_scale
                bump_evidence("missing_axes", missing_axes)
                for axis in self._v2_order_axes(missing_axes):
                    train_axis(axis, 0.045 * kind_scale, prefer_current=False)
            elif wrong_axes:
                if matched_wrong:
                    delta = -0.18 * kind_scale
                    entry["wrong_count"] = int(entry.get("wrong_count", 0)) + 1
                    wrong_axis_counts = entry.setdefault("wrong_axes", {})
                    for axis in self._v2_order_axes(matched_wrong):
                        wrong_axis_counts[axis] = int(wrong_axis_counts.get(axis, 0)) + 1
                        train_axis(axis, 0.12 * kind_scale, prefer_current=False)
                    primary_before, _ = self._v2_scores_primary(effective_before)
                    weights[primary_before] = float(weights.get(primary_before, 0.0)) - 0.055 * kind_scale
                elif matched_satisfied:
                    delta = 0.18 * kind_scale
                    entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
                    for axis in self._v2_order_axes(matched_satisfied):
                        train_axis(axis, 0.070 * kind_scale)
                else:
                    delta = 0.02 * kind_scale

                wanted_axes = entry.setdefault("wanted_axes", {})
                for axis in self._v2_order_axes(wrong_axes):
                    wanted_axes[axis] = int(wanted_axes.get(axis, 0)) + 1
                satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
                for axis in self._v2_order_axes(matched_satisfied):
                    satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
                bump_evidence("wrong_axes", wrong_axes)
                bump_evidence("satisfied_axes", satisfied_axes)
            elif missing_axes:
                for axis in self._v2_order_axes(missing_axes - matched_missing):
                    train_axis(axis, 0.14 * kind_scale, prefer_current=False)
                if matched_missing:
                    delta = (0.34 + (0.08 if matched_regressed else 0.0)) * kind_scale
                    entry["missing_count"] = int(entry.get("missing_count", 0)) + 1
                    for axis in self._v2_order_axes(matched_missing):
                        train_axis(axis, (0.18 + (0.05 if axis in matched_regressed else 0.0)) * kind_scale)
                elif matched_resolved:
                    delta = 0.46 * kind_scale
                    entry["resolved_count"] = int(entry.get("resolved_count", 0)) + 1
                    for axis in self._v2_order_axes(matched_resolved):
                        train_axis(axis, 0.15 * kind_scale)
                elif matched_satisfied:
                    delta = 0.20 * kind_scale
                    entry["satisfied_count"] = int(entry.get("satisfied_count", 0)) + 1
                    for axis in self._v2_order_axes(matched_satisfied):
                        train_axis(axis, 0.075 * kind_scale)
                else:
                    delta = 0.04 * kind_scale

                wanted_axes = entry.setdefault("wanted_axes", {})
                for axis in self._v2_order_axes(matched_missing):
                    wanted_axes[axis] = int(wanted_axes.get(axis, 0)) + 1
                satisfied_axis_counts = entry.setdefault("satisfied_axes", {})
                for axis in self._v2_order_axes(matched_satisfied):
                    satisfied_axis_counts[axis] = int(satisfied_axis_counts.get(axis, 0)) + 1
                resolved_axis_counts = entry.setdefault("resolved_axes", {})
                for axis in self._v2_order_axes(matched_resolved):
                    resolved_axis_counts[axis] = int(resolved_axis_counts.get(axis, 0)) + 1
                bump_evidence("missing_axes", missing_axes)
                bump_evidence("satisfied_axes", satisfied_axes)
                bump_evidence("resolved_axes", resolved_axes)
                bump_evidence("regressed_axes", regressed_axes)
            else:
                delta = reward * 0.20 * kind_scale
                for category, score in effective_before.items():
                    if float(score) >= 0.32:
                        weights[category] = float(weights.get(category, 0.0)) + reward * 0.035 * kind_scale

            entry["category_weights"] = {
                category: round(max(-1.5, min(2.5, float(value))), 6)
                for category, value in weights.items()
            }
            entry["category_evidence_count"] = int(entry.get("category_evidence_count", 0)) + (
                1 if count_category_evidence else 0
            )
            if isinstance(context_sense, dict) and count_category_evidence:
                sense_weights = self._v2_clean_category_scores(context_sense.get("category_weights", {}))
                for category, value in entry["category_weights"].items():
                    sense_weights[category] = float(sense_weights.get(category, 0.0)) + (
                        float(value) - float(weights_before.get(category, 0.0))
                    ) * (1.15 * kind_scale)
                context_sense["category_weights"] = {
                    category: round(max(-1.5, min(2.5, float(value))), 6)
                    for category, value in sense_weights.items()
                }
                context_sense["category_evidence_count"] = int(context_sense.get("category_evidence_count", 0)) + 1
                context_sense["effective_category_scores"] = self._v2_effective_category_scores(
                    machine_scores,
                    context_sense["category_weights"],
                    int(context_sense.get("category_evidence_count", 0)),
                )
                sense_evidence = context_sense.setdefault("rating_evidence", {})
                sense_evidence[rating_profile.get("key", "unknown")] = (
                    int(sense_evidence.get(rating_profile.get("key", "unknown"), 0)) + 1
                )
                self._v2_prune_context_senses(entry)
            entry["effective_category_scores"] = self._v2_effective_category_scores(
                machine_scores,
                entry["category_weights"],
                int(entry.get("category_evidence_count", 0)),
            )
            primary, confidence = self._v2_scores_primary(entry["effective_category_scores"])
            entry["primary"] = primary
            entry["confidence"] = round(float(confidence), 4)
            entry["category_scores"] = dict(entry["effective_category_scores"])
            entry["score"] = round(max(-3.0, min(6.0, float(entry.get("score", 0.0)) + delta)), 4)
            entry["last_seen_iter"] = int(iter_num)
            memory[text] = entry
            touched.append(text)
            if len(trained) < 8 and entry.get("kind") in {"phrase", "token", "ngram"}:
                trained.append(
                    f"{text}[{entry.get('kind')}:{entry['machine_primary']}->{primary}, "
                    f"ctx={phrase.get('context_signature', 'none') or 'none'}, "
                    f"e={int(entry.get('category_evidence_count', 0))}, "
                    f"top={self._v2_top_category_summary(entry['effective_category_scores'], limit=2)}]"
                )

        if not touched:
            return "Lucky memory: no prompt phrases stored."
        kind_summary = ", ".join(f"{name}:{count}" for name, count in sorted(kind_counts.items())) or "none"
        suppression_summary = (
            f" Appearance auto-injection suppressed: {', '.join(suppressed[:8])}{'...' if len(suppressed) > 8 else ''}."
            if suppressed else
            ""
        )
        return (
            f"Category memory trained: {len(touched)} concept unit(s) ({kind_summary}). "
            f"Sample: {', '.join(trained) if trained else ', '.join(touched[:6])}. "
            f"{preferred_context_status}{suppression_summary} "
            f"{self._v2_axis_feedback_status(axis_feedback)}"
        )

    def _v2_shape_compatible(self, payload, conditioning):
        return self._serialized_conditioning_compatible(payload, conditioning)

    def _v2_store_conditioning_average(self, slot, payload):
        count = int(slot.get("count", 0))
        existing = slot.get("conditioning")
        if count <= 0 or not isinstance(existing, dict):
            slot["conditioning"] = payload
            slot["count"] = 1
            return
        try:
            previous = serializable_to_tensor(existing)
            current = serializable_to_tensor(payload).to(previous.device)
            if list(previous.shape) != list(current.shape):
                slot["conditioning"] = payload
                slot["count"] = 1
                return
            averaged = (previous * count + current) / float(count + 1)
            slot["conditioning"] = tensor_to_serializable(averaged.cpu())
            slot["count"] = count + 1
        except Exception:
            slot["conditioning"] = payload
            slot["count"] = 1

    def _v2_update_axis_conditioning_memory(self, global_state, payload, axis_feedback):
        if not isinstance(payload, dict):
            return
        memory = global_state.setdefault("axis_conditioning_memory", {})
        for axis in self._v2_order_axes(axis_feedback.get("satisfied_axes", [])):
            axis_slot = memory.setdefault(axis, {})
            positive = axis_slot.setdefault("positive", {})
            self._v2_store_conditioning_average(positive, payload)
        for axis in self._v2_order_axes(axis_feedback.get("resolved_axes", [])):
            axis_slot = memory.setdefault(axis, {})
            positive = axis_slot.setdefault("positive", {})
            self._v2_store_conditioning_average(positive, payload)
            axis_slot["resolved_count"] = int(axis_slot.get("resolved_count", 0)) + 1
        for axis in self._v2_order_axes(axis_feedback.get("missing_axes", [])):
            axis_slot = memory.setdefault(axis, {})
            negative = axis_slot.setdefault("negative", {})
            self._v2_store_conditioning_average(negative, payload)
        for axis in self._v2_order_axes(axis_feedback.get("regressed_axes", [])):
            axis_slot = memory.setdefault(axis, {})
            axis_slot["regressed_count"] = int(axis_slot.get("regressed_count", 0)) + 1

    def _v2_update_conditioning_memory(self, global_state, last_run, rating_profile, axis_feedback=None):
        if not isinstance(last_run, dict) or rating_profile.get("skip_learning"):
            return
        if rating_profile.get("key") == "wrong_appearance":
            return
        payload = last_run.get("conditioning")
        if not isinstance(payload, dict):
            return
        key = rating_profile.get("key", "")
        axis_feedback = axis_feedback or self._v2_axis_feedback(
            rating_profile,
            global_state.get("last_missing_axes", []),
        )
        self._v2_update_axis_conditioning_memory(global_state, payload, axis_feedback)
        if key == "like":
            count = int(global_state.get("liked_conditioning_count", 0))
            if count <= 0 or not isinstance(global_state.get("liked_conditioning"), dict):
                global_state["liked_conditioning"] = payload
                global_state["liked_conditioning_count"] = 1
                return
            try:
                previous = serializable_to_tensor(global_state["liked_conditioning"])
                current = serializable_to_tensor(payload).to(previous.device)
                if list(previous.shape) != list(current.shape):
                    global_state["liked_conditioning"] = payload
                    global_state["liked_conditioning_count"] = 1
                    return
                averaged = (previous * count + current) / float(count + 1)
                global_state["liked_conditioning"] = tensor_to_serializable(averaged.cpu())
                global_state["liked_conditioning_count"] = count + 1
            except Exception:
                global_state["liked_conditioning"] = payload
                global_state["liked_conditioning_count"] = 1
        elif key == "awful" or float(rating_profile.get("reward", 0.0)) < -0.35:
            global_state["bad_conditioning"] = payload
            global_state["bad_conditioning_count"] = int(global_state.get("bad_conditioning_count", 0)) + 1

    def _v2_update_streaks(self, global_state, rating_profile):
        reward = float(rating_profile.get("reward", 0.0))
        avg = float(global_state.get("avg_reward_ema", 0.0))
        avg = 0.86 * avg + 0.14 * reward
        global_state["avg_reward_ema"] = round(avg, 6)
        if rating_profile.get("key") == "like":
            global_state["good_streak"] = int(global_state.get("good_streak", 0)) + 1
            global_state["bad_streak"] = 0
        elif reward < -0.25:
            global_state["bad_streak"] = int(global_state.get("bad_streak", 0)) + 1
            global_state["good_streak"] = 0
        else:
            global_state["good_streak"] = 0
            global_state["bad_streak"] = 0
        global_state["last_rating_label"] = rating_profile.get("label", "")
        global_state["last_missing_axes"] = list(rating_profile.get("missing_axes", []))

    def _v2_auto_strength(self, global_state):
        avg = float(global_state.get("avg_reward_ema", 0.0))
        good = int(global_state.get("good_streak", 0))
        bad = int(global_state.get("bad_streak", 0))
        strength = 0.030
        if good >= 3 or avg > 0.55:
            strength *= 0.42
        elif good >= 1 or avg > 0.25:
            strength *= 0.68
        if bad >= 3 or avg < -0.45:
            strength *= 2.20
        elif bad >= 1 or avg < -0.15:
            strength *= 1.45
        return max(0.008, min(0.085, strength))

    def _v2_apply_conditioning_payload(self, mixed, payload, strength):
        if not self._v2_shape_compatible(payload, mixed):
            return mixed
        try:
            target = serializable_to_tensor(payload).to(device=mixed.device, dtype=mixed.dtype)
            return mixed.lerp(target, strength)
        except Exception:
            return mixed

    def _v2_repel_conditioning_payload(self, mixed, payload, strength):
        if not self._v2_shape_compatible(payload, mixed):
            return mixed
        try:
            target = serializable_to_tensor(payload).to(device=mixed.device, dtype=mixed.dtype)
            return mixed + (mixed - target) * strength
        except Exception:
            return mixed

    def _v2_apply_conditioning_memory(self, conditioning, global_state, rating_profile, axis_feedback=None, intent_family_slot=None):
        if not isinstance(conditioning, torch.Tensor):
            return conditioning, "Adaptation: unavailable."
        original = conditioning.clone()
        mixed = conditioning.clone()
        strength = self._v2_auto_strength(global_state)
        axis_feedback = axis_feedback or self._v2_axis_feedback(
            rating_profile,
            global_state.get("last_missing_axes", []),
        )
        liked_payload = global_state.get("liked_conditioning")
        if self._v2_shape_compatible(liked_payload, mixed):
            try:
                liked = serializable_to_tensor(liked_payload).to(device=mixed.device, dtype=mixed.dtype)
                mixed = mixed.lerp(liked, strength)
            except Exception:
                pass
        mixed, family_delta_status = self._v2_apply_intent_family_delta(mixed, intent_family_slot, strength)
        axis_memory = global_state.get("axis_conditioning_memory", {})
        axis_actions = []
        missing_axes = set(axis_feedback.get("missing_axes", []))
        satisfied_axes = set(axis_feedback.get("satisfied_axes", []))
        for axis in V2_FEEDBACK_AXES:
            axis_slot = axis_memory.get(axis, {}) if isinstance(axis_memory, dict) else {}
            positive = axis_slot.get("positive", {}).get("conditioning") if isinstance(axis_slot, dict) else None
            negative = axis_slot.get("negative", {}).get("conditioning") if isinstance(axis_slot, dict) else None
            if axis in missing_axes:
                before = mixed
                mixed = self._v2_apply_conditioning_payload(mixed, positive, min(0.070, strength * 1.10))
                mixed = self._v2_repel_conditioning_payload(mixed, negative, min(0.055, strength * 0.82))
                if mixed is not before:
                    axis_actions.append(f"{axis}:repair")
            elif axis in satisfied_axes:
                before = mixed
                mixed = self._v2_apply_conditioning_payload(mixed, positive, min(0.026, strength * 0.34))
                if mixed is not before:
                    axis_actions.append(f"{axis}:preserve")
        bad_payload = global_state.get("bad_conditioning")
        if self._v2_shape_compatible(bad_payload, mixed) and float(rating_profile.get("reward", 0.0)) < 0.0:
            try:
                bad = serializable_to_tensor(bad_payload).to(device=mixed.device, dtype=mixed.dtype)
                repel = (mixed - bad) * min(0.055, strength * 0.72)
                mixed = mixed + repel
            except Exception:
                pass
        delta = mixed - original
        original_norm = original.norm(dim=-1, keepdim=True).clamp_min(1e-8)
        max_delta = original_norm * 0.075
        delta_norm = delta.norm(dim=-1, keepdim=True).clamp_min(1e-8)
        scale = torch.minimum(torch.ones_like(delta_norm), max_delta / delta_norm)
        mixed = original + delta * scale
        mixed = torch.clamp(mixed, min=-60.0, max=60.0)
        mixed = mixed / mixed.norm(dim=-1, keepdim=True).clamp_min(1e-8) * original_norm
        return mixed, (
            f"Adaptation: automatic strength {strength:.3f} | "
            f"good streak {int(global_state.get('good_streak', 0))} | "
            f"bad streak {int(global_state.get('bad_streak', 0))} | "
            f"reward ema {float(global_state.get('avg_reward_ema', 0.0)):+.3f} | "
            f"axis memory {', '.join(axis_actions) if axis_actions else 'idle'} | "
            f"{family_delta_status}"
        )

    def _v2_emphasized_prompt(self, prompt, phrases, global_state, rating_profile):
        missing_axes = set(global_state.get("last_missing_axes", [])) | set(rating_profile.get("missing_axes", []))
        if not missing_axes:
            return prompt, "Prompt emphasis: none."
        candidates = []
        for phrase in phrases:
            phrase_axes = self._v2_axes_for_scores(
                phrase.get("effective_category_scores", phrase.get("category_scores", {})) or
                {phrase.get("primary", "details"): 1.0}
            )
            if phrase_axes & missing_axes:
                candidates.append(phrase.get("text", ""))
        candidates = [item for item in candidates if item]
        if not candidates:
            return prompt, "Prompt emphasis: no matching current phrases."
        emphasized = []
        seen = set()
        for text in candidates:
            key = text.lower()
            if key in seen:
                continue
            seen.add(key)
            emphasized.append(text)
            if len(emphasized) >= 3:
                break
        return f"{prompt}, {', '.join(emphasized)}", f"Prompt emphasis: repeated {', '.join(emphasized)}."

    def _v2_prompt_contains_text(self, prompt, text):
        prompt_key = re.sub(r"\s+", " ", str(prompt or "").strip().lower())
        text_key = re.sub(r"\s+", " ", str(text or "").strip().lower())
        if not prompt_key or not text_key:
            return False
        if text_key in prompt_key:
            return True
        prompt_words = set(self._v2_phrase_words(prompt_key))
        text_words = set(self._v2_phrase_words(text_key))
        return bool(text_words) and text_words <= prompt_words

    def _v2_repair_intent_roots(self, text):
        roots = set()
        for word in self._v2_phrase_words(text):
            roots.add(word)
            if len(word) > 4 and word.endswith("ing"):
                stem = word[:-3]
                roots.add(stem)
                if len(stem) > 2 and stem[-1] == stem[-2]:
                    roots.add(stem[:-1])
                roots.add(stem + "e")
            elif len(word) > 3 and word.endswith("ed"):
                stem = word[:-2]
                roots.add(stem)
                roots.add(stem + "e")
            elif len(word) > 3 and word.endswith("s"):
                roots.add(word[:-1])
        return {root for root in roots if len(root) >= 3}

    def _v2_repair_candidate_matches_intent(self, text, intent_roots):
        if not intent_roots:
            return False
        candidate_roots = self._v2_repair_intent_roots(text)
        return bool(candidate_roots & intent_roots)

    def _v2_user_intent_prompt_is_vague(self, prompt):
        text = re.sub(r"\s+", " ", str(prompt or "").strip().lower())
        if not text:
            return True
        words = set(self._v2_phrase_words(text))
        if not words:
            return True
        vague_words = {
            "figure", "out", "something", "anything", "whatever", "surprise", "random",
            "choice", "choose", "decide", "good", "nice", "cool", "best", "you", "your",
            "it", "make", "do",
        }
        if len(words) <= 5 and words <= vague_words:
            return True
        if re.fullmatch(r"(figure it out|surprise me|anything|whatever|your choice|you decide|do something|make it good|make it cool)", text):
            return True
        scores = self._v2_heuristic_scores(text)
        return len(words) <= 3 and not self._v2_axes_for_scores(scores)

    def _v2_repair_reference_context(self, text, reference_phrases):
        candidate_roots = self._v2_repair_intent_roots(text)
        if not candidate_roots:
            return {}
        best_phrase = None
        best_overlap = 0
        for phrase in reference_phrases or []:
            if not isinstance(phrase, dict):
                continue
            overlap = len(candidate_roots & self._v2_repair_intent_roots(phrase.get("text", "")))
            if overlap > best_overlap:
                best_phrase = phrase
                best_overlap = overlap
        if not isinstance(best_phrase, dict):
            return {}
        return best_phrase.get("context", {}) if isinstance(best_phrase.get("context", {}), dict) else {}

    def _v2_repair_context_matches_memory(self, text, entry, reference_phrases):
        if not isinstance(entry, dict):
            return True
        context = self._v2_repair_reference_context(text, reference_phrases)
        context_words = context.get("context_words", []) if isinstance(context, dict) else []
        senses = entry.get("context_senses", {})
        if isinstance(senses, dict) and senses:
            sense, similarity = self._v2_best_context_sense(entry, context)
            if isinstance(sense, dict) and similarity >= 0.30:
                return True
            return len(self._v2_phrase_words(text)) > 1 and not context_words
        learned_context = entry.get("context", {})
        learned_words = learned_context.get("context_words", []) if isinstance(learned_context, dict) else []
        if learned_words and context_words:
            return self._v2_context_similarity(learned_context, context) >= 0.30
        if learned_words and len(self._v2_phrase_words(text)) <= 1:
            return False
        return True

    def _v2_repair_prompt_for_missing_axes(
        self,
        prompt,
        current_phrases,
        global_state,
        previous_run,
        axis_feedback,
        intent_phrases=None,
        intent_family_slot=None,
        allow_axis_fallback=True,
    ):
        missing_axes = set(axis_feedback.get("missing_axes", [])) if isinstance(axis_feedback, dict) else set()
        missing_axes = set(self._v2_order_axes(missing_axes))
        if not missing_axes:
            return prompt, "Prompt repair: none.", []

        candidates = {}
        intent_roots = self._v2_repair_intent_roots(prompt)
        reference_phrases = [
            phrase
            for phrase in list(current_phrases or []) + list(intent_phrases or [])
            if isinstance(phrase, dict)
        ]
        for item in intent_phrases or []:
            intent_roots |= self._v2_repair_intent_roots(item.get("text", ""))

        def candidate_axes(item):
            if not isinstance(item, dict):
                return set()
            axes = self._v2_phrase_axes(item)
            wanted_axes = item.get("wanted_axes", {})
            if isinstance(wanted_axes, dict):
                axes |= {axis for axis, count in wanted_axes.items() if int(count or 0) > 0}
            axis_counts = item.get("axes", {})
            if isinstance(axis_counts, dict):
                axes |= {axis for axis, count in axis_counts.items() if int(count or 0) > 0}
            return axes

        def add_candidate(text, axes, score, source, cluster=None, memory_entry=None):
            clean = self._v2_clean_phrase_text(text)
            if not clean:
                return
            if not self._v2_prompt_repair_text_allowed(clean):
                return
            words = self._v2_phrase_words(clean)
            if len(words) > 16:
                return
            axes = set(axes or set()) & missing_axes
            if not axes or self._v2_prompt_contains_text(prompt, clean):
                return
            if source != "intent" and not self._v2_repair_candidate_matches_intent(clean, intent_roots):
                return
            if source != "intent" and not self._v2_repair_context_matches_memory(clean, memory_entry, reference_phrases):
                return
            safe_cluster = [
                self._v2_clean_phrase_text(item)
                for item in (cluster or [])
                if (
                    self._v2_prompt_repair_text_allowed(item) and
                    self._v2_repair_candidate_matches_intent(item, intent_roots) and
                    self._v2_repair_context_matches_memory(item, memory_entry, reference_phrases)
                )
            ]
            safe_cluster = [item for item in safe_cluster if item]
            existing = candidates.get(clean)
            payload = {
                "text": clean,
                "axes": axes,
                "score": float(score),
                "source": source,
                "cluster": safe_cluster,
            }
            if existing is None or payload["score"] > existing["score"]:
                candidates[clean] = payload

        for item in intent_phrases or []:
            axes = candidate_axes(item)
            if axes & missing_axes:
                add_candidate(item.get("text", ""), axes, 3.0, "intent")

        if isinstance(intent_family_slot, dict):
            for entry in intent_family_slot.get("intent_preference_phrases", {}).values():
                if not isinstance(entry, dict):
                    continue
                if int(entry.get("seen_count", 0)) < 2:
                    continue
                axes = candidate_axes(entry)
                if not (axes & missing_axes):
                    continue
                score = (
                    1.35 +
                    min(0.75, int(entry.get("seen_count", 0)) * 0.14) +
                    min(0.60, float(entry.get("score", 0.0)) * 0.16)
                )
                add_candidate(
                    entry.get("text", ""),
                    axes,
                    score,
                    "intent_preference",
                    memory_entry=entry,
                )

        if isinstance(previous_run, dict):
            for item in self._v2_concept_units_for_run(previous_run):
                if item.get("kind") == "token":
                    continue
                axes = candidate_axes(item)
                if axes & missing_axes:
                    add_candidate(item.get("text", ""), axes, 1.75, "previous", memory_entry=item)

        preferred_memory = global_state.get("preferred_context_memory", {}) if isinstance(global_state, dict) else {}
        for entry in preferred_memory.values() if isinstance(preferred_memory, dict) else []:
            if not isinstance(entry, dict):
                continue
            axes = candidate_axes(entry)
            if not (axes & missing_axes):
                continue
            cluster = [
                self._v2_clean_phrase_text(item)
                for item in entry.get("phrases", [])
            ]
            cluster = [
                item for item in cluster
                if item and self._v2_prompt_repair_text_allowed(item) and not self._v2_prompt_contains_text(prompt, item)
            ]
            score = (
                2.0 +
                float(entry.get("score", 0.0)) * 0.34 +
                min(1.8, int(entry.get("liked_count", 0) or 0) * 0.42) +
                min(0.8, int(entry.get("satisfied_count", 0) or 0) * 0.16)
            )
            if cluster:
                for offset, item_text in enumerate(cluster[:8]):
                    add_candidate(item_text, axes, score - offset * 0.03, "preferred_context", cluster=cluster[:8], memory_entry=entry)
            else:
                add_candidate(entry.get("anchor", ""), axes, score, "preferred_context", memory_entry=entry)

        memory = global_state.get("phrase_memory", {}) if isinstance(global_state, dict) else {}
        for text, entry in memory.items():
            if not isinstance(entry, dict):
                continue
            axes = candidate_axes(entry)
            wanted_axes = entry.get("wanted_axes", {}) if isinstance(entry.get("wanted_axes", {}), dict) else {}
            axis_hits = sum(int(wanted_axes.get(axis, 0) or 0) for axis in missing_axes)
            evidence = int(entry.get("category_evidence_count", 0) or 0)
            score = (
                1.0 +
                float(entry.get("score", 0.0)) * 0.18 +
                min(1.2, axis_hits * 0.34) +
                min(0.5, evidence * 0.05) +
                min(0.4, int(entry.get("liked_count", 0) or 0) * 0.08) -
                min(1.0, int(entry.get("bad_count", 0) or 0) * 0.20) -
                min(0.9, int(entry.get("wrong_count", 0) or 0) * 0.16)
            )
            if axes & missing_axes and score > 0.60:
                add_candidate(entry.get("text", text), axes, score, "memory", memory_entry=entry)

        current_axes = set()
        for phrase in current_phrases or []:
            current_axes |= candidate_axes(phrase)
        missing_not_represented = missing_axes - current_axes

        source_priority = {
            "intent": 4,
            "intent_preference": 3,
            "previous": 3,
            "preferred_context": 2,
            "memory": 1,
        }
        ranked = sorted(
            candidates.values(),
            key=lambda item: (
                bool(item["axes"] & missing_not_represented),
                source_priority.get(item["source"], 0),
                item["score"],
                len(item["text"]),
            ),
            reverse=True,
        )
        selected = []
        selected_texts = set()
        seen_axes = set()
        for item in ranked:
            cluster = item.get("cluster", []) if item.get("source") == "preferred_context" else []
            for cluster_text in cluster:
                if (
                    cluster_text in selected_texts or
                    self._v2_prompt_contains_text(prompt, cluster_text) or
                    not self._v2_prompt_repair_text_allowed(cluster_text)
                ):
                    continue
                cluster_item = dict(item)
                cluster_item["text"] = cluster_text
                selected.append(cluster_item)
                selected_texts.add(cluster_text)
                seen_axes |= item["axes"]
                if len(selected) >= 10:
                    break
            if len(selected) >= 10:
                break
            if item["text"] in selected_texts:
                continue
            selected.append(item)
            selected_texts.add(item["text"])
            seen_axes |= item["axes"]
            if len(selected) >= 10 or (len(selected) >= 4 and missing_axes <= seen_axes):
                break

        if not selected:
            fallback_axes = set(V2_FEEDBACK_AXES) - missing_axes
            if allow_axis_fallback and fallback_axes:
                fallback_feedback = dict(axis_feedback or {})
                fallback_feedback["missing_axes"] = self._v2_order_axes(fallback_axes)
                fallback_prompt, fallback_status, fallback_candidates = self._v2_repair_prompt_for_missing_axes(
                    prompt,
                    current_phrases,
                    global_state,
                    previous_run,
                    fallback_feedback,
                    intent_phrases=intent_phrases,
                    intent_family_slot=intent_family_slot,
                    allow_axis_fallback=False,
                )
                if fallback_candidates:
                    return (
                        fallback_prompt,
                        "Prompt repair: no candidates for requested axes "
                        f"({', '.join(self._v2_order_axes(missing_axes))}); "
                        f"showing other-axis repair candidates. {fallback_status}",
                        fallback_candidates,
                    )
                return (
                    prompt,
                    "Prompt repair: no stored candidates for requested axes "
                    f"({', '.join(self._v2_order_axes(missing_axes))}); "
                    "no other-axis repair candidates either.",
                    [],
                )
            return prompt, f"Prompt repair: no stored candidates for missing {', '.join(self._v2_order_axes(missing_axes))}.", []

        additions = [item["text"] for item in selected]
        repaired = f"{prompt}, {', '.join(additions)}" if str(prompt or "").strip() else ", ".join(additions)
        source_counts = {}
        for item in selected:
            source_counts[item["source"]] = int(source_counts.get(item["source"], 0)) + 1
        source_summary = ", ".join(f"{source}:{count}" for source, count in sorted(source_counts.items()))
        return (
            repaired,
            f"Prompt repair: added {len(additions)} phrase(s) for missing {', '.join(self._v2_order_axes(missing_axes))} ({source_summary}): {', '.join(additions)}.",
            selected,
        )

    def _v2_apply_perfect_repair_phrases(
        self,
        prompt,
        intent_family_slot,
        intent_prompt="",
        intent_phrases=None,
        clip=None,
        encode_cache=None,
    ):
        if not isinstance(intent_family_slot, dict):
            return prompt, "Perfect repairs: none.", []
        repairs = intent_family_slot.get("perfect_repairs", {})
        if not isinstance(repairs, dict) or not repairs:
            return prompt, "Perfect repairs: none.", []
        rejected = intent_family_slot.get("rejected_repairs", {})
        rejected = rejected if isinstance(rejected, dict) else {}
        additions = []
        for entry in sorted(
            repairs.values(),
            key=lambda item: (
                int(item.get("count", 0)) if isinstance(item, dict) else 0,
                int(item.get("last_seen_iter", 0)) if isinstance(item, dict) else 0,
            ),
            reverse=True,
        ):
            if not isinstance(entry, dict):
                continue
            text = self._v2_clean_phrase_text(entry.get("text", ""))
            if (
                not text or
                self._v2_prompt_contains_text(prompt, text) or
                not self._v2_prompt_repair_text_allowed(text) or
                not self._v2_memory_entry_matches_current_scene(
                    text,
                    intent_prompt or prompt,
                    intent_phrases or [],
                    clip=clip,
                    encode_cache=encode_cache,
                )
            ):
                continue
            rejection = rejected.get(text)
            if (
                isinstance(rejection, dict) and
                int(rejection.get("count", 0) or 0) >= int(entry.get("count", 0) or 0) and
                not self._v2_phrase_represented_by(text, intent_phrases or [])
            ):
                continue
            additions.append(text)
            if len(additions) >= 6:
                break
        if not additions:
            return prompt, "Perfect repairs: already represented.", []
        repaired = f"{prompt}, {', '.join(additions)}" if str(prompt or "").strip() else ", ".join(additions)
        return (
            repaired,
            f"Perfect repairs: preserved {len(additions)} Perfect-proven phrase(s): {', '.join(additions)}.",
            [{"text": text, "source": "perfect_repair", "action": "added"} for text in additions],
        )

    def _v2_serializable_repair_candidates(self, candidates):
        serializable = []
        for item in candidates or []:
            if not isinstance(item, dict):
                continue
            axes = set(item.get("axes", []))
            serializable.append({
                "text": str(item.get("text", "")),
                "axes": self._v2_order_axes(axes),
                "score": round(float(item.get("score", 0.0)), 6),
                "source": str(item.get("source", "")),
                "cluster": [
                    str(text)
                    for text in item.get("cluster", [])
                    if str(text).strip()
                ],
            })
        return serializable

    def _v2_advisor_prompt(self, prompt, intent_prompt, rating_label, axis_feedback, phrases, repair_candidates, previous_run=None, has_image=False):
        axes = ", ".join(self._v2_order_axes(set(axis_feedback.get("missing_axes", [])))) or "none"
        wrong = ", ".join(self._v2_order_axes(set(axis_feedback.get("wrong_axes", [])))) or "none"
        previous_run = previous_run if isinstance(previous_run, dict) else {}
        previous_prompt = self._v2_prompt_key(previous_run.get("prompt", ""))
        previous_encoded = self._v2_prompt_key(previous_run.get("encoded_prompt", ""))
        phrase_summary = "; ".join(
            f"{item.get('text', '')} [{item.get('primary', 'details')}]"
            for item in (phrases or [])[:10]
            if str(item.get("text", "")).strip()
        ) or "none"
        candidate_summary = "; ".join(
            str(item.get("text", ""))
            for item in (repair_candidates or [])[:8]
            if isinstance(item, dict) and str(item.get("text", "")).strip()
        ) or "none"
        return (
            f"<start_of_turn>system\n{V2_PROMPT_ADVISOR_SYSTEM_PROMPT.strip()}<end_of_turn>\n"
            "<start_of_turn>user\n"
            f"Current positive prompt: {self._v2_prompt_key(prompt)}\n"
            f"Original user intent: {self._v2_prompt_key(intent_prompt) or 'same as current prompt'}\n"
            f"Prompt that caused the rating: {previous_prompt or 'none recorded'}\n"
            f"Encoded prompt that caused the rating: {previous_encoded or previous_prompt or 'none recorded'}\n"
            f"Rating for previous output: {rating_label}\n"
            f"Missing axes to repair: {axes}\n"
            f"Wrong axes to avoid repeating: {wrong}\n"
            f"Source image available to advisor: {'yes' if has_image else 'no'}\n"
            f"Current prompt phrase analysis: {phrase_summary}\n"
            f"Rule-based repair candidates already found: {candidate_summary}\n"
            "Task: produce the safest next positive prompt for the current generation. "
            "Keep the prompt unchanged if the rating does not justify a safe repair."
            "<end_of_turn>\n<start_of_turn>model\n"
        )

    def _v2_generate_advisor_text(self, clip, advisor_prompt, seed=None, image=None, thinking=True):
        if clip is None or not hasattr(clip, "generate") or not hasattr(clip, "decode"):
            return "", "Advisor: unavailable; connected CLIP does not expose text generation."
        try:
            try:
                tokens = clip.tokenize(advisor_prompt, image=image, min_length=1, thinking=bool(thinking))
            except TypeError:
                try:
                    tokens = clip.tokenize(advisor_prompt, image=image)
                except TypeError:
                    tokens = clip.tokenize(advisor_prompt)
            generated_ids = clip.generate(
                tokens,
                do_sample=False,
                max_length=384,
                temperature=0.7,
                top_k=64,
                top_p=0.95,
                min_p=0.05,
                repetition_penalty=1.05,
                presence_penalty=0.0,
                seed=seed if seed else None,
            )
            text = clip.decode(generated_ids, skip_special_tokens=True)
            return str(text or "").strip(), "Advisor: generated."
        except Exception as error:
            return "", f"Advisor: generation failed: {error}"

    def _v2_parse_advisor_response(self, text):
        text = re.sub(r"```.*?```", "", str(text or ""), flags=re.DOTALL).strip()
        text = re.sub(r"</?start_of_turn>|<end_of_turn>", "", text).strip()
        diagnostic = ""
        repaired = ""
        diagnostic_match = re.search(r"(?im)^\s*DIAGNOSTIC\s*:\s*(.+)$", text)
        repaired_match = re.search(r"(?ims)^\s*REPAIRED_PROMPT\s*:\s*(.+)$", text)
        if diagnostic_match:
            diagnostic = re.sub(r"\s+", " ", diagnostic_match.group(1)).strip()
        if repaired_match:
            repaired = repaired_match.group(1).strip()
            repaired = re.split(r"(?im)^\s*(?:DIAGNOSTIC|REPAIRED_PROMPT)\s*:", repaired)[0].strip()
        elif text and not diagnostic_match:
            repaired = text.strip()
        repaired = re.sub(r"(?im)^\s*(?:DIAGNOSTIC|REPAIRED_PROMPT)\s*:\s*", "", repaired).strip()
        repaired = re.sub(r"\s+", " ", repaired).strip(" \t\r\n\"'")
        if len(repaired) > 1600:
            repaired = repaired[:1600].rsplit(" ", 1)[0].strip()
        return diagnostic, repaired

    def _v2_validate_advisor_prompt(self, current_prompt, advised_prompt, intent_prompt, intent_phrases, clip=None, encode_cache=None):
        current = self._v2_prompt_key(current_prompt)
        advised = self._v2_prompt_key(advised_prompt)
        intent = self._v2_prompt_key(intent_prompt) or current
        if not advised:
            return False, "empty prompt"
        if self._prompt_looks_like_refusal(advised):
            return False, "refusal text"
        if advised == current:
            return True, "unchanged"
        if len(advised) > max(180, len(current) * 3 + 320):
            return False, "too long"
        body_similarity = self._v2_prompt_body_similarity(advised, intent)
        semantic_similarity = self._v2_text_semantic_similarity(clip, advised, intent, encode_cache=encode_cache) if clip is not None else 0.0
        if body_similarity < 0.30 and semantic_similarity < 0.58:
            return False, "too far from intent"
        generated_phrases = self._ordered_prompt_phrases(advised)
        current_phrases = self._ordered_prompt_phrases(current)
        protected_categories = {"appearance", "subject", "environment"}
        for phrase in generated_phrases[:24]:
            text = phrase.get("text", "")
            if self._v2_phrase_represented_by(text, current_phrases) or self._v2_phrase_represented_by(text, intent_phrases or []):
                continue
            category = self._v2_primary_category_for_text(text)
            if category in protected_categories:
                return False, f"new protected {category} phrase"
        return True, "validated"

    def _v2_prompt_advisor(
        self,
        clip,
        mode,
        current_prompt,
        intent_prompt,
        rating_label,
        axis_feedback,
        phrases,
        repair_candidates,
        previous_run=None,
        intent_phrases=None,
        allow_prompt_change=False,
        seed=0,
        image=None,
        thinking=True,
        encode_cache=None,
    ):
        mode = self._v2_advisor_mode(mode)
        if mode == "Off":
            return current_prompt, "Advisor: off.", "", False
        if not self._v2_prompt_key(current_prompt):
            return current_prompt, "Advisor: skipped; prompt empty.", "", False
        repair_signal = bool(axis_feedback.get("missing_axes") or axis_feedback.get("wrong_axes"))
        if mode == "Repair prompt" and not repair_signal:
            return current_prompt, "Advisor: skipped; no missing/wrong repair signal.", "", False
        advisor_prompt = self._v2_advisor_prompt(
            current_prompt,
            intent_prompt,
            rating_label,
            axis_feedback,
            phrases,
            repair_candidates,
            previous_run=previous_run,
            has_image=image is not None,
        )
        raw, status = self._v2_generate_advisor_text(clip, advisor_prompt, seed=seed, image=image, thinking=thinking)
        if not raw:
            return current_prompt, status, "", False
        diagnostic, advised = self._v2_parse_advisor_response(raw)
        if mode == "Diagnostics" or not allow_prompt_change:
            reason = "diagnostics only" if mode == "Diagnostics" else "prompt changes disabled"
            return current_prompt, f"Advisor: {reason}. {diagnostic or advised or 'No diagnostic returned.'}", diagnostic or advised, False
        valid, reason = self._v2_validate_advisor_prompt(
            current_prompt,
            advised,
            intent_prompt,
            intent_phrases or [],
            clip=clip,
            encode_cache=encode_cache,
        )
        if not valid:
            return current_prompt, f"Advisor: rejected generated repair ({reason}). {diagnostic}".strip(), diagnostic, False
        if self._v2_prompt_key(advised) == self._v2_prompt_key(current_prompt):
            return current_prompt, f"Advisor: generated prompt unchanged. {diagnostic}".strip(), diagnostic, False
        return advised, f"Advisor: applied generated repair. {diagnostic}".strip(), diagnostic, True

    def _v2_scene_builder_category_for_entry(self, entry):
        category = str(entry.get("primary", "") if isinstance(entry, dict) else "").lower()
        if category in self.CATEGORY_DESCRIPTIONS:
            return category
        scores = entry.get("effective_category_scores", entry.get("category_scores", {})) if isinstance(entry, dict) else {}
        if isinstance(scores, dict):
            primary, _ = self._v2_scores_primary(scores)
            return primary
        return self._v2_primary_category_for_text(entry.get("text", "") if isinstance(entry, dict) else entry)

    def _v2_scene_entry_locked(self, item):
        return isinstance(item, dict) and bool(item.get("category_locked"))

    def _v2_sync_scene_builder_memory(self, state, global_state, previous_run, iter_num):
        if not isinstance(state, dict) or not isinstance(previous_run, dict):
            return "Scene Builder sync: no previous run."
        scene_db = state.get("scene_builder")
        if not isinstance(scene_db, dict):
            return "Scene Builder sync: unavailable."
        memory = scene_db.setdefault("universal_memory", {})
        if not isinstance(memory, dict):
            memory = {}
            scene_db["universal_memory"] = memory
        phrase_memory = global_state.get("phrase_memory", {}) if isinstance(global_state, dict) else {}
        timestamp = self._v2_now_iso()
        touched = []
        skipped_locked = 0
        for phrase in self._v2_concept_units_for_run(previous_run):
            text = self._v2_clean_phrase_text(phrase.get("text", ""))
            if not text:
                continue
            key = self._v2_scene_key(text)
            entry = phrase_memory.get(text, phrase)
            category = self._v2_scene_builder_category_for_entry(entry)
            if category not in self.CATEGORY_DESCRIPTIONS:
                category = "details"
            existing = memory.get(key)
            if isinstance(existing, dict) and self._v2_scene_entry_locked(existing):
                existing["count"] = int(existing.get("count", 0) or 0) + 1
                existing["updated_at"] = timestamp
                skipped_locked += 1
                continue
            row = dict(existing or {})
            row["text"] = row.get("text") or text
            row["source"] = "positive"
            row["category"] = category
            row["category_source"] = "refiner"
            row["category_locked"] = False
            row["tokens"] = self._v2_phrase_words(text)
            row["count"] = int(row.get("count", 0) or 0) + 1
            row.setdefault("created_at", timestamp)
            row["updated_at"] = timestamp
            row["wildcard"] = bool(row.get("wildcard"))
            memory[key] = row
            touched.append(text)
            if len(touched) >= 24:
                break
        if not touched and not skipped_locked:
            return "Scene Builder sync: no phrase updates."
        return (
            f"Scene Builder sync: updated {len(touched)} unlocked phrase(s)"
            f"{', skipped ' + str(skipped_locked) + ' locked' if skipped_locked else ''}."
        )

    def _v2_memory_entry_matches_current_scene(self, text, prompt, intent_phrases=None, clip=None, encode_cache=None):
        clean = self._v2_clean_phrase_text(text)
        prompt_clean = self._v2_clean_phrase_text(prompt)
        if not clean:
            return False
        if self._v2_user_intent_prompt_is_vague(prompt_clean):
            return True
        if self._v2_repair_candidate_matches_intent(clean, self._v2_repair_intent_roots(prompt_clean)):
            return True
        if self._v2_phrase_represented_by(clean, intent_phrases or []):
            return True
        if clip is not None and prompt_clean:
            return self._v2_text_semantic_similarity(clip, clean, prompt_clean, encode_cache=encode_cache) >= 0.62
        return False

    def _v2_wildcard_phrase_for_segment(self, segment, wildcard_items):
        segment_clean = self._v2_clean_phrase_text(segment)
        if not segment_clean:
            return ""
        for phrase in wildcard_items:
            if self._v2_phrase_texts_match(segment_clean, phrase):
                return phrase
            if self._v2_prompt_contains_text(segment_clean, phrase) or self._v2_prompt_contains_text(phrase, segment_clean):
                return phrase
        return ""

    def _v2_wildcard_phrases_similar(self, left, right, clip=None, encode_cache=None):
        if self._v2_phrase_texts_match(left, right):
            return True
        left_roots = self._v2_repair_intent_roots(left)
        right_roots = self._v2_repair_intent_roots(right)
        if left_roots and right_roots:
            overlap = left_roots & right_roots
            if overlap and (len(overlap) / float(max(1, min(len(left_roots), len(right_roots))))) >= 0.50:
                return True
        if clip is None:
            return False
        return self._v2_text_semantic_similarity(clip, left, right, encode_cache=encode_cache) >= 0.82

    def _v2_resolve_scene_builder_wildcards(self, prompt, scene_db, clip=None, encode_cache=None):
        if not isinstance(scene_db, dict) or not str(prompt or "").strip():
            return prompt, "Scene Builder wildcard cleanup: unavailable."
        memory = scene_db.get("universal_memory", {})
        if not isinstance(memory, dict):
            return prompt, "Scene Builder wildcard cleanup: unavailable."
        wildcard_items = [
            self._v2_clean_phrase_text(item.get("text", ""))
            for item in memory.values()
            if isinstance(item, dict) and bool(item.get("wildcard"))
        ]
        wildcard_items = [item for item in wildcard_items if item]
        if not wildcard_items:
            return prompt, "Scene Builder wildcard cleanup: none."

        parts = re.split(r"([,;.\n]+)", str(prompt or ""))
        kept_wildcards = []
        removed = []
        for index in range(0, len(parts), 2):
            segment = parts[index].strip()
            phrase = self._v2_wildcard_phrase_for_segment(segment, wildcard_items)
            if not phrase:
                continue
            if any(self._v2_wildcard_phrases_similar(phrase, kept, clip, encode_cache) for kept in kept_wildcards):
                removed.append(segment)
                parts[index] = ""
                continue
            kept_wildcards.append(phrase)
        if not removed:
            return prompt, "Scene Builder wildcard cleanup: no duplicates."
        output = "".join(parts)
        output = re.sub(r"\s*([,;.])\s*([,;.]\s*)+", r"\1 ", output)
        output = re.sub(r"(?:^|[\s,;.])+\n", "\n", output)
        output = re.sub(r"\s+", " ", output.replace("\n", ", ")).strip(" ,;.")
        return output, f"Scene Builder wildcard cleanup: removed {len(removed)} duplicate wildcard phrase(s)."

    def _v2_compose_lucky_prompt(
        self,
        prompt,
        phrases,
        global_state,
        intent_prompt="",
        intent_phrases=None,
        clip=None,
        encode_cache=None,
    ):
        memory = global_state.get("phrase_memory", {})
        scored = []
        for text, entry in memory.items():
            if not isinstance(entry, dict):
                continue
            if not self._v2_auto_inject_entry_allowed(entry, prompt):
                continue
            primary = str(entry.get("primary", "details")).lower()
            if primary in {"action", "camera", "details"} and not self._v2_memory_entry_matches_current_scene(
                entry.get("text", text),
                intent_prompt or prompt,
                intent_phrases,
                clip=clip,
                encode_cache=encode_cache,
            ):
                continue
            score = float(entry.get("score", 0.0))
            score += min(1.0, int(entry.get("liked_count", 0)) * 0.14)
            score += min(1.2, sum(int(v) for v in entry.get("wanted_axes", {}).values()) * 0.10)
            score -= min(1.6, int(entry.get("bad_count", 0)) * 0.22)
            score -= min(1.1, int(entry.get("wrong_count", 0)) * 0.18)
            score -= min(1.8, int(entry.get("auto_inject_blocked_count", 0)) * 0.65)
            if score <= 0.10:
                continue
            scored.append((score, entry.get("primary", "details"), str(entry.get("text", text)).strip()))
        scored = sorted(scored, key=lambda item: (item[0], item[1] == "action"), reverse=True)
        current_texts = [str(item.get("text", "")).strip().lower() for item in phrases]
        selected = []
        seen = set()
        for phrase in current_texts:
            if phrase and phrase not in seen:
                selected.append(phrase)
                seen.add(phrase)
        for _, _, text in scored:
            key = text.lower()
            if not key or key in seen:
                continue
            selected.append(text)
            seen.add(key)
            if len(selected) >= 14:
                break
        if not selected:
            return prompt, "Lucky: on | memory empty, used current prompt."
        lucky_prompt = ", ".join(selected)
        return lucky_prompt, f"Lucky: on | composed {len(selected)} phrase(s): {', '.join(selected[:8])}{'...' if len(selected) > 8 else ''}."

    def _v2_aspect_bucket(self, width, height):
        width = max(1, int(width or 1))
        height = max(1, int(height or 1))
        ratio = width / float(height)
        if ratio >= 1.95:
            return "ultrawide"
        if ratio >= 1.20:
            return "landscape"
        if ratio <= 0.52:
            return "vertical"
        if ratio <= 0.83:
            return "portrait"
        return "square"

    def _v2_image_fingerprint(self, image):
        if not isinstance(image, torch.Tensor) or image.numel() == 0:
            return ""
        try:
            frame = image.detach().float()
            if frame.dim() == 4:
                frame = frame[0]
            if frame.dim() != 3:
                return ""
            if frame.shape[-1] in {1, 3, 4}:
                frame = frame.permute(2, 0, 1).unsqueeze(0)
            else:
                frame = frame.unsqueeze(0)
            sample = F.interpolate(frame[:, :3], size=(8, 8), mode="bilinear", align_corners=False)
            quantized = torch.clamp(sample[0].detach().cpu(), 0.0, 1.0).mul(255).round().to(torch.uint8)
            return md5(quantized.numpy().tobytes()).hexdigest()[:16]
        except Exception:
            return ""

    def _v2_image_metadata(self, source_image, previous_metadata=None):
        if not isinstance(source_image, torch.Tensor) or source_image.numel() == 0:
            return {}, "Vision image: none."
        shape = list(source_image.shape)
        if len(shape) < 3:
            return {}, "Vision image: unsupported shape."
        if len(shape) >= 4:
            batch = int(shape[0])
            height = int(shape[1])
            width = int(shape[2])
        else:
            batch = 1
            height = int(shape[0])
            width = int(shape[1])
        fingerprint = self._v2_image_fingerprint(source_image)
        previous_fingerprint = ""
        if isinstance(previous_metadata, dict):
            previous_fingerprint = str(previous_metadata.get("fingerprint", ""))
        changed = bool(previous_fingerprint and fingerprint and previous_fingerprint != fingerprint)
        metadata = {
            "width": width,
            "height": height,
            "batch": batch,
            "aspect_ratio": round(width / float(max(1, height)), 6),
            "aspect_bucket": self._v2_aspect_bucket(width, height),
            "fingerprint": fingerprint,
            "changed_from_previous": changed,
        }
        changed_text = "changed" if changed else ("same/first" if fingerprint else "unknown")
        return metadata, (
            f"Vision image: {width}x{height} {metadata['aspect_bucket']} "
            f"ratio={metadata['aspect_ratio']:.3f} fingerprint={fingerprint or 'none'} {changed_text}."
        )

    def _v2_clip_vision_summary(self, clip_vision_output):
        tensors = []

        def walk(value, path="root", depth=0):
            if depth > 3 or value is None:
                return
            if isinstance(value, torch.Tensor):
                tensors.append((path, value))
                return
            if isinstance(value, dict):
                for key, item in value.items():
                    walk(item, f"{path}.{key}", depth + 1)
                return
            for key in ("image_embeds", "last_hidden_state", "penultimate_hidden_states", "hidden_states"):
                if hasattr(value, key):
                    walk(getattr(value, key), f"{path}.{key}", depth + 1)

        walk(clip_vision_output)
        summaries = []
        for key, tensor in tensors[:8]:
            try:
                detached = tensor.detach().float().cpu()
                summaries.append({
                    "key": key,
                    "shape": list(detached.shape),
                    "mean": round(float(detached.mean().item()), 6),
                    "std": round(float(detached.std().item()) if detached.numel() > 1 else 0.0, 6),
                    "norm": round(float(detached.norm().item()), 6),
                })
            except Exception:
                continue
        if not summaries:
            return {}, "CLIP Vision: none."
        digest_source = json.dumps(summaries, sort_keys=True)
        summary = {
            "tensor_count": len(tensors),
            "fields": summaries,
            "fingerprint": md5(digest_source.encode("utf-8")).hexdigest()[:16],
        }
        return summary, f"CLIP Vision: {len(tensors)} tensor field(s), fingerprint={summary['fingerprint']}."

    def _v2_update_vision_memory(self, global_state, clip_vision_output=None, source_image=None):
        memory = global_state.setdefault("vision_memory", {})
        previous_image = memory.get("last_image") if isinstance(memory.get("last_image"), dict) else {}
        image_metadata, image_status = self._v2_image_metadata(source_image, previous_image)
        clip_summary, clip_status = self._v2_clip_vision_summary(clip_vision_output)
        current = {
            "image": image_metadata,
            "clip_vision": clip_summary,
        }
        if image_metadata:
            memory["last_image"] = image_metadata
        if clip_summary:
            memory["last_clip_vision"] = clip_summary
        memory["last_context"] = current
        return current, f"{image_status} {clip_status}"

    def _v2_update_negative_prompt_memory(self, global_state, previous_run, rating_profile, axis_feedback=None):
        memory = global_state.setdefault("negative_prompt_memory", {})
        tags = memory.setdefault("tags", {})
        if not isinstance(previous_run, dict) or rating_profile.get("skip_learning"):
            return "Negative repair memory: no update."
        axis_feedback = axis_feedback or self._v2_axis_feedback(rating_profile, global_state.get("last_missing_axes", []))
        wrong_axes = set(axis_feedback.get("wrong_axes", []))
        missing_axes = set(axis_feedback.get("missing_axes", []))
        reward = float(rating_profile.get("reward", 0.0))
        should_store = bool(wrong_axes) or rating_profile.get("key") == "awful" or reward < -0.30
        if not should_store:
            return "Negative repair memory: no poor-rated tags added."

        raw_intent_prompt = self._v2_prompt_key(previous_run.get("intent_prompt", ""))
        has_explicit_intent = bool(
            raw_intent_prompt and
            not previous_run.get("intent_prompt_is_vague") and
            not self._v2_user_intent_prompt_is_vague(raw_intent_prompt)
        )
        intent_prompt = raw_intent_prompt if has_explicit_intent else ""
        intent_phrases = (
            [phrase for phrase in previous_run.get("intent_phrases", []) or [] if isinstance(phrase, dict)]
            if has_explicit_intent else
            []
        )
        family_key, family_slot, _ = self._v2_intent_family_slot(global_state, intent_prompt, create=False)
        added = []
        skipped_intent = 0

        def remember_tag(text, phrase_axes, source, intent_locked=False):
            nonlocal skipped_intent
            text = self._v2_clean_phrase_text(text)
            if not text:
                return
            if intent_locked or self._v2_negative_text_conflicts_with_request(
                text,
                current_prompt=previous_run.get("prompt", "") if has_explicit_intent else "",
                intent_prompt=intent_prompt,
                intent_phrases=intent_phrases,
                family_slot=family_slot,
            ):
                skipped_intent += 1
                return
            if wrong_axes and not (phrase_axes & wrong_axes):
                return
            if not wrong_axes and missing_axes and not (phrase_axes & missing_axes):
                return
            entry = tags.setdefault(text, {
                "text": text,
                "count": 0,
                "axes": {},
                "family_key": family_key,
                "source": source,
                "probe_count": 0,
                "last_rating": "",
                "last_seen_iter": 0,
            })
            entry["count"] = int(entry.get("count", 0)) + (2 if wrong_axes else 1)
            entry["family_key"] = family_key or entry.get("family_key", "")
            entry["source"] = source
            entry["last_rating"] = rating_profile.get("label", "")
            entry["last_seen_iter"] = int(global_state.get("total_iterations", 0)) + 1
            axes = entry.setdefault("axes", {})
            for axis in self._v2_order_axes((phrase_axes & (wrong_axes or missing_axes)) or phrase_axes):
                axes[axis] = int(axes.get(axis, 0)) + 1
            added.append(text)

        for phrase in previous_run.get("phrases", []) or []:
            if not isinstance(phrase, dict):
                continue
            remember_tag(
                phrase.get("text", ""),
                self._v2_phrase_axes(phrase),
                "enhancer_or_prompt",
                intent_locked=has_explicit_intent and bool(phrase.get("intent_locked")),
            )

        for candidate in previous_run.get("repair_candidates", []) or []:
            if not isinstance(candidate, dict):
                continue
            remember_tag(
                candidate.get("text", ""),
                set(candidate.get("axes", [])),
                "repair_candidate",
                intent_locked=False,
            )

        if not added:
            suffix = f" Skipped {skipped_intent} intent-locked tag(s)." if skipped_intent else ""
            return f"Negative repair memory: no matching poor-rated tags.{suffix}"
        memory["tags"] = dict(sorted(
            tags.items(),
            key=lambda item: (int(item[1].get("count", 0)), int(item[1].get("last_seen_iter", 0))),
            reverse=True,
        )[:80])
        suffix = f" Skipped {skipped_intent} intent-locked tag(s)." if skipped_intent else ""
        return f"Negative repair memory: added {len(added)} poor-rated tag(s): {', '.join(added[:8])}.{suffix}"

    def _v2_negative_probe_due(self, text, entry, iter_num):
        if not isinstance(entry, dict):
            return False
        count = int(entry.get("count", 0))
        if count <= 1:
            return False
        digest = int(md5(str(text or "").encode("utf-8")).hexdigest()[:6], 16)
        return (int(iter_num or 0) + digest) % 11 == 0

    def _v2_repair_negative_prompt(
        self,
        negative_prompt,
        global_state,
        axis_feedback=None,
        current_prompt="",
        intent_prompt="",
        intent_phrases=None,
        intent_family_slot=None,
    ):
        prompt = self._v2_prompt_key(negative_prompt)
        memory = global_state.get("negative_prompt_memory", {})
        tags = memory.get("tags", {}) if isinstance(memory, dict) else {}
        if not isinstance(tags, dict) or not tags:
            return prompt, "Negative repair: no stored poor-rated tags."
        wanted_axes = set(axis_feedback.get("missing_axes", [])) | set(axis_feedback.get("wrong_axes", [])) if isinstance(axis_feedback, dict) else set()
        ranked = []
        withheld_intent = 0
        withheld_family = 0
        withheld_probe = 0
        current_family_key = intent_family_slot.get("family_key", "") if isinstance(intent_family_slot, dict) else ""
        for text, entry in tags.items():
            if not isinstance(entry, dict):
                continue
            clean = self._v2_clean_phrase_text(entry.get("text", text))
            if not clean or self._v2_prompt_contains_text(prompt, clean):
                continue
            if self._v2_negative_text_conflicts_with_request(
                clean,
                current_prompt=current_prompt,
                intent_prompt=intent_prompt,
                intent_phrases=intent_phrases or [],
                family_slot=intent_family_slot,
            ):
                withheld_intent += 1
                continue
            entry_family = str(entry.get("family_key", ""))
            if entry_family and current_family_key and entry_family != current_family_key:
                withheld_family += 1
                continue
            if self._v2_negative_probe_due(clean, entry, int(global_state.get("total_iterations", 0)) + 1):
                entry["probe_count"] = int(entry.get("probe_count", 0)) + 1
                withheld_probe += 1
                continue
            axes = set(entry.get("axes", {}).keys()) if isinstance(entry.get("axes"), dict) else set()
            axis_bonus = 1.0 if wanted_axes and axes & wanted_axes else 0.0
            score = float(entry.get("count", 0)) + axis_bonus + min(0.5, int(entry.get("last_seen_iter", 0)) * 0.001)
            ranked.append((score, clean))
        ranked.sort(reverse=True)
        additions = [text for _, text in ranked[:8]]
        if not additions:
            withheld = []
            if withheld_intent:
                withheld.append(f"intent/current {withheld_intent}")
            if withheld_family:
                withheld.append(f"other-family {withheld_family}")
            if withheld_probe:
                withheld.append(f"probe {withheld_probe}")
            detail = f" Withheld: {', '.join(withheld)}." if withheld else ""
            return prompt, f"Negative repair: stored poor-rated tags already present.{detail}"
        repaired = f"{prompt}, {', '.join(additions)}" if prompt else ", ".join(additions)
        withheld = []
        if withheld_intent:
            withheld.append(f"intent/current {withheld_intent}")
        if withheld_family:
            withheld.append(f"other-family {withheld_family}")
        if withheld_probe:
            withheld.append(f"probe {withheld_probe}")
        detail = f" Withheld: {', '.join(withheld)}." if withheld else ""
        return repaired, f"Negative repair: added {len(additions)} persistent poor-rated tag(s): {', '.join(additions)}.{detail}"

    def _v2_lora_type(self, lora_type):
        lora_type = str(lora_type or "general").strip().lower()
        return "action" if lora_type == "concept" else lora_type

    def _v2_lora_words(self, text):
        return {
            word.strip().lower()
            for word in re.split(r"[\s,;:_\\/\-().\[\]{}]+", text or "")
            if self._is_valuable_token(word.strip())
        }

    def _v2_lora_feedback_axes(self, lora_type):
        lora_type = self._v2_lora_type(lora_type)
        if lora_type == "action":
            return {"action"}
        if lora_type == "quality":
            return {"quality"}
        if lora_type == "style":
            return {"quality", "details"}
        if lora_type == "character":
            return {"action", "details"}
        return set(V2_FEEDBACK_AXES)

    def _v2_update_lora_suggestions(self, lora_stack, prompt_history, global_state, phrases, rating_profile, axis_feedback=None):
        if not isinstance(lora_stack, dict) or not lora_stack.get("loras"):
            return "LoRA suggestions: no FunPack LoRA stack connected."
        memory = global_state.setdefault("lora_weight_memory", {})
        phrase_by_category = {}
        all_phrase_words = set()
        for phrase in phrases:
            scores = (
                phrase.get("effective_category_scores", phrase.get("category_scores", {})) or
                {phrase.get("primary", "details"): 1.0}
            )
            primary, confidence = self._v2_scores_primary(scores)
            words = self._v2_lora_words(phrase.get("text", ""))
            all_phrase_words |= words
            phrase_by_category.setdefault(primary, set()).update(words)
            for category, score in scores.items():
                if category != primary and float(score or 0.0) >= max(0.30, confidence * 0.72):
                    phrase_by_category.setdefault(category, set()).update(words)
        suggestions = {}
        parts = []
        axis_feedback = axis_feedback or self._v2_axis_feedback(
            rating_profile,
            global_state.get("last_missing_axes", []),
        )
        missing_axes = set(axis_feedback.get("missing_axes", []))
        satisfied_axes = set(axis_feedback.get("satisfied_axes", []))
        resolved_axes = set(axis_feedback.get("resolved_axes", []))
        reward = float(rating_profile.get("reward", 0.0))
        for entry in lora_stack.get("loras", []):
            name = entry.get("name", "")
            raw_type = entry.get("type", "general")
            lora_type = self._v2_lora_type(raw_type)
            lora_id = entry.get("id") or self._lora_state_id(name, raw_type)
            lora_words = self._v2_lora_words(name)
            relation = 0.20 if lora_type == "general" else 0.0
            if lora_words and all_phrase_words:
                relation = max(relation, len(lora_words & all_phrase_words) / max(1, len(lora_words)))
            if lora_type == "action":
                relation = max(relation, 0.62 if phrase_by_category.get("action") else 0.0)
            elif lora_type == "quality":
                relation = max(relation, 0.62 if phrase_by_category.get("quality") else 0.0)
            elif lora_type == "style":
                relation = max(relation, 0.54 if phrase_by_category.get("style") or phrase_by_category.get("camera") else 0.0)
            elif lora_type == "character":
                relation = max(relation, 0.58 if phrase_by_category.get("subject") or phrase_by_category.get("appearance") else 0.0)

            state = memory.setdefault(lora_id, {"name": name, "type": lora_type, "offset_ratio": 0.0, "iterations": 0})
            offset = float(state.get("offset_ratio", 0.0))
            step = 0.036 if lora_type == "action" else 0.026
            lora_axes = self._v2_lora_feedback_axes(lora_type)
            if rating_profile.get("key") == "like":
                offset += step * max(0.18, relation) * 0.55
            elif rating_profile.get("key") == "awful":
                offset -= step * max(0.22, relation) * 1.20
            elif lora_axes & missing_axes:
                offset += step * max(0.28, relation) * 1.35
            elif lora_axes & resolved_axes:
                offset += step * max(0.18, relation) * 0.62
            elif lora_axes & satisfied_axes:
                offset += step * max(0.12, relation) * 0.18
            else:
                offset += step * reward * max(0.12, relation)
            offset = _clamp(offset, -0.75, 0.55)
            state["offset_ratio"] = round(float(offset), 6)
            state["iterations"] = int(state.get("iterations", 0)) + 1
            state["type"] = lora_type
            state["name"] = name
            base_model = float(entry.get("base_model_weight", entry.get("model_weight", 1.0)))
            model_weight = base_model * (1.0 + offset)
            action = (
                "mute" if model_weight == 0.0 else
                "reduce" if abs(model_weight) < abs(base_model) else
                "hold" if abs(model_weight - base_model) < 1e-6 else
                "boost"
            )
            suggestions[lora_id] = {
                "name": name,
                "type": lora_type,
                "model_weight": model_weight,
                "base_model_weight": base_model,
                "offset_ratio": offset,
                "relation": relation,
                "action": action,
                "rating_label": rating_profile.get("label", ""),
                "rating_range": (
                    "reduce awful"
                    if rating_profile.get("key") == "awful" else
                    "boost " + "+".join(self._v2_order_axes(lora_axes & missing_axes))
                    if lora_axes & missing_axes else
                    "preserve " + "+".join(self._v2_order_axes(lora_axes & resolved_axes))
                    if lora_axes & resolved_axes else
                    rating_profile.get("key", "")
                ),
            }
            parts.append(f"{name}[{lora_type}] rel={relation:.2f} next={model_weight:+.3f}")
        prompt_history["lora_weight_suggestions"] = suggestions
        return "LoRA suggestions: " + " | ".join(parts)

    def refine_v2(self, positive_prompt, clip=None, rating="Missing action", refinement_key="",
                  seed=0, reset_session=False, lora_stack=None, im_feeling_lucky=False, user_intent_prompt="",
                  refinement_key_input="", positive_conditioning=None, clip_vision_output=None,
                  source_image=None, negative_prompt="", mode="Refine", advisor_mode="Off", advisor_thinking=True,
                  advisor_clip=None):
        if seed != 0:
            torch.manual_seed(seed)
            random.seed(seed)

        encode_cache = {}
        linked_refinement_key = str(refinement_key_input or "").strip()
        if linked_refinement_key:
            refinement_key = linked_refinement_key

        rating_profile = normalize_refiner_v2_rating(rating)
        rating_label = rating_profile.get("label", str(rating))
        execution_mode = self._v2_execution_mode(mode)
        learning_mode = execution_mode == "Learning"
        advisor_mode = self._v2_advisor_mode(advisor_mode)
        advisor_clip = advisor_clip if advisor_clip is not None else clip
        state, state_status = self._v2_load_state(refinement_key, reset_session=reset_session)
        scene_db, scene_builder_status = self._v2_scene_builder_db(state)
        global_state = state.setdefault("global", {})
        global_state.setdefault("phrase_memory", {})
        global_state.setdefault("axis_conditioning_memory", {})
        global_state.setdefault("negative_prompt_memory", {})
        global_state.setdefault("lora_weight_memory", {})
        global_state.setdefault("preferred_context_memory", {})
        global_state.setdefault("intent_alignment_memory", {})
        global_state.setdefault("intent_family_memory", {})
        global_state.setdefault("perfect_anchors", {})
        global_state.setdefault("variant_evidence", {})
        global_state.setdefault("intent_preference_phrases", {})
        global_state.setdefault("conditioning_deltas", {})
        global_state.setdefault("active_repair_axes", [])
        global_state.setdefault("vision_memory", {})
        global_state.setdefault("loss_history", [])
        previous_run = state.get("last_run")
        previous_run_refusal = self._v2_run_looks_like_refusal(previous_run)
        has_previous_run = isinstance(previous_run, dict) and not previous_run_refusal
        if previous_run_refusal:
            learning_profile = dict(rating_profile)
            learning_profile["skip_learning"] = True
            learning_profile["refusal_filtered"] = True
        else:
            learning_profile = rating_profile if has_previous_run else dict(V2_RATING_PROFILES["Initial discovery"], label="Initial discovery")
        previous_rating_label = str(global_state.get("last_rating_label", "Initial discovery"))
        previous_missing_axes = (
            list(global_state.get("last_missing_axes", []))
            if previous_rating_label != "Initial discovery" else
            None
        )
        axis_feedback = self._v2_axis_feedback(learning_profile, previous_missing_axes)

        memory_status = self._v2_update_phrase_memory(
            global_state,
            previous_run,
            learning_profile,
            int(global_state.get("total_iterations", 0)) + 1,
            axis_feedback,
        )
        scene_sync_status = self._v2_sync_scene_builder_memory(
            state,
            global_state,
            previous_run,
            int(global_state.get("total_iterations", 0)) + 1,
        )
        intent_family_status, _ = self._v2_update_intent_family_memory(
            global_state,
            previous_run,
            learning_profile,
            int(global_state.get("total_iterations", 0)) + 1,
            axis_feedback,
        )
        negative_memory_status = self._v2_update_negative_prompt_memory(
            global_state,
            previous_run,
            learning_profile,
            axis_feedback,
        )
        intent_learning_status = self._v2_update_intent_alignment_memory(
            global_state,
            previous_run,
            learning_profile,
            int(global_state.get("total_iterations", 0)) + 1,
            axis_feedback,
        )
        memory_status = f"{memory_status}\n{scene_sync_status}\n{intent_family_status}\n{intent_learning_status}"
        self._v2_update_conditioning_memory(global_state, previous_run, learning_profile, axis_feedback)
        if has_previous_run and not learning_profile.get("skip_learning"):
            self._v2_update_streaks(global_state, learning_profile)
        repair_feedback, repair_persistence_status = self._v2_active_repair_feedback(
            global_state,
            axis_feedback,
            learning_profile,
        )

        vision_context, vision_status = self._v2_update_vision_memory(
            global_state,
            clip_vision_output=clip_vision_output,
            source_image=source_image,
        )
        analysis_prompt = self._v2_prompt_key(positive_prompt)
        intent_prompt = self._v2_prompt_key(user_intent_prompt)
        intent_prompt_is_vague = self._v2_user_intent_prompt_is_vague(intent_prompt)
        current_prompt_refusal = self._prompt_looks_like_refusal(analysis_prompt)
        phrases = [] if current_prompt_refusal else self._v2_classify_phrases(
            clip,
            self._ordered_prompt_phrases(analysis_prompt),
            global_state,
            encode_cache=encode_cache,
            scene_db=scene_db,
        )
        intent_phrases = []
        if intent_prompt and not intent_prompt_is_vague and not current_prompt_refusal:
            intent_phrases = self._v2_classify_phrases(
                clip,
                self._ordered_prompt_phrases(intent_prompt),
                global_state,
                encode_cache=encode_cache,
                scene_db=scene_db,
            )
        intent_source_prompt = self._v2_intent_source_prompt(analysis_prompt, intent_prompt, intent_prompt_is_vague)
        current_family_key, current_family_slot, current_family_similarity = self._v2_intent_family_slot(
            global_state,
            intent_source_prompt,
            create=bool(intent_source_prompt and not current_prompt_refusal),
        )
        self._v2_mark_intent_locks(phrases, intent_source_prompt, intent_phrases, current_family_slot)
        self._v2_mark_intent_locks(intent_phrases, intent_source_prompt, intent_phrases, current_family_slot)
        self._v2_mark_semantic_intent_locks(
            clip,
            phrases,
            intent_source_prompt,
            intent_phrases,
            encode_cache=encode_cache,
        )
        refusal_status = ""
        repair_status = "Prompt repair: none."
        repair_candidates = []
        intent_alignment_status = "Intent alignment: no explicit original intent."
        intent_alignment_adjustments = []
        perfect_repair_status = "Perfect repairs: none."
        perfect_repair_adjustments = []
        wildcard_status = "Scene Builder wildcard cleanup: none."
        advisor_status = "Advisor: off."
        advisor_diagnostic = ""
        advisor_applied = False
        if current_prompt_refusal:
            prompt_to_encode = analysis_prompt
            lucky_status = "Prompt refusal filter: enhancer refusal detected; current prompt will not be stored or learned."
            encoded_role = "refusal passthrough"
            refusal_status = "Current prompt refused by enhancer; storage skipped."
        elif learning_mode:
            prompt_to_encode = analysis_prompt
            lucky_status = "Learning mode: observing only; Lucky, intent alignment, prompt repair, and prompt wildcard cleanup skipped."
            intent_alignment_status = "Intent alignment: skipped in Learning mode."
            perfect_repair_status = "Perfect repairs: skipped in Learning mode."
            repair_status = "Prompt repair: skipped in Learning mode."
            wildcard_status = "Scene Builder wildcard cleanup: skipped in Learning mode."
            encoded_role = "learning passthrough"
        elif im_feeling_lucky and clip is not None:
            prompt_to_encode, lucky_status = self._v2_compose_lucky_prompt(
                analysis_prompt,
                phrases,
                global_state,
                intent_prompt=intent_source_prompt,
                intent_phrases=intent_phrases,
                clip=clip,
                encode_cache=encode_cache,
            )
            prompt_to_encode, wildcard_status = self._v2_resolve_scene_builder_wildcards(
                prompt_to_encode,
                scene_db,
                clip=clip,
                encode_cache=encode_cache,
            )
            encoded_role = "lucky prompt"
        else:
            aligned_prompt, intent_alignment_status, intent_alignment_adjustments = self._v2_apply_intent_alignment_memory(
                analysis_prompt,
                phrases,
                intent_prompt,
                intent_phrases,
                global_state,
            )
            aligned_prompt, perfect_repair_status, perfect_repair_adjustments = self._v2_apply_perfect_repair_phrases(
                aligned_prompt,
                current_family_slot,
                intent_prompt=intent_source_prompt,
                intent_phrases=intent_phrases,
                clip=clip,
                encode_cache=encode_cache,
            )
            prompt_to_encode, emphasis_status = self._v2_emphasized_prompt(
                aligned_prompt,
                phrases,
                global_state,
                {**learning_profile, "missing_axes": repair_feedback.get("missing_axes", [])},
            )
            prompt_to_encode, repair_status, repair_candidates = self._v2_repair_prompt_for_missing_axes(
                prompt_to_encode,
                phrases,
                global_state,
                previous_run,
                repair_feedback,
                intent_phrases=intent_phrases,
                intent_family_slot=current_family_slot,
            )
            prompt_to_encode, wildcard_status = self._v2_resolve_scene_builder_wildcards(
                prompt_to_encode,
                scene_db,
                clip=clip,
                encode_cache=encode_cache,
            )
            if im_feeling_lucky and clip is None:
                lucky_status = f"Lucky: unavailable without CLIP | connected CONDITIONING accepted. {emphasis_status}"
            else:
                lucky_status = f"Lucky: off | trained memory only. {emphasis_status}"
            encoded_role = "current prompt"

        if not current_prompt_refusal:
            prompt_to_encode, advisor_status, advisor_diagnostic, advisor_applied = self._v2_prompt_advisor(
                advisor_clip,
                advisor_mode,
                prompt_to_encode,
                intent_source_prompt,
                rating_label,
                repair_feedback,
                phrases,
                repair_candidates,
                previous_run=previous_run,
                intent_phrases=intent_phrases,
                allow_prompt_change=not learning_mode and advisor_mode == "Repair prompt",
                seed=seed,
                image=source_image,
                thinking=advisor_thinking,
                encode_cache=encode_cache,
            )
            if advisor_applied:
                encoded_role = "advisor repaired prompt"

        cond, meta, encode_status, conditioning_owner = self._v2_conditioning_source(
            clip,
            prompt_to_encode,
            positive_conditioning,
            encode_cache=encode_cache,
        )
        fallback_graph = render_refinement_loss_graph(refinement_key, "v2", "clip", 0, 0.0, [])
        if not isinstance(cond, torch.Tensor):
            status = f"ERROR: V2 could not prepare conditioning | {encode_status}"
            training_info = f"Rating: {rating_label}\n{memory_status}\n{lucky_status}\n{encode_status}"
            return ([], status, training_info, fallback_graph, [])

        if learning_mode:
            refined = cond
            adaptation_status = "Adaptation: Learning mode observation only; conditioning vectors passed through unchanged."
            repaired_negative_prompt = str(negative_prompt or "")
            negative_repair_status = "Negative repair: skipped in Learning mode."
        else:
            refined, adaptation_status = self._v2_apply_conditioning_memory(
                cond,
                global_state,
                learning_profile,
                repair_feedback,
                intent_family_slot=current_family_slot,
            )
            repaired_negative_prompt, negative_repair_status = self._v2_repair_negative_prompt(
                negative_prompt,
                global_state,
                repair_feedback,
                current_prompt=prompt_to_encode,
                intent_prompt=intent_source_prompt,
                intent_phrases=intent_phrases,
                intent_family_slot=current_family_slot,
            )
        negative_conditioning = []
        negative_encode_status = "negative prompt empty"
        if repaired_negative_prompt and clip is not None:
            negative_cond, negative_meta, negative_encode_status = self._v2_encode_prompt(
                clip,
                repaired_negative_prompt,
                encode_cache=encode_cache,
            )
            if isinstance(negative_cond, torch.Tensor):
                negative_conditioning = [(negative_cond, negative_meta)]
        elif repaired_negative_prompt:
            negative_encode_status = "negative prompt available but CLIP missing"
        prompt_key = self._v2_prompt_key(analysis_prompt)
        prompt_history = None
        if current_prompt_refusal:
            lora_status = "LoRA suggestions: skipped for prompt-enhancer refusal."
        else:
            prompt_history = state.setdefault("prompt_histories", {}).setdefault(prompt_key, {
                "canonical_prompt": analysis_prompt,
                "history": [],
            })
            lora_status = self._v2_update_lora_suggestions(
                lora_stack,
                prompt_history,
                global_state,
                phrases,
                learning_profile,
                axis_feedback,
            )

        should_record_iteration = bool(has_previous_run or not current_prompt_refusal)
        if should_record_iteration:
            global_state["total_iterations"] = int(global_state.get("total_iterations", 0)) + 1
        else:
            global_state["total_iterations"] = int(global_state.get("total_iterations", 0))
        learning_loss = max(0.02, (1.0 - max(-1.0, min(1.0, float(global_state.get("avg_reward_ema", 0.0))))) * 0.5)
        loss_history = list(global_state.get("loss_history", []))[-511:]
        if should_record_iteration:
            loss_history.append({
                "total_iteration": int(global_state["total_iterations"]),
                "learning_loss": round(float(learning_loss), 6),
                "rating": int(rating_profile.get("legacy_score", 6)),
                "rating_label": rating_label,
                "similarity": 1.0,
                "scheduler_mode": "automatic",
                "mode": "clip",
            })
        global_state["loss_history"] = loss_history

        phrase_preview = "\n".join(
            (
                f"- {item['text']} ["
                f"{item.get('machine_primary', item['primary'])}->{item['primary']}:"
                f"{item['confidence']:.2f}/{item['source']}/"
                f"ctx={item.get('context_source', 'none')}/"
                f"e{int(item.get('category_evidence_count', 0))}]"
            )
            for item in phrases[:10]
        ) or "none"
        category_diagnostics = self._v2_category_diagnostics(phrases)
        training_guidance = self._v2_training_guidance(
            has_previous_run,
            learning_profile,
            axis_feedback,
            phrases,
            memory_status,
            lora_status,
        )
        prompt_preview = re.sub(r"\s+", " ", prompt_to_encode).strip()
        if len(prompt_preview) > 240:
            prompt_preview = prompt_preview[:237].rstrip() + "..."
        state_repair_candidates = self._v2_serializable_repair_candidates(repair_candidates)
        state_intent_alignment_adjustments = [
            {
                "text": str(item.get("text", "")),
                "source": str(item.get("source", "")),
                "action": str(item.get("action", "")),
            }
            for item in list(intent_alignment_adjustments) + list(perfect_repair_adjustments)
            if isinstance(item, dict) and str(item.get("text", "")).strip()
        ]

        if prompt_history is not None:
            prompt_history["canonical_prompt"] = analysis_prompt
            prompt_history.setdefault("history", [])
            prompt_history["history"] = list(prompt_history.get("history", []))[-119:]
            prompt_history["history"].append({
                "iteration": int(global_state["total_iterations"]),
                "rating_label": rating_label if has_previous_run else "Unrated",
                "encoded_role": encoded_role,
                "prompt": prompt_to_encode,
                "negative_prompt": repaired_negative_prompt,
                "phrases": phrases,
                "repair_candidates": state_repair_candidates,
                "vision_context": vision_context,
                "intent_alignment_adjustments": state_intent_alignment_adjustments,
                "advisor": {
                    "mode": advisor_mode,
                    "status": advisor_status,
                    "diagnostic": advisor_diagnostic,
                    "applied": bool(advisor_applied),
                },
            })

        if current_prompt_refusal:
            state["last_run"] = None
        else:
            state["last_run"] = {
                "prompt": analysis_prompt,
                "encoded_prompt": prompt_to_encode,
                "conditioning": tensor_to_serializable(refined.detach().cpu()),
                "source_conditioning": tensor_to_serializable(cond.detach().cpu()),
                "phrases": phrases,
                "intent_prompt": intent_prompt,
                "intent_phrases": intent_phrases,
                "intent_prompt_is_vague": bool(intent_prompt_is_vague),
                "intent_source_prompt": intent_source_prompt,
                "intent_family_key": current_family_key,
                "intent_family_similarity": round(float(current_family_similarity), 6),
                "intent_alignment_adjustments": state_intent_alignment_adjustments,
                "advisor": {
                    "mode": advisor_mode,
                    "status": advisor_status,
                    "diagnostic": advisor_diagnostic,
                    "applied": bool(advisor_applied),
                },
                "repair_candidates": state_repair_candidates,
                "negative_prompt": repaired_negative_prompt,
                "vision_context": vision_context,
                "encoded_role": encoded_role,
                "auto_injected_prompt_additions": [
                    item.get("text", "")
                    for item in state_repair_candidates
                    if isinstance(item, dict) and str(item.get("text", "")).strip()
                ],
                "rating_label": "Unrated",
                "iteration": int(global_state["total_iterations"]),
            }
        state["global"] = global_state
        self._v2_save_state(state, refinement_key)

        loss_graph = render_refinement_loss_graph(
            refinement_key=refinement_key,
            scheduler_mode="v2-auto",
            mode="clip",
            total_iterations=int(global_state["total_iterations"]),
            latest_learning_loss=float(learning_loss),
            points=loss_history[-256:],
        )
        refusal_status_line = f"\n{refusal_status}" if refusal_status else ""
        status = (
            f"V2 {state_status} | {conditioning_owner} | Mode {execution_mode} | Rating {rating_label} | Iter {global_state['total_iterations']} | "
            f"Learning {'trained previous run' if has_previous_run and not learning_profile.get('skip_learning') else 'not applied'}\n"
            f"{adaptation_status}\n"
            f"{training_guidance}\n"
            f"{scene_builder_status}\n"
            f"{repair_persistence_status}\n"
            f"{lucky_status}"
            f"\nIntent family: {current_family_key[:8] if current_family_key else 'none'} sim={current_family_similarity:.2f}"
            f"\n{intent_alignment_status}"
            f"\n{perfect_repair_status}"
            f"\n{repair_status}"
            f"\n{advisor_status}"
            f"\n{wildcard_status}"
            f"\n{negative_repair_status}"
            f"\n{vision_status}"
            f"{refusal_status_line}"
        )
        if previous_run_refusal:
            learning_reason = "previous prompt was an enhancer refusal"
        elif learning_profile.get("skip_learning"):
            learning_reason = "rating skipped learning"
        elif not has_previous_run:
            learning_reason = "waiting for a rated previous run"
        else:
            learning_reason = "rating trained memory"

        training_info = "\n\n".join([
            (
                "Run\n"
                f"Mode: {execution_mode}\n"
                f"Encoded: {encoded_role} | {encode_status}\n"
                f"Rating: {rating_label}\n"
                f"Prompt: {prompt_preview}"
            ),
            (
                "Learning\n"
                f"Target: {'previous run' if has_previous_run else 'none yet'}\n"
                f"Applied: {'yes' if has_previous_run and not learning_profile.get('skip_learning') else 'no'}\n"
                f"Reason: {learning_reason}\n"
                f"{self._v2_axis_feedback_status(axis_feedback)}\n"
                f"{scene_builder_status}\n"
                f"{repair_persistence_status}\n"
                f"{memory_status}\n"
                f"{negative_memory_status}"
            ),
            (
                "Prompt Analysis\n"
                f"{category_diagnostics}\n"
                f"Category compact view:\n{phrase_preview}"
            ),
            (
                "Adaptation\n"
                f"{adaptation_status}\n"
                f"{repair_persistence_status}\n"
                f"{lucky_status}\n"
                f"Intent family: {current_family_key[:8] if current_family_key else 'none'} sim={current_family_similarity:.2f}\n"
                f"{intent_alignment_status}\n"
                f"{perfect_repair_status}\n"
                f"{repair_status}\n"
                f"{advisor_status}\n"
                f"{wildcard_status}\n"
                f"{negative_repair_status}\n"
                f"Negative encode: {negative_encode_status}\n"
                f"{vision_status}"
                f"{refusal_status_line}"
            ),
            (
                "Guidance\n"
                f"{training_guidance}"
            ),
            (
                "LoRA\n"
                f"{lora_status}"
            ),
        ])
        return ([(refined, meta)], status, training_info, loss_graph, negative_conditioning)


class FunPackSaveRefinementLatent:
    CATEGORY = "FunPack/Refinement"
    RETURN_TYPES = ("LATENT", "STRING")
    RETURN_NAMES = ("latent", "status")
    FUNCTION = "save_latent"
    DESCRIPTION = "Saves a latent tensor bundle under a refinement key for FunPack Video Refiner latent refinement."

    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "latent": ("LATENT",),
                "refinement_key": ("STRING", {"default": "my_style_v1", "multiline": False}),
                "mode": (["ltx2", "wan"], {"default": "ltx2"}),
            }
        }

    @classmethod
    def IS_CHANGED(cls, **kwargs):
        return float("nan")

    def save_latent(self, latent, refinement_key, mode):
        if isinstance(latent, dict) and not latent_is_plain_video_tensor(latent):
            FunPackVideoRefiner()._raise_wrong_latent(latent)

        samples = latent_samples(latent)
        if samples is None:
            return (clone_latent(latent), "No latent samples tensor found.")

        FunPackVideoRefiner()._save_latent_reference(latent, refinement_key, mode)
        return (clone_latent(latent), f"Saved latent for key '{refinement_key}' ({mode}) shape={tuple(samples.shape)}")


class FunPackPromptCombiner:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "main_prompt": ("STRING", {
                    "multiline": True,
                    "default": "masterpiece, best quality, detailed background"
                }),
                "delimiter": ("STRING", {
                    "multiline": False,
                    "default": ",",
                }),
            },
            "optional": {
                "prompt1": ("STRING", {"default": "", "multiline": True}),
                "prompt2": ("STRING", {"default": "", "multiline": True}),
                "prompt3": ("STRING", {"default": "", "multiline": True}),
                "prompt4": ("STRING", {"default": "", "multiline": True}),
                "prompt5": ("STRING", {"default": "", "multiline": True}),
            }
        }

    RETURN_TYPES = ("STRING", "STRING", "STRING", "STRING", "STRING", "STRING")
    RETURN_NAMES = ("out1", "out2", "out3", "out4", "out5", "random")
    FUNCTION = "combine"
    CATEGORY = "FunPack"
    OUTPUT_NODE = False

    def combine(self, main_prompt, delimiter=",",
                prompt1="", prompt2="", prompt3="", prompt4="", prompt5=""):

        main = main_prompt.strip()

        def merge(base, delim, addon):
            addon = addon.strip()
            if not addon:
                return base
            if not base:
                return addon
            return f"{base}{delim}{addon}"

        results = []
        for p in (prompt1, prompt2, prompt3, prompt4, prompt5):
            combined = merge(main, delimiter, p)
            results.append(combined)
        
        random_choice = random.choice(results)

        return (*results, random_choice)


class FunPackLorebookEnhancer:
    """
    Injects context from SillyTavern-style lorebook JSON files.
    Always appends activated entries to the END of the prompt.
    Supports multiple lorebooks, constants, selective filtering, probability, etc.
    """
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "prompt": ("STRING", {
                    "multiline": True,
                    "default": "The brave explorer enters an ancient forest temple at twilight, discovering glowing runes on the walls."
                }),
            },
            "optional": {
                "lorebook_1": ("STRING", {"default": "", "multiline": False}),
                "lorebook_2": ("STRING", {"default": "", "multiline": False}),
                "lorebook_3": ("STRING", {"default": "", "multiline": False}),
                "lorebook_4": ("STRING", {"default": "", "multiline": False}),
                "entry_delimiter": ("STRING", {"default": "", "multiline": True}),
                "context_history": ("STRING", {"multiline": True, "default": ""}),
                "scan_depth": ("INT", {
                    "default": 4,
                    "min": 1,
                    "max": 12,
                    "step": 1
                }),
            }
        }

    RETURN_TYPES = ("STRING", "STRING")
    RETURN_NAMES = ("enhanced_prompt", "injected_content")
    FUNCTION = "enhance"
    CATEGORY = "FunPack"
    OUTPUT_NODE = True

    def _match_keys(self, keys, text):
        if not keys:
            return False
        if isinstance(keys, str):
            keys = [k.strip() for k in keys.split(",") if k.strip()]
        text = text.lower()
        for key in keys:
            key = key.strip()
            if not key:
                continue
            if key.startswith("/") and key.endswith("/"):
                try:
                    pattern = re.compile(key[1:-1], re.IGNORECASE)
                    if pattern.search(text):
                        return True
                except:
                    continue
            elif key.lower() in text:
                return True
        return False

    def _match_secondary(self, secs, text, logic):
        if not secs:
            return True
        if isinstance(secs, str):
            secs = [s.strip() for s in secs.split(",") if s.strip()]
        matches = [self._match_keys([s], text) for s in secs]

        if logic == 0:   return any(matches)
        elif logic == 1: return all(matches)
        elif logic == 2: return not any(matches)
        elif logic == 3: return not all(matches)
        return True

    def _process_lorebook(self, path, scan_text, activated):
        if not path or not os.path.exists(path):
            print(f"[Lorebook Enhancer] File not found: {path}")
            return activated

        try:
            with open(path, 'r', encoding='utf-8') as f:
                lorebook = json.load(f)

            entries = lorebook.get("entries", [])
            if isinstance(entries, dict):
                entries = list(entries.values())

            for entry in entries:
                if not entry.get("enabled", True):
                    continue

                is_constant = entry.get("constant", False)

                is_constant = entry.get("constant", False)

                if not is_constant:
                    keys = entry.get("keys", entry.get("key", []))              # Fixed: use "keys" (plural) first
                    if not self._match_keys(keys, scan_text):
                        continue

                if entry.get("selective", False):
                    sec_keys = entry.get("keysecondary", []) or entry.get("secondary_keys", [])
                    logic = entry.get("selectiveLogic", 0)
                    if not self._match_secondary(sec_keys, scan_text, logic):
                        continue

                if entry.get("selective", False):
                    sec_keys = entry.get("keysecondary", []) or entry.get("secondary_keys", [])
                    logic = entry.get("selectiveLogic", 0)
                    if not self._match_secondary(sec_keys, scan_text, logic):
                        continue

                prob = entry.get("extensions", {}).get("probability", 100)
                if random.randint(1, 100) > prob:
                    continue

                activated.append(entry)

        except json.JSONDecodeError as e:
            print(f"[Lorebook Enhancer] JSON decode error in {path}: {str(e)}")
        except Exception as e:
            print(f"[Lorebook Enhancer] Failed to process {path}: {type(e).__name__}: {str(e)}")

        return activated

    def enhance(self, prompt,
                lorebook_1="", lorebook_2="", lorebook_3="", lorebook_4="",
                context_history="", scan_depth=4, entry_delimiter=""):

        full_text = (context_history + "\n" + prompt).lower()
        lines = full_text.splitlines()
        scan_text = "\n".join(lines[-scan_depth:]) if scan_depth > 0 else full_text

        activated = []
        for path in [lorebook_1, lorebook_2, lorebook_3, lorebook_4]:
            if path.strip():
                activated = self._process_lorebook(path.strip(), scan_text, activated)

        if not activated:
            return (prompt, "No lorebook entries were triggered.")

        activated.sort(key=lambda e: e.get("insertion_order", e.get("order", 0)))

        injected = []
        enhanced = prompt.strip()  # clean up any trailing space

        for entry in activated:
            content = entry.get("content", "").strip()
            if not content:
                continue
            prefixed_content = f"{entry_delimiter}{content}" if entry_delimiter else content

            # Always append to the end - this is the requested behavior
            if enhanced:
                enhanced += "\n" + prefixed_content
            else:
                enhanced = prefixed_content

            source = entry.get("comment") or entry.get("name") or f"uid:{entry.get('uid','?')}" or "unnamed"
            injected.append(f"[{source}] {prefixed_content}")

        injected_text = "\n\n".join(injected) if injected else "No content injected"
        return (enhanced, injected_text)

class FunPackPromptEnhancer:
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "user_prompt": ("STRING", {"multiline": True, "default": "A photo of a [subject] in a [setting]. [action]."}),
                "system_prompt": ("STRING", {
                    "multiline": True,
                    "default": "<You are a creative AI assistant tasked with describing videos.\n\nDescribe the video by detailing the following aspects:\n1. The main content and theme of the video.\n2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects.\n3. Actions, events, behaviors temporal relationships, physical movement changes of the objects.\n4. background environment, light, style and atmosphere.\n5. camera angles, movements, and transitions used in the video:"
                }),
                "model_path_type": (["Local Safetensors", "HuggingFace Pretrained"],),
                "model_path": ("STRING", {"multiline": False, "default": "mlabonne/NeuralLlama-3-8B-Instruct-abliterated"}),
                "llm_safetensors_file": (folder_paths.get_filename_list('clip'),),
                "top_p": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.05, "default": 0.75}),
                "top_k": ("INT", {"min": 0, "max": 1000, "step": 1, "default": 40}),
                "temperature": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.01, "default": 0.6}),
                "max_new_tokens": ("INT", {"min": 64, "max": 4096, "step": 64, "default": 512}),
                "repetition_penalty": ("FLOAT", {"min": 0.0, "max": 3.0, "step": 0.01, "default": 1.0}),
            }
        }

    RETURN_TYPES = ("STRING",)
    RETURN_NAMES = ("enhanced_prompt",)
    FUNCTION = "enhance_prompt"
    CATEGORY = "FunPack"

    def enhance_prompt(self, user_prompt, system_prompt, model_path_type, model_path, llm_safetensors_file, top_p, top_k, temperature, max_new_tokens, repetition_penalty):
        llm_model = None
        llm_tokenizer = None
        llm_model_device = "cuda" if torch.cuda.is_available() else "cpu"

        print(f"[FunPackPromptEnhancer] Starting prompt enhancement...")

        try:
            if model_path_type == "HuggingFace Pretrained":
                print(f"[FunPackPromptEnhancer] Loading LLM from HuggingFace pretrained: {model_path}")
                llm_tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
                llm_model = AutoModelForCausalLM.from_pretrained(model_path, ignore_mismatched_sizes=True, trust_remote_code=True)
            elif model_path_type == "Local Safetensors":
                print(f"[FunPackPromptEnhancer] Loading LLM from local safetensors file: {llm_safetensors_file}")
                full_safetensors_path = folder_paths.get_full_path('clip', llm_safetensors_file)

                llm_tokenizer = AutoTokenizer.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", trust_remote_code=True)

                config = AutoConfig.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", trust_remote_code=True)
                model_base = AutoModelForCausalLM.from_config(config, trust_remote_code=True)

                state_dict = load_file(full_safetensors_path, device="cpu")
                model_base.load_state_dict(state_dict, strict=False)
                llm_model = model_base

            llm_model = llm_model.eval().to(torch.bfloat16 if llm_model_device == "cuda" else torch.float32).to(llm_model_device).requires_grad_(False)
            print(f"[FunPackPromptEnhancer] LLM model loaded successfully to {llm_model_device}!")

            # Model detection to apply correct chat template
            llm_tokenizer.chat_template = """{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' %}{% if loop.first %}{{ '<|begin_of_text|>' + content }}{% else %}{{ content }}{% endif %}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' if add_generation_prompt else '' }}"""

            messages = [
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": user_prompt}
            ]

            if llm_tokenizer.pad_token_id is None:
                llm_tokenizer.pad_token = llm_tokenizer.eos_token
                llm_tokenizer.pad_token_id = llm_tokenizer.eos_token_id

            llm_tokens = llm_tokenizer.apply_chat_template(
                messages,
                add_generation_prompt=True,
                return_tensors="pt",
                tokenize=True
            ).to(llm_model_device)

            print("[FunPackPromptEnhancer] Generating enhanced prompt...")
            with torch.no_grad():
                generated_ids = llm_model.generate(
                    **llm_tokens,
                    do_sample=True,
                    top_p=top_p,
                    top_k=top_k,
                    temperature=temperature,
                    max_new_tokens=max_new_tokens,
                    repetition_penalty=repetition_penalty,
                    pad_token_id=llm_tokenizer.pad_token_id
                )

            output_text = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True)
            print(f"[FunPackPromptEnhancer] Enhanced prompt generated: {output_text}")

            return (output_text,)

        except Exception as e:
            print(f"[FunPackPromptEnhancer] Error during prompt enhancement: {e}")
            raise

        finally:
            if llm_model is not None:
                del llm_model
                llm_model = None
            if llm_tokenizer is not None:
                del llm_tokenizer
                llm_tokenizer = None
            if torch.cuda.is_available():
                torch.cuda.empty_cache()
            gc.collect()
            print("[FunPackPromptEnhancer] LLM model and tokenizer unloaded and memory cleared.")


class FunPackStoryWriter:
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "user_prompt": ("STRING", {"multiline": True, "default": "A photo of a [subject] in a [setting]. [action]."}),
                "prompt1": ("STRING", {"multiline": False, "default": ""}),
                "prompt2": ("STRING", {"multiline": False, "default": ""}),
                "prompt3": ("STRING", {"multiline": False, "default": ""}),
                "prompt4": ("STRING", {"multiline": False, "default": ""}),
                "prompt5": ("STRING", {"multiline": False, "default": ""}),
                "story_system_prompt": ("STRING", {
                    "multiline": True,
                    "default": ""
                }),
                "sequence_system_prompt": ("STRING", {
                    "multiline": True,
                    "default": ""
                }),
                "model_path_type": (["Local Safetensors", "HuggingFace Pretrained"],),
                "model_path": ("STRING", {"multiline": False, "default": "mlabonne/NeuralLlama-3-8B-Instruct-abliterated"}),
                "llm_safetensors_file": (folder_paths.get_filename_list('clip'),),
                "prompt_count": ("INT", {"min": 1, "max": 5, "step": 1, "default": 3}),
                "top_p": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.05, "default": 0.75}),
                "top_k": ("INT", {"min": 0, "max": 1000, "step": 1, "default": 40}),
                "min_p": ("FLOAT", {"min": 0.0, "max": 1.0, "step": 0.01, "default": 0.1}),
                "temperature": ("FLOAT", {"min": 0.0, "max": 2.0, "step": 0.01, "default": 0.6}),
                "max_new_tokens": ("INT", {"min": 64, "max": 4096, "step": 64, "default": 512}),
                "repetition_penalty": ("FLOAT", {"min": 0.0, "max": 3.0, "step": 0.01, "default": 1.0}),
                "mode": (["Sequences from story", "Sequences from user prompt"],),
                "vision_input": ("STRING", {"multiline": True, "default": "Put outputs of your VL model here to make the Story Writer aware of the starting image."}),
                "sanity_check": ("BOOLEAN", {"default": True, "label": "Enable Sanity Check"}),
                "sanity_check_system_prompt": ("STRING", {
                    "multiline": True,
                    "default": "Analyze the given sequence and perform a correction, if the sequence does not match the given requirements:\n1. The sequence is related to given user's prompt.\n2. The sequence contains only physically possible actions.\n3. The sequence contains information about characters, their appearances, positioning, actions, camera angle, focus and zoom.\n4. The sequence is fully describing the requested action.\n\nOutput ONLY corrected sequence, or return it unchanged if it matches the requirements. No additional text except for sequence is allowed."
                }),
                "disable_continuity": ("BOOLEAN", {"default": False, "label": "Enable/disable continuity - if enabled, does not provide the history of previously generated sequences when generating new one."}),
                "provide_current_id": ("BOOLEAN", {"default": True, "label": "If true, provides current sequence ID to the model even if continuity is disabled."}),
            }
        }

    RETURN_TYPES = ("STRING","STRING","STRING","STRING","STRING",)
    RETURN_NAMES = ("prompt1","prompt2","prompt3","prompt4","prompt5",)
    FUNCTION = "write_story"
    CATEGORY = "FunPack"

    def write_story(self, user_prompt, prompt1, prompt2, prompt3, prompt4, prompt5, story_system_prompt, sequence_system_prompt, model_path_type, model_path, llm_safetensors_file, prompt_count, top_p, top_k, min_p, temperature, max_new_tokens, repetition_penalty, mode, vision_input, sanity_check, sanity_check_system_prompt, disable_continuity, provide_current_id):
        llm_model = None
        llm_tokenizer = None
        llm_model_device = "cuda" if torch.cuda.is_available() else "cpu"

        print(f"[FunPackPromptEnhancer] Making initial story...")

        try:
            if model_path_type == "HuggingFace Pretrained":
                print(f"[FunPackStoryWriter] Loading LLM from HuggingFace pretrained: {model_path}")
                llm_tokenizer = AutoTokenizer.from_pretrained(model_path, ignore_mismatched_sizes=True, trust_remote_code=True)
                llm_model = AutoModelForCausalLM.from_pretrained(model_path, ignore_mismatched_sizes=True, trust_remote_code=True)
            elif model_path_type == "Local Safetensors":
                print(f"[FunPackStoryWriter] Loading LLM from local safetensors file: {llm_safetensors_file}")
                full_safetensors_path = folder_paths.get_full_path('clip', llm_safetensors_file)

                llm_tokenizer = AutoTokenizer.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", ignore_mismatched_sizes=True, trust_remote_code=True)

                config = AutoConfig.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers", trust_remote_code=True)
                model_base = AutoModelForCausalLM.from_config(config, trust_remote_code=True)

                state_dict = load_file(full_safetensors_path, device="cpu")
                model_base.load_state_dict(state_dict, strict=False)
                llm_model = model_base

            llm_model = llm_model.eval().to(torch.bfloat16 if llm_model_device == "cuda" else torch.float32).to(llm_model_device).requires_grad_(False)
            print(f"[FunPackStoryWriter] LLM model loaded successfully to {llm_model_device}!")

            # Applying correct chat template
            llm_tokenizer.chat_template = """{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' %}{% if loop.first %}{{ '<|begin_of_text|>' + content }}{% else %}{{ content }}{% endif %}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' if add_generation_prompt else '' }}"""

            # Inside write_story method, after model loading and template/pad fix

            outputs = [""] * 5

            prompts = [""] * 5

            prompts[0] = prompt1
            prompts[1] = prompt2
            prompts[2] = prompt3
            prompts[3] = prompt4
            prompts[4] = prompt5

            recommended_loras = None

            # ── Initialize messages ONCE, depending on mode ────────────────────────────
            if mode == "Sequences from story":
                messages = [
                    {"role": "system", "content": story_system_prompt},
                    {"role": "user", "content": user_prompt}
                ]

            # Generate hidden story
                llm_tokens = llm_tokenizer.apply_chat_template(
                    messages, add_generation_prompt=True, return_tensors="pt", tokenize=True
                ).to(llm_model_device)

                print("[FunPackStoryWriter] Generating hidden story...")
                with torch.no_grad():
                    generated_ids = llm_model.generate(
                        **llm_tokens,
                        do_sample=True,
                        top_p=top_p,
                        top_k=top_k,
                        min_p=min_p,
                        temperature=temperature,
                        max_new_tokens=max_new_tokens,
                        repetition_penalty=repetition_penalty,
                        pad_token_id=llm_tokenizer.pad_token_id,
                        eos_token_id=llm_tokenizer.eos_token_id,
                    )

                story = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True).strip()
                print(f"[FunPackStoryWriter] Hidden story: {story[:150]}...")

                messages.append({"role": "assistant", "content": story})

            else:  # "Sequences from user prompt"
                messages = [
                    {"role": "system", "content": sequence_system_prompt},
                    {"role": "user", "content": user_prompt}
                ]

            # ── Now generate sequences — only add new instruction + append output ─────
            for seq_idx in range(prompt_count):
                # Add **only** the fresh sequence instruction each time
                if disable_continuity == True and provide_current_id == False:
                    messages = [
                        {"role": "system", "content": sequence_system_prompt},
                        {"role": "user", "content": user_prompt}
                    ]

                    if prompts[seq_idx]:
                        messages.append({"role": "user", "content": f"""User's request for current scene only: {prompts[seq_idx]}"""})

                elif disable_continuity == True and provide_current_id == True:
                    messages = [
                        {"role": "system", "content": sequence_system_prompt},
                        {"role": "user", "content": f"""Current request ID: {seq_idx+1}\nUser's instruction for all requests: {user_prompt}"""}
                    ]

                    if prompts[seq_idx]:
                        messages.append({"role": "user", "content": f"""User's request for current scene only: {prompts[seq_idx]}"""})

                else:
                    messages = [
                        {"role": "system", "content": sequence_system_prompt},
                        {"role": "user", "content": f"""Total amount of requests in this batch: {prompt_count}\nCurrently generating request ID {seq_idx+1} out of {prompt_count}\nRequests left in queue: {prompt_count - seq_idx - 1}\nUser's instruction for all requests: {user_prompt}"""},
                        {"role": "assistant", "content": f"""History:{chr(10).join([f"ID {i}: {text}" for i, text in enumerate(outputs[:seq_idx])]) if seq_idx > 0 else "No history available."}"""}
                    ]
                    if prompts[seq_idx]:
                        messages.append({"role": "user", "content": f"""User's request for current scene only: {prompts[seq_idx]}"""})

                if vision_input is not None:
                    messages.append({"role": "user", "content": f"""Reference image description (this is the starting image in the video batch): {vision_input}"""})

                llm_tokens = llm_tokenizer.apply_chat_template(
                    messages, add_generation_prompt=True, return_tensors="pt", tokenize=True
                ).to(llm_model_device)

                print(f"[FunPackStoryWriter] Generating ID {seq_idx + 1}/{prompt_count}...")
                with torch.no_grad():
                    generated_ids = llm_model.generate(
                        **llm_tokens,
                        do_sample=True,
                        top_p=top_p,
                        top_k=top_k,
                        min_p=min_p,
                        temperature=temperature,
                        max_new_tokens=max_new_tokens,
                        repetition_penalty=repetition_penalty,
                        pad_token_id=llm_tokenizer.pad_token_id,
                        eos_token_id=llm_tokenizer.eos_token_id,
                    )

                seq_text = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True).strip()

                # Append generated sequence — this is what chains everything
                messages.append({"role": "assistant", "content": seq_text})

                if sanity_check == False:
                    print(f"[FunPackStoryWriter] ID {seq_idx + 1} (sanity check skipped): {seq_text}...")

                # Performing sanity check - comparing sequence text to user's prompt according to rules in sanity check system prompt.
                if sanity_check == True:
                    if mode == "Sequences from story" and disable_continuity == False:
                        sanity_messages = [
                            {"role": "system", "content": sanity_check_system_prompt},
                            {"role": "user", "content": f"""Original story: {story}\n
                            Rules: {story_system_prompt}\n
                            User's instruction: {user_prompt}\n
                             Previous response: {outputs[seq_idx] if seq_idx > 0 else "No history available."}\n
                             Response to validate and correct if rules were broken: {seq_text}"""}
                        ]
                    elif mode == "Sequences from user prompt" and disable_continuity == False:
                        sanity_messages = [
                            {"role": "system", "content": sanity_check_system_prompt},
                            {"role": "user", "content": f"""User's instruction: {user_prompt}\n
                            Rules: {sequence_system_prompt}\n
                             Previous response: {outputs[seq_idx] if seq_idx > 0 else "No history available."}\n
                             Response to validate and correct if rules were broken: {seq_text}"""}
                        ]
                    else:
                        sanity_messages = [
                            {"role": "system", "content": sanity_check_system_prompt},
                            {"role": "user", "content": f"""User's instruction: {user_prompt}
                            Rules: {sequence_system_prompt}
                            Response to validate and correct if rules were broken: {seq_text}"""}
                        ]
                    llm_tokens = llm_tokenizer.apply_chat_template(
                        sanity_messages, add_generation_prompt=True, return_tensors="pt", tokenize=True
                        ).to(llm_model_device)

                    print(f"[FunPackStoryWriter] Performing sanity check on ID {seq_idx + 1}/{prompt_count}...")
                    with torch.no_grad():
                        generated_ids = llm_model.generate(
                            **llm_tokens,
                            do_sample=False,
                            top_p=top_p,
                            top_k=top_k,
                            min_p=min_p,
                            temperature=temperature,
                            max_new_tokens=1024,
                            repetition_penalty=1.05,
                            pad_token_id=llm_tokenizer.pad_token_id,
                            eos_token_id=llm_tokenizer.eos_token_id,
                        )

                    seq_text = llm_tokenizer.decode(generated_ids[0][llm_tokens['input_ids'].shape[1]:], skip_special_tokens=True).strip()
                    print(f"[FunPackStoryWriter] ID {seq_idx + 1} (sanity check performed): {seq_text}...")

                outputs[seq_idx] = seq_text

            return tuple(outputs)

        except Exception as e:
            print(f"[FunPackStoryWriter] Error during prompt enhancement: {e}")
            raise

        finally:
            if llm_model is not None:
                del llm_model
                llm_model = None
            if llm_tokenizer is not None:
                del llm_tokenizer
                llm_tokenizer = None
            if torch.cuda.is_available():
                torch.cuda.empty_cache()
            gc.collect()
            print("[FunPackStoryWriter] LLM model and tokenizer unloaded and memory cleared.")