2727 lines
135 KiB
Python
2727 lines
135 KiB
Python
"""
|
||
XB-ToolBox Llama 节点
|
||
====================
|
||
基于 ComfyUI-llama-cpp_vlm 适配，提供 LLM/VLM 本地推理支持。
|
||
|
||
节点列表：
|
||
- XB_llamaModelLoader : Llama 模型加载器
|
||
- XB_llamaInstruct : Llama 指令推理
|
||
- XB_llamaParameters : Llama 推理参数
|
||
- XB_llamaUnloadModel : Llama 卸载模型
|
||
- XB_llamaCleanStates : Llama 清理状态
|
||
- XB_llamaBBoxes2BBox : BBoxes 转 BBox
|
||
- XB_llamaUnpackCodeBlock : 解包代码块
|
||
- XB_llamaPromptEnhancer : 提示词增强预设
|
||
|
||
A卡适配项 (相比原版):
|
||
- 自动检测 AMD/NVIDIA GPU 并调整显存计算
|
||
- ROCm / HIP 后端 n_gpu_layers 优化分配
|
||
- Vulkan 后端回退支持 (旧 AMD 卡)
|
||
- GPU 信息打印便于调试
|
||
"""
|
||
|
||
import os
|
||
import io
|
||
import gc
|
||
import json
|
||
import base64
|
||
import random
|
||
import urllib.request
|
||
import torch
|
||
|
||
import numpy as np
|
||
from PIL import Image, ImageDraw
|
||
from scipy.ndimage import gaussian_filter
|
||
from .support_llama.cqdm import cqdm
|
||
from .support_llama.gguf_layers import get_layer_count
|
||
from .support_llama.prompt_enhancer_preset import *
|
||
from .support_llama.presets.storyboard import (
|
||
STORYBOARD_SKELETON, MODEL_RULES, STYLE_RULES, CAMERA_LOGIC, LANGUAGE_RULES, build_storyboard_prompt
|
||
)
|
||
|
||
import folder_paths
|
||
import comfy.model_management as mm
|
||
import comfy.utils
|
||
|
||
try:
|
||
from llama_cpp import Llama
|
||
from llama_cpp.llama_chat_format import (
|
||
Llava15ChatHandler, Llava16ChatHandler, MoondreamChatHandler,
|
||
NanoLlavaChatHandler, Llama3VisionAlphaChatHandler, MiniCPMv26ChatHandler
|
||
)
|
||
_HAS_LLAMA_CPP = True
|
||
except Exception as _e:
|
||
# 未安装 llama-cpp-python：本地 LLM 推理不可用，其余节点照常加载
|
||
print(f"[XB-llama] llama-cpp-python 未安装（本地模型不可用，其余节点正常）：{_e}")
|
||
Llama = None
|
||
Llava15ChatHandler = Llava16ChatHandler = MoondreamChatHandler = None
|
||
NanoLlavaChatHandler = Llama3VisionAlphaChatHandler = MiniCPMv26ChatHandler = None
|
||
_HAS_LLAMA_CPP = False
|
||
|
||
# =============================================================================
|
||
# A卡 / N卡 检测工具
|
||
# =============================================================================
|
||
|
||
def is_rocm() -> bool:
|
||
"""检测是否为 AMD ROCm 环境"""
|
||
try:
|
||
return torch.cuda.is_available() and hasattr(torch.version, "hip") and torch.version.hip is not None
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def is_nvidia() -> bool:
|
||
"""检测是否为 NVIDIA CUDA 环境"""
|
||
try:
|
||
return torch.cuda.is_available() and not is_rocm()
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def get_gpu_name() -> str:
|
||
"""获取 GPU 名称"""
|
||
try:
|
||
if torch.cuda.is_available():
|
||
return torch.cuda.get_device_name(0)
|
||
except Exception:
|
||
pass
|
||
return "Unknown"
|
||
|
||
|
||
def get_vram_gb() -> float:
|
||
"""获取 GPU 显存大小 (GB)"""
|
||
try:
|
||
if torch.cuda.is_available():
|
||
return torch.cuda.get_device_properties(0).total_memory / (1024 ** 3)
|
||
except Exception:
|
||
pass
|
||
return 0.0
|
||
|
||
|
||
def get_amd_arch() -> str:
|
||
"""获取 AMD GPU 架构代号"""
|
||
if not is_rocm():
|
||
return ""
|
||
try:
|
||
raw = torch.cuda.get_device_properties(0).gcnArchName
|
||
return raw.split(":")[0] if raw else "unknown"
|
||
except Exception:
|
||
return "unknown"
|
||
|
||
|
||
# AMD 架构显存效率系数: 某些AMD架构的GGML offload效率与N卡不同
|
||
_AMD_VRAM_FACTOR = {
|
||
"gfx1201": 1.55, # RDNA4
|
||
"gfx1200": 1.55,
|
||
"gfx1151": 1.50, # RDNA3.5
|
||
"gfx1150": 1.50,
|
||
"gfx1103": 1.50, # RDNA3
|
||
"gfx1102": 1.50,
|
||
"gfx1101": 1.50,
|
||
"gfx1100": 1.50,
|
||
"gfx1037": 1.65, # RDNA2
|
||
"gfx1036": 1.65,
|
||
"gfx1035": 1.65,
|
||
"gfx1034": 1.65,
|
||
"gfx1032": 1.65,
|
||
"gfx1031": 1.65,
|
||
"gfx1030": 1.65,
|
||
"gfx1012": 1.70, # RDNA1
|
||
"gfx1011": 1.70,
|
||
"gfx1010": 1.70,
|
||
"gfx942": 1.40, # CDNA3 (MI300X)
|
||
"gfx90a": 1.40, # CDNA2
|
||
"gfx908": 1.45, # CDNA
|
||
"gfx906": 1.70, # Vega
|
||
}
|
||
|
||
|
||
def get_vram_factor() -> float:
|
||
"""获取当前 GPU 的显存系数"""
|
||
if is_nvidia():
|
||
return 1.55
|
||
if is_rocm():
|
||
arch = get_amd_arch()
|
||
for k, v in _AMD_VRAM_FACTOR.items():
|
||
if arch.startswith(k):
|
||
return v
|
||
return 1.60 # AMD 默认
|
||
return 1.55 # CPU / fallback
|
||
|
||
|
||
def print_gpu_info():
|
||
"""打印 GPU 信息用于调试"""
|
||
gpu_type = "ROCm (AMD)" if is_rocm() else ("CUDA (NVIDIA)" if is_nvidia() else "CPU")
|
||
gpu_name = get_gpu_name()
|
||
vram = get_vram_gb()
|
||
arch = get_amd_arch() if is_rocm() else ""
|
||
factor = get_vram_factor()
|
||
arch_str = f", arch={arch}" if arch else ""
|
||
print(f"[XB-llama] GPU 检测: {gpu_type}, {gpu_name}, VRAM={vram:.1f}GB{arch_str}, factor={factor}")
|
||
|
||
|
||
# =============================================================================
|
||
# Chat Handler 导入 (兼容不同版本 llama-cpp-python)
|
||
# =============================================================================
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import MTMDChatHandler
|
||
chat_handlers_extra = ["DeepSeek-OCR"]
|
||
_MTMD = True
|
||
except Exception:
|
||
_MTMD = False
|
||
chat_handlers_extra = []
|
||
|
||
chat_handlers = ["None", "LLaVA-1.5", "LLaVA-1.6", "Moondream2", "nanoLLaVA", "llama3-Vision-Alpha", "MiniCPM-v2.6"]
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import Gemma3ChatHandler
|
||
chat_handlers += ["Gemma3"]
|
||
except Exception:
|
||
Gemma3ChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import Gemma4ChatHandler
|
||
chat_handlers += ["Gemma4"]
|
||
except Exception:
|
||
Gemma4ChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import Qwen25VLChatHandler
|
||
chat_handlers += ["Qwen2.5-VL", "MinerU2.5-Pro"]
|
||
except Exception:
|
||
Qwen25VLChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import Qwen3VLChatHandler
|
||
chat_handlers += ["Qwen3-VL", "Qwen3-VL-Thinking"]
|
||
except Exception:
|
||
Qwen3VLChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import Qwen35ChatHandler
|
||
chat_handlers += ["Qwen3.5", "Qwen3.5-Thinking", "Qwen3.6", "Qwen3.6-Thinking"]
|
||
except Exception:
|
||
Qwen35ChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import (GLM46VChatHandler, LFM2VLChatHandler, GLM41VChatHandler)
|
||
chat_handlers += ["GLM-4.6V", "GLM-4.6V-Thinking", "GLM-4.1V-Thinking", "LFM2-VL"]
|
||
except Exception:
|
||
GLM46VChatHandler = None
|
||
LFM2VLChatHandler = None
|
||
GLM41VChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import LFM25VLChatHandler
|
||
chat_handlers += ["LFM2.5-VL"]
|
||
except Exception:
|
||
LFM25VLChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import GraniteDoclingChatHandler
|
||
chat_handlers += ["Granite-Docling"]
|
||
except Exception:
|
||
GraniteDoclingChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import MiniCPMv45ChatHandler
|
||
chat_handlers += ["MiniCPM-v4.5", "MiniCPM-v4.5-Thinking"]
|
||
except Exception:
|
||
MiniCPMv45ChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import MiniCPMv46ChatHandler
|
||
chat_handlers += ["MiniCPM-v4.6", "MiniCPM-v4.6-Thinking"]
|
||
except Exception:
|
||
MiniCPMv46ChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import PaddleOCRChatHandler
|
||
chat_handlers += ["PaddleOCR-VL-1.5"]
|
||
except Exception:
|
||
PaddleOCRChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import Qwen3ASRChatHandler
|
||
chat_handlers += ["Qwen3-ASR"]
|
||
except Exception:
|
||
Qwen3ASRChatHandler = None
|
||
|
||
try:
|
||
from llama_cpp.llama_chat_format import Step3VLChatHandler
|
||
chat_handlers += ["Step3-VL"]
|
||
except Exception:
|
||
Step3VLChatHandler = None
|
||
|
||
chat_handlers += chat_handlers_extra
|
||
|
||
# 未装 llama-cpp-python 时，下拉只保留 None（本地模型不可选）
|
||
if not _HAS_LLAMA_CPP:
|
||
chat_handlers = ["None"]
|
||
|
||
|
||
# =============================================================================
|
||
# AnyType / 存储类
|
||
# =============================================================================
|
||
|
||
class AnyType(str):
|
||
def __ne__(self, __value: object) -> bool:
|
||
return False
|
||
|
||
|
||
class LLAMA_CPP_STORAGE:
|
||
llm = None
|
||
chat_handler = None
|
||
current_config = None
|
||
messages = {}
|
||
sys_prompts = {}
|
||
|
||
@classmethod
|
||
def clean_state(cls, id=-1):
|
||
if id == -1:
|
||
cls.messages.clear()
|
||
cls.sys_prompts.clear()
|
||
else:
|
||
cls.messages.pop(f"{id}", None)
|
||
cls.sys_prompts.pop(f"{id}", None)
|
||
|
||
@classmethod
|
||
def clean(cls, all=False):
|
||
try:
|
||
cls.llm.close()
|
||
except Exception:
|
||
pass
|
||
|
||
try:
|
||
cls.chat_handler._exit_stack.close()
|
||
except Exception:
|
||
pass
|
||
|
||
cls.llm = None
|
||
cls.chat_handler = None
|
||
cls.current_config = None
|
||
if all:
|
||
cls.clean_state()
|
||
|
||
gc.collect()
|
||
mm.soft_empty_cache()
|
||
|
||
@classmethod
|
||
def load_model(cls, config):
|
||
if Llama is None:
|
||
raise RuntimeError(
|
||
"未安装 llama-cpp-python，无法加载本地模型。\n"
|
||
"N卡: https://github.com/JamePeng/llama-cpp-python/releases\n"
|
||
"A卡: 请使用 ROCm/HIP 编译的 llama-cpp-python"
|
||
)
|
||
|
||
def get_chat_handler(chat_handler):
|
||
match chat_handler:
|
||
case "Qwen3.5" | "Qwen3.5-Thinking" | "Qwen3.6" | "Qwen3.6-Thinking":
|
||
return Qwen35ChatHandler
|
||
case "Qwen3-VL" | "Qwen3-VL-Thinking":
|
||
return Qwen3VLChatHandler
|
||
case "Qwen3-ASR":
|
||
return Qwen3ASRChatHandler
|
||
case "Qwen2.5-VL" | "MinerU2.5-Pro":
|
||
return Qwen25VLChatHandler
|
||
case "LLaVA-1.5":
|
||
return Llava15ChatHandler
|
||
case "LLaVA-1.6":
|
||
return Llava16ChatHandler
|
||
case "Moondream2":
|
||
return MoondreamChatHandler
|
||
case "nanoLLaVA":
|
||
return NanoLlavaChatHandler
|
||
case "llama3-Vision-Alpha":
|
||
return Llama3VisionAlphaChatHandler
|
||
case "MiniCPM-v2.6":
|
||
return MiniCPMv26ChatHandler
|
||
case "MiniCPM-v4.5" | "MiniCPM-v4.5-Thinking":
|
||
return MiniCPMv45ChatHandler
|
||
case "MiniCPM-v4.6" | "MiniCPM-v4.6-Thinking":
|
||
return MiniCPMv46ChatHandler
|
||
case "Gemma3":
|
||
return Gemma3ChatHandler
|
||
case "Gemma4":
|
||
return Gemma4ChatHandler
|
||
case "GLM-4.6V" | "GLM-4.6V-Thinking":
|
||
return GLM46VChatHandler
|
||
case "GLM-4.1V-Thinking":
|
||
return GLM41VChatHandler
|
||
case "LFM2-VL":
|
||
return LFM2VLChatHandler
|
||
case "LFM2.5-VL":
|
||
return LFM25VLChatHandler
|
||
case "Granite-Docling":
|
||
return GraniteDoclingChatHandler
|
||
case "DeepSeek-OCR":
|
||
return MTMDChatHandler
|
||
case "PaddleOCR-VL-1.5":
|
||
return PaddleOCRChatHandler
|
||
case "Step3-VL":
|
||
return Step3VLChatHandler
|
||
case "None":
|
||
return None
|
||
case _:
|
||
raise ValueError(f'未知模型类型: "{chat_handler}"')
|
||
|
||
cls.clean(all=True)
|
||
cls.current_config = config.copy()
|
||
model = config["model"]
|
||
mmproj = config["mmproj"]
|
||
chat_handler = config["chat_handler"]
|
||
n_ctx = config["n_ctx"]
|
||
vram_limit = config["vram_limit"]
|
||
image_max_tokens = config["image_max_tokens"]
|
||
image_min_tokens = config["image_min_tokens"]
|
||
n_gpu_layers = -1
|
||
|
||
model_path = os.path.join(folder_paths.models_dir, 'LLM', model)
|
||
handler = get_chat_handler(chat_handler)
|
||
|
||
# A卡/N卡统一的显存感知层数计算
|
||
vram_factor = get_vram_factor()
|
||
if vram_limit != -1:
|
||
gguf_layers = get_layer_count(model_path) or 32
|
||
gguf_size = os.path.getsize(model_path) * vram_factor / (1024 ** 3)
|
||
gguf_layer_size = gguf_size / gguf_layers
|
||
|
||
if mmproj and mmproj != "None":
|
||
mmproj_path = os.path.join(folder_paths.models_dir, 'LLM', mmproj)
|
||
if chat_handler == "None":
|
||
raise ValueError('"chat_handler" 不能为 None! (加载了 mmproj 视觉模块)')
|
||
|
||
if vram_limit != -1:
|
||
mmproj_size = os.path.getsize(mmproj_path) * vram_factor / (1024 ** 3)
|
||
n_gpu_layers = max(1, int((vram_limit - mmproj_size) / gguf_layer_size))
|
||
|
||
print(f"[XB-llama] 加载视觉模块: {mmproj}")
|
||
|
||
think_mode = "Thinking" in chat_handler
|
||
kwargs = {"clip_model_path": mmproj_path, "verbose": False}
|
||
if chat_handler in ["Qwen3-VL", "Qwen3-VL-Thinking"]:
|
||
kwargs["force_reasoning"] = think_mode
|
||
kwargs["image_max_tokens"] = image_max_tokens
|
||
kwargs["image_min_tokens"] = image_min_tokens
|
||
elif chat_handler in ["MiniCPM-v4.5", "GLM-4.6V", "Qwen3.5"]:
|
||
kwargs["enable_thinking"] = think_mode
|
||
|
||
if _MTMD:
|
||
kwargs["image_max_tokens"] = image_max_tokens
|
||
kwargs["image_min_tokens"] = image_min_tokens
|
||
|
||
try:
|
||
cls.chat_handler = handler(**kwargs)
|
||
except Exception as e:
|
||
raise RuntimeError(
|
||
f"{e}\n请更新 llama-cpp-python 版本\n"
|
||
"N卡: https://github.com/JamePeng/llama-cpp-python/releases\n"
|
||
"A卡: 请使用 ROCm/HIP 编译的 llama-cpp-python"
|
||
)
|
||
|
||
else:
|
||
if vram_limit != -1:
|
||
n_gpu_layers = max(1, int(vram_limit / gguf_layer_size))
|
||
if handler is not None:
|
||
cls.chat_handler = handler(verbose=False)
|
||
else:
|
||
cls.chat_handler = None
|
||
|
||
print(f"[XB-llama] 加载模型: {model}")
|
||
print(f"[XB-llama] n_gpu_layers = {n_gpu_layers} (0=仅CPU, -1=全部GPU)")
|
||
cls.llm = Llama(
|
||
model_path,
|
||
chat_handler=cls.chat_handler,
|
||
n_gpu_layers=n_gpu_layers,
|
||
n_ctx=n_ctx,
|
||
verbose=False
|
||
)
|
||
|
||
|
||
any_type = AnyType("*")
|
||
|
||
# 模型卸载钩子
|
||
if not hasattr(mm, "unload_all_models_backup"):
|
||
mm.unload_all_models_backup = mm.unload_all_models
|
||
|
||
def patched_unload_all_models(*args, **kwargs):
|
||
LLAMA_CPP_STORAGE.clean(all=True)
|
||
result = mm.unload_all_models_backup(*args, **kwargs)
|
||
return result
|
||
|
||
mm.unload_all_models = patched_unload_all_models
|
||
print("[XB-llama] 模型卸载钩子已注册!")
|
||
|
||
# LLM 模型文件夹注册
|
||
llm_extensions = ['.ckpt', '.pt', '.bin', '.pth', '.safetensors', '.gguf']
|
||
folder_paths.folder_names_and_paths["LLM"] = ([os.path.join(folder_paths.models_dir, "LLM")], llm_extensions)
|
||
|
||
# 打印 GPU 信息
|
||
print_gpu_info()
|
||
|
||
# =============================================================================
|
||
# 预设提示词
|
||
# =============================================================================
|
||
|
||
preset_prompts = {
|
||
# ── 空 ──
|
||
"Empty - Nothing": "",
|
||
|
||
# ── 描述 [EN] ──
|
||
"Normal - Describe [EN]": "Describe this @.",
|
||
"Prompt Style - Tags [EN]": "Your task is to generate a clean list of comma-separated tags for a text-to-@ AI, based *only* on the visual information in the @. Limit the output to a maximum of 50 unique tags. Strictly describe visual elements like subject, clothing, environment, colors, lighting, and composition. Do not include abstract concepts, interpretations, marketing terms, or technical jargon (e.g., no 'SEO', 'brand-aligned', 'viral potential'). The goal is a concise list of visual descriptors. Avoid repeating tags.",
|
||
"Prompt Style - Simple [EN]": "Analyze the @ and generate a simple, single-sentence text-to-@ prompt. Describe the main subject and the setting concisely.",
|
||
"Prompt Style - Detailed [EN]": "Generate a detailed, artistic text-to-@ prompt based on the @. Combine the subject, their actions, the environment, lighting, and overall mood into a single, cohesive paragraph of about 2-3 sentences. Focus on key visual details.",
|
||
"Prompt Style - Extreme Detailed [EN]": "Generate an extremely detailed and descriptive text-to-@ prompt from the @. Create a rich paragraph that elaborates on the subject's appearance, textures of clothing, specific background elements, the quality and color of light, shadows, and the overall atmosphere. Aim for a highly descriptive and immersive prompt.",
|
||
"Prompt Style - Cinematic [EN]": "Act as a master prompt engineer. Create a highly detailed and evocative prompt for an @ generation AI. Describe the subject, their pose, the environment, the lighting, the mood, and the artistic style (e.g., photorealistic, cinematic, painterly). Weave all elements into a single, natural language paragraph, focusing on visual impact.",
|
||
"Creative - Detailed Analysis [EN]": "Describe this @ in detail, breaking down the subject, attire, accessories, background, and composition into separate sections.",
|
||
"Creative - Summarize Video [EN]": "Summarize the key events and narrative points in this video.",
|
||
"Creative - Short Story [EN]": "Write a short, imaginative story inspired by this @ or video.",
|
||
"Creative - Refine & Expand Prompt [EN]": "Refine and enhance the following user prompt for creative text-to-@ generation. Keep the meaning and keywords, make it more expressive and visually rich. Output **only the improved prompt text itself**, without any reasoning steps, thinking process, or additional commentary.",
|
||
"Vision - *Bounding Box [EN]": 'Locate every instance that belongs to the following categories: "#". Report bbox coordinates in {"bbox_2d": [x1, y1, x2, y2], "label": "string"} JSON format as a List.',
|
||
|
||
# ── 描述 [ZH] ──
|
||
"Normal - 描述 [ZH]": "请描述这张@。",
|
||
"Prompt Style - 标签 [ZH]": "你的任务是基于@中的视觉信息，为文生@AI生成一个干净的逗号分隔标签列表，最多50个独特标签。严格描述视觉元素，如主体、服装、环境、颜色、光照和构图。不要包含抽象概念、解读、营销术语或技术术语。目标是生成简洁的视觉描述列表，避免重复标签。",
|
||
"Prompt Style - 简洁 [ZH]": "分析这张@，生成一个简洁的单句文生@提示词，简明扼要地描述主体和场景。",
|
||
"Prompt Style - 详细 [ZH]": "基于这张@，生成一个详细、富有艺术感的文生@提示词。将主体、动作、环境、光照和整体氛围融合为一段约2-3句的连贯描述，聚焦关键视觉细节。",
|
||
"Prompt Style - 极度详细 [ZH]": "基于这张@，生成一个极度详细和描述性的文生@提示词。用丰富的段落详细描述主体的外观、服装质感、背景元素、光线质量和颜色、阴影以及整体氛围。力求生成高度描述性和沉浸感的提示词。",
|
||
"Prompt Style - 电影级 [ZH]": "作为一名提示词工程大师，为文生@AI创建一个高度详细、富有感染力的提示词。描述主体、姿态、环境、光照、情绪和艺术风格（如写实摄影、电影级、油画风）。将所有元素编织成一段自然流畅的描述，聚焦视觉冲击力。",
|
||
"Creative - 详细分析 [ZH]": "详细描述这张@，将主体、着装、配饰、背景和构图分解为独立的部分进行说明。",
|
||
"Creative - 视频摘要 [ZH]": "总结这段视频中的关键事件和叙事要点。",
|
||
"Creative - 短故事 [ZH]": "为这张@或视频写一个简短、富有想象力的故事。",
|
||
"Creative - 润色扩充提示词 [ZH]": "优化并增强以下用户提示词，用于创意文生@生成。保留原意和关键词，使其更具表现力和视觉丰富度。**仅输出优化后的提示词文本**，不要包含任何推理步骤、思考过程或额外评论。",
|
||
"Vision - BBox检测 [ZH]": '定位所有属于以下类别的实例："#"。以 {"bbox_2d": [x1, y1, x2, y2], "label": "string"} 的JSON列表格式报告边界框坐标。',
|
||
}
|
||
preset_tags = list(preset_prompts.keys())
|
||
|
||
|
||
# =============================================================================
|
||
# 工具函数
|
||
# =============================================================================
|
||
|
||
def image2base64(image):
|
||
img = Image.fromarray(image)
|
||
buffered = io.BytesIO()
|
||
img.save(buffered, format="JPEG", quality=85)
|
||
img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
|
||
return img_base64
|
||
|
||
|
||
def parse_json(json_str):
|
||
json_output = json_str.strip().removeprefix("```json").removesuffix("```")
|
||
try:
|
||
parsed = json.loads(json_output)
|
||
except Exception as e:
|
||
raise ValueError(f"无法解析 JSON 数据!\n{e}")
|
||
return parsed
|
||
|
||
|
||
def scale_image(image: torch.Tensor, max_size: int = 128):
|
||
img_np = np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8)
|
||
img_pil = Image.fromarray(img_np)
|
||
w, h = img_pil.size
|
||
scale = min(max_size / max(w, h), 1.0)
|
||
new_w, new_h = int(w * scale), int(h * scale)
|
||
img_resized = img_pil.resize((new_w, new_h), Image.Resampling.LANCZOS)
|
||
return np.array(img_resized)
|
||
|
||
|
||
def qwen3bbox(image, json_data):
|
||
img = Image.fromarray(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
|
||
bboxes = []
|
||
for item in json_data:
|
||
x0, y0, x1, y1 = item["bbox_2d"]
|
||
size = 1000
|
||
x0 = x0 / size * img.width
|
||
y0 = y0 / size * img.height
|
||
x1 = x1 / size * img.width
|
||
y1 = y1 / size * img.height
|
||
bboxes.append((x0, y0, x1, y1))
|
||
return bboxes
|
||
|
||
|
||
def draw_bbox(image, json_data, mode):
|
||
label_colors = {}
|
||
img = Image.fromarray(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
|
||
draw = ImageDraw.Draw(img)
|
||
|
||
for item in json_data:
|
||
try:
|
||
label = item["label"]
|
||
except Exception:
|
||
try:
|
||
label = item["text_content"]
|
||
except Exception:
|
||
label = "bbox"
|
||
x0, y0, x1, y1 = item["bbox_2d"]
|
||
if mode in ["Qwen3-VL", "Qwen2.5-VL"]:
|
||
size = 1000
|
||
x0 = x0 / size * img.width
|
||
y0 = y0 / size * img.height
|
||
x1 = x1 / size * img.width
|
||
y1 = y1 / size * img.height
|
||
bbox = (x0, y0, x1, y1)
|
||
|
||
if label not in label_colors:
|
||
label_colors[label] = tuple(random.randint(80, 180) for _ in range(3))
|
||
color = label_colors[label]
|
||
draw.rectangle(bbox, outline=color, width=4)
|
||
text_y = max(0, y0 - 10)
|
||
text_size = draw.textbbox((x0, text_y), label)
|
||
draw.rectangle([text_size[0], text_size[1] - 2, text_size[2] + 4, text_size[3] + 2], fill=color)
|
||
draw.text((x0 + 2, text_y), label, fill=(255, 255, 255))
|
||
return torch.from_numpy(np.array(img).astype(np.float32) / 255.0).unsqueeze(0)
|
||
|
||
|
||
# =============================================================================
|
||
# 节点定义
|
||
# =============================================================================
|
||
|
||
class XB_llamaModelLoader:
|
||
"""Llama 模型加载器 - 支持 N卡/A卡"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
all_llms = folder_paths.get_filename_list("LLM")
|
||
model_list = [f for f in all_llms if "mmproj" not in f.lower()]
|
||
mmproj_list = [f for f in all_llms if "mmproj" in f.lower()] or ["None"]
|
||
|
||
return {"required": {
|
||
"model": (model_list,),
|
||
"mmproj": (mmproj_list,),
|
||
"chat_handler": (chat_handlers, {"default": "None"}),
|
||
"n_ctx": ("INT", {
|
||
"default": 8192,
|
||
"min": 1024, "max": 327680, "step": 128,
|
||
"tooltip": "上下文长度上限"
|
||
}),
|
||
"vram_limit": ("INT", {
|
||
"default": -1,
|
||
"min": -1, "max": 1024, "step": 1,
|
||
"tooltip": "显存使用上限(GB), -1=不限制\n参考值, 实际可能略超"
|
||
}),
|
||
"image_min_tokens": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 32}),
|
||
"image_max_tokens": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 32}),
|
||
}}
|
||
|
||
RETURN_TYPES = ("LLAMACPPMODEL",)
|
||
RETURN_NAMES = ("llama_model",)
|
||
FUNCTION = "loadmodel"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def loadmodel(self, model, mmproj, chat_handler, n_ctx, vram_limit, image_min_tokens, image_max_tokens):
|
||
custom_config = {
|
||
"model": model,
|
||
"mmproj": mmproj,
|
||
"chat_handler": chat_handler,
|
||
"n_ctx": n_ctx,
|
||
"vram_limit": vram_limit,
|
||
"image_min_tokens": image_min_tokens,
|
||
"image_max_tokens": image_max_tokens
|
||
}
|
||
if not LLAMA_CPP_STORAGE.llm or LLAMA_CPP_STORAGE.current_config != custom_config:
|
||
print("[XB-llama] 开始加载模型...")
|
||
LLAMA_CPP_STORAGE.load_model(custom_config)
|
||
return (custom_config,)
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# 在线 API 后端 (对接「JZL - 🌐 LLM-API 设置」节点)
|
||
# 设计: 「指令推理」的「llama模型」输入同时接受
|
||
# LLAMACPPMODEL (本地模型配置 dict) 与 STRING (API 配置 JSON)
|
||
# ═══════════════════════════════════════════════════════════════
|
||
|
||
_API_TIMEOUT = 1800
|
||
_API_MAX_IMAGE_SIDE = 1568 # 单图上传时的最长边上限，避免 base64 体积过大
|
||
|
||
|
||
def parse_api_config(value):
|
||
"""把输入解析成 API 配置字典; 不是 API 配置则返回 None"""
|
||
if not isinstance(value, str):
|
||
return None
|
||
text = value.strip()
|
||
if not text.startswith("{"):
|
||
return None
|
||
try:
|
||
cfg = json.loads(text)
|
||
except Exception:
|
||
return None
|
||
if not isinstance(cfg, dict):
|
||
return None
|
||
has_model = str(cfg.get("model") or "").strip()
|
||
has_url = str(cfg.get("base_url") or "").strip()
|
||
return cfg if (has_model or has_url) else None
|
||
|
||
|
||
def _split_data_url(url):
|
||
"""把 data:image/jpeg;base64,XXXX 拆成 (mime, base64)"""
|
||
if not isinstance(url, str) or not url.startswith("data:"):
|
||
return "image/jpeg", ""
|
||
try:
|
||
head, b64 = url.split(",", 1)
|
||
except ValueError:
|
||
return "image/jpeg", ""
|
||
mime = head[5:].split(";")[0] or "image/jpeg"
|
||
return mime, b64
|
||
|
||
|
||
def _iter_content_parts(content):
|
||
"""把 OpenAI 风格 content 归一化: 产出 ("text", str) 或 ("image", mime, b64)"""
|
||
if isinstance(content, str):
|
||
if content:
|
||
yield ("text", content)
|
||
return
|
||
if isinstance(content, list):
|
||
for item in content:
|
||
if isinstance(item, str):
|
||
yield ("text", item)
|
||
elif isinstance(item, dict):
|
||
t = item.get("type")
|
||
if t == "text":
|
||
yield ("text", str(item.get("text", "")))
|
||
elif t == "image_url":
|
||
url = item.get("image_url") or ""
|
||
if isinstance(url, dict):
|
||
url = url.get("url", "")
|
||
mime, b64 = _split_data_url(url)
|
||
if b64:
|
||
yield ("image", mime, b64)
|
||
else:
|
||
yield ("text", "[图片加载失败]")
|
||
return
|
||
if content:
|
||
yield ("text", str(content))
|
||
|
||
|
||
def _content_to_text(content):
|
||
"""把 OpenAI 风格的 content 压平成纯文本 (图片记作占位符)"""
|
||
out = []
|
||
for part in _iter_content_parts(content):
|
||
out.append(part[1] if part[0] == "text" else "[图片]")
|
||
return "\n".join(out)
|
||
|
||
|
||
def _http_post_json(url, payload, headers):
|
||
req = urllib.request.Request(
|
||
url,
|
||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||
method="POST",
|
||
headers={"Content-Type": "application/json", **headers},
|
||
)
|
||
with urllib.request.urlopen(req, timeout=_API_TIMEOUT) as resp:
|
||
return json.loads(resp.read().decode("utf-8"))
|
||
|
||
|
||
def _api_call_openai(base_url, api_key, model, messages, temperature, max_tokens, thinking=None):
|
||
"""OpenAI 兼容 (OpenAI/DeepSeek/Qwen/GLM/Kimi/Ollama/vLLM/LM Studio)"""
|
||
url = (base_url or "https://api.openai.com/v1").rstrip("/") + "/chat/completions"
|
||
payload = {
|
||
"model": model,
|
||
"messages": messages,
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
}
|
||
if thinking in ("enabled", "disabled"):
|
||
payload["thinking"] = {"type": thinking}
|
||
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
|
||
data = _http_post_json(url, payload, headers)
|
||
if not data.get("choices"):
|
||
raise RuntimeError(f"[API 错误] 响应无 choices: {json.dumps(data, ensure_ascii=False)[:500]}")
|
||
msg = data["choices"][0].get("message", {}) or {}
|
||
return msg.get("content") or msg.get("reasoning_content") or ""
|
||
|
||
|
||
def _api_call_anthropic(api_key, model, messages, temperature, max_tokens):
|
||
system, chat = "", []
|
||
for m in messages:
|
||
parts = []
|
||
for part in _iter_content_parts(m.get("content", "")):
|
||
if part[0] == "text":
|
||
parts.append({"type": "text", "text": part[1]})
|
||
else:
|
||
parts.append({"type": "image", "source": {
|
||
"type": "base64", "media_type": part[1], "data": part[2]}})
|
||
if m["role"] == "system":
|
||
system = "\n".join(p["text"] for p in parts if p["type"] == "text")
|
||
else:
|
||
chat.append({"role": m["role"], "content": parts})
|
||
data = _http_post_json(
|
||
"https://api.anthropic.com/v1/messages",
|
||
{"model": model, "max_tokens": max_tokens, "temperature": temperature,
|
||
"system": system, "messages": chat},
|
||
{"x-api-key": api_key, "anthropic-version": "2023-06-01"},
|
||
)
|
||
parts = [b.get("text", "") for b in data.get("content", [])
|
||
if isinstance(b, dict) and b.get("type") == "text"]
|
||
if not parts:
|
||
raise RuntimeError(f"[API 错误] 响应无文本: {json.dumps(data, ensure_ascii=False)[:500]}")
|
||
return "".join(parts)
|
||
|
||
|
||
def _api_call_gemini(api_key, model, messages, temperature, max_tokens):
|
||
system_parts, contents = [], []
|
||
for m in messages:
|
||
parts = []
|
||
for part in _iter_content_parts(m.get("content", "")):
|
||
if part[0] == "text":
|
||
parts.append({"text": part[1]})
|
||
else:
|
||
parts.append({"inline_data": {"mime_type": part[1], "data": part[2]}})
|
||
if m["role"] == "system":
|
||
system_parts.extend(p for p in parts if "text" in p)
|
||
else:
|
||
contents.append({"role": "user" if m["role"] == "user" else "model", "parts": parts})
|
||
if not contents:
|
||
contents = [{"role": "user", "parts": [{"text": ""}]}]
|
||
payload = {
|
||
"contents": contents,
|
||
"generationConfig": {"temperature": temperature, "maxOutputTokens": max_tokens},
|
||
}
|
||
if system_parts:
|
||
payload["systemInstruction"] = {"parts": system_parts}
|
||
url = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
|
||
data = _http_post_json(url, payload, {})
|
||
try:
|
||
return data["candidates"][0]["content"]["parts"][0]["text"]
|
||
except (KeyError, IndexError, TypeError):
|
||
raise RuntimeError(f"[API 错误] 响应格式异常: {json.dumps(data, ensure_ascii=False)[:500]}")
|
||
|
||
|
||
def api_chat_completion(cfg, messages):
|
||
"""统一入口: 按 provider 分发, 返回纯文本"""
|
||
provider = str(cfg.get("provider") or "")
|
||
model = str(cfg.get("model") or "").strip()
|
||
api_key = str(cfg.get("api_key") or "").strip()
|
||
base_url = str(cfg.get("base_url") or "").strip()
|
||
temperature = cfg.get("temperature")
|
||
max_tokens = cfg.get("max_tokens")
|
||
temperature = 0.6 if temperature is None else temperature
|
||
max_tokens = 8192 if max_tokens is None else max_tokens
|
||
if not model:
|
||
raise ValueError(
|
||
"[XB-llama] API 配置缺少模型名\n"
|
||
"请在「JZL - 🌐 LLM-API 设置」弹窗中填写模型并保存"
|
||
)
|
||
if "Anthropic" in provider:
|
||
return _api_call_anthropic(api_key, model, messages, temperature, max_tokens)
|
||
if "Gemini" in provider:
|
||
return _api_call_gemini(api_key, model, messages, temperature, max_tokens)
|
||
return _api_call_openai(base_url, api_key, model, messages, temperature, max_tokens, cfg.get("thinking"))
|
||
|
||
|
||
class XB_llamaInstruct:
|
||
"""Llama 指令推理节点 (本地模型 / 在线 API 双后端)"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
# 用 "*" (any) 而不是 "LLAMACPPMODEL,STRING" + forceInput:
|
||
# forceInput 会让前端往 widgets_values 里多塞一个占位值, 导致节点参数整体错位一格
|
||
"llama_model": (any_type, {
|
||
"tooltip": "接「XB-llama - 📦 模型加载器」→ 本地模型\n"
|
||
"接「JZL - 🌐 LLM-API 设置」→ 在线 API\n"
|
||
"两种后端随时切换，插哪边用哪边"
|
||
}),
|
||
"preset_prompt": (preset_tags, {"default": preset_tags[1]}),
|
||
"custom_prompt": ("STRING", {"default": "", "multiline": True, "placeholder": '用户提示词\n\n带 "*" 的预设提示中用作占位符 (如BBox检测中的目标名称)\n否则将覆盖预设提示词'}),
|
||
"system_prompt": ("STRING", {"multiline": True, "default": ""}),
|
||
"inference_mode": (["one by one", "images", "video"], {
|
||
"default": "one by one",
|
||
"tooltip": "one by one: 逐张读取\nimages: 一次性读取所有图片\nvideo: 将输入图像视为视频帧"
|
||
}),
|
||
"max_frames": ("INT", {
|
||
"default": 24, "min": 2, "max": 1024, "step": 1,
|
||
"tooltip": '从输入视频中均匀采样的帧数 (仅 "video" 模式)'
|
||
}),
|
||
"max_size": ("INT", {
|
||
"default": 256, "min": 128, "max": 16384, "step": 64,
|
||
"tooltip": '"images" 和 "video" 模式下输入图像的最大尺寸'
|
||
}),
|
||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "step": 1}),
|
||
"force_offload": ("BOOLEAN", {
|
||
"default": False,
|
||
"tooltip": "推理后卸载模型以释放显存"
|
||
}),
|
||
"save_states": ("BOOLEAN", {
|
||
"default": False,
|
||
"tooltip": "在内存中保留此对话的上下文"
|
||
}),
|
||
},
|
||
"hidden": {
|
||
"unique_id": "UNIQUE_ID",
|
||
},
|
||
"optional": {
|
||
"parameters": ("LLAMACPPARAMS",),
|
||
"images": ("IMAGE",),
|
||
"queue_handler": (any_type, {"tooltip": "用于控制 instruct 节点的执行顺序"}),
|
||
},
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING", "STRING", "INT")
|
||
RETURN_NAMES = ("output", "output_list", "state_uid")
|
||
OUTPUT_IS_LIST = (False, True, False)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def sanitize_messages(self, messages):
|
||
clean_messages = messages.copy()
|
||
for msg in clean_messages:
|
||
content = msg.get("content")
|
||
if isinstance(content, list):
|
||
for item in content:
|
||
if isinstance(item, dict) and item.get("type") == "image_url":
|
||
item["image_url"]["url"] = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADElEQVQImWP4//8/AAX+Av5Y8msOAAAAAElFTkSuQmCC"
|
||
return clean_messages
|
||
|
||
@staticmethod
|
||
def _timed_completion(llm, label, **kwargs):
|
||
"""包装 create_chat_completion, 计时并打印 T/s"""
|
||
import time
|
||
t0 = time.perf_counter()
|
||
output = llm.create_chat_completion(**kwargs)
|
||
t1 = time.perf_counter()
|
||
elapsed = t1 - t0
|
||
try:
|
||
tokens = output.get("usage", {}).get("completion_tokens", 0)
|
||
except Exception:
|
||
tokens = 0
|
||
tps = tokens / elapsed if elapsed > 0 else 0
|
||
print(f"[XB-llama] {label} → {tokens} tokens / {elapsed:.2f}s = {tps:.1f} T/s")
|
||
return output
|
||
|
||
@staticmethod
|
||
def _timed_api_completion(api_cfg, label, messages):
|
||
"""在线 API 推理, 返回与 llama-cpp-python 一致的响应结构"""
|
||
import time
|
||
t0 = time.perf_counter()
|
||
text = api_chat_completion(api_cfg, messages)
|
||
elapsed = time.perf_counter() - t0
|
||
model = api_cfg.get("model", "")
|
||
print(f"[XB-llama] {label} (在线API: {model}) → {elapsed:.2f}s")
|
||
return {"choices": [{"message": {"content": text}}]}
|
||
|
||
def process(self, llama_model, preset_prompt, custom_prompt, system_prompt, inference_mode, max_frames, max_size, seed, force_offload, save_states, unique_id, parameters=None, images=None, queue_handler=None):
|
||
# ── 后端判定: dict = 本地模型; JSON 字符串 = 在线 API ──
|
||
api_cfg = parse_api_config(llama_model)
|
||
is_api = api_cfg is not None
|
||
|
||
if is_api:
|
||
print(f"[XB-llama] 在线 API 后端: {api_cfg.get('provider', '')} / {api_cfg.get('model', '')}")
|
||
else:
|
||
if not isinstance(llama_model, (dict, list)):
|
||
raise ValueError(
|
||
"[XB-llama] 「llama模型」输入无法识别!\n"
|
||
" 接「XB-llama - 📦 模型加载器」= 本地模型\n"
|
||
" 接「JZL - 🌐 LLM-API 设置」= 在线 API"
|
||
)
|
||
if not LLAMA_CPP_STORAGE.llm:
|
||
LLAMA_CPP_STORAGE.load_model(llama_model)
|
||
|
||
if parameters is None:
|
||
parameters = {}
|
||
|
||
if _MTMD and not is_api:
|
||
parameters.pop("present_penalty", None)
|
||
|
||
_uid = parameters.get("state_uid", None)
|
||
_parameters = parameters.copy()
|
||
_parameters.pop("state_uid", None)
|
||
uid = unique_id.rpartition('.')[-1] if _uid in (None, -1) else _uid
|
||
|
||
last_sys_prompt = LLAMA_CPP_STORAGE.sys_prompts.get(f"{uid}", None)
|
||
video_input = inference_mode == "video"
|
||
system_prompts = "请将输入的图片序列当做视频而不是静态帧序列, " + system_prompt if video_input else system_prompt
|
||
if last_sys_prompt != system_prompts:
|
||
messages = []
|
||
LLAMA_CPP_STORAGE.clean_state()
|
||
LLAMA_CPP_STORAGE.sys_prompts[f"{uid}"] = system_prompts
|
||
if system_prompts.strip():
|
||
messages.append({"role": "system", "content": system_prompts})
|
||
else:
|
||
if save_states:
|
||
try:
|
||
print(f"[XB-llama] 加载状态和历史 id={uid}...")
|
||
messages = LLAMA_CPP_STORAGE.messages.get(f"{uid}", [])
|
||
except Exception:
|
||
messages = []
|
||
else:
|
||
messages = []
|
||
|
||
out1 = ""
|
||
out2 = []
|
||
user_content = []
|
||
if custom_prompt.strip() and "*" not in preset_prompt:
|
||
user_content.append({"type": "text", "text": custom_prompt})
|
||
else:
|
||
p = preset_prompts[preset_prompt].replace("#", custom_prompt.strip()).replace("@", "video" if video_input else "image")
|
||
user_content.append({"type": "text", "text": p})
|
||
|
||
if images is not None:
|
||
frames = images
|
||
if video_input:
|
||
indices = np.linspace(0, len(images) - 1, max_frames, dtype=int)
|
||
frames = [images[i] for i in indices]
|
||
|
||
if is_api:
|
||
# ── 在线 API 视觉推理 ──
|
||
if inference_mode == "one by one":
|
||
tmp_list = []
|
||
image_content = {"type": "image_url", "image_url": {"url": ""}}
|
||
user_content.append(image_content)
|
||
messages.append({"role": "user", "content": user_content})
|
||
print(f"[XB-llama] 开始处理 {len(frames)} 张图像 (在线API)")
|
||
|
||
for i, image in enumerate(cqdm(frames)):
|
||
if mm.processing_interrupted():
|
||
raise mm.InterruptProcessingException()
|
||
data = image2base64(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
|
||
for item in user_content:
|
||
if item.get("type") == "image_url":
|
||
item["image_url"]["url"] = f"data:image/jpeg;base64,{data}"
|
||
break
|
||
output = self._timed_api_completion(api_cfg, f"one by one {i + 1}/{len(frames)}", messages=messages)
|
||
text = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out2.append(text)
|
||
if len(frames) > 1:
|
||
tmp_list.append(f"====== Image {i + 1} ======")
|
||
tmp_list.append(text)
|
||
|
||
out1 = "\n\n".join(tmp_list)
|
||
else:
|
||
for image in frames:
|
||
if len(frames) > 1:
|
||
data = image2base64(scale_image(image, max_size))
|
||
else:
|
||
data = image2base64(scale_image(image, _API_MAX_IMAGE_SIDE))
|
||
user_content.append({
|
||
"type": "image_url",
|
||
"image_url": {"url": f"data:image/jpeg;base64,{data}"},
|
||
})
|
||
|
||
messages.append({"role": "user", "content": user_content})
|
||
output = self._timed_api_completion(api_cfg, f"{inference_mode} mode", messages=messages)
|
||
out1 = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out2 = [line for line in out1.split('\n') if line.strip()]
|
||
else:
|
||
# ── 本地模型视觉推理 ──
|
||
_ch = LLAMA_CPP_STORAGE.chat_handler
|
||
if _ch is None:
|
||
raise ValueError(
|
||
"检测到图像输入, 但 chat_handler=None (纯文本模型)!\n"
|
||
"请选择支持视觉的 chat_handler, 例如 Qwen3-VL / Qwen3.5 / MiniCPM-v4.6 等"
|
||
)
|
||
# LLaVA系 handler 有 clip_model_path 属性, 检查是否为空
|
||
if hasattr(_ch, "clip_model_path") and _ch.clip_model_path is None:
|
||
raise ValueError(
|
||
"检测到图像输入, 但 mmproj 未正确加载!\n"
|
||
"请确认在模型加载器中选择了正确的 mmproj 文件"
|
||
)
|
||
# Qwen3.5 等新 handler 不暴露 clip_model_path, 直接放行
|
||
# (如果模型实际不支持图像, llama-cpp-python 会自己报错)
|
||
|
||
if inference_mode == "one by one":
|
||
tmp_list = []
|
||
image_content = {"type": "image_url", "image_url": {"url": ""}}
|
||
user_content.append(image_content)
|
||
messages.append({"role": "user", "content": user_content})
|
||
print(f"[XB-llama] 开始处理 {len(frames)} 张图像")
|
||
|
||
import time
|
||
_total_tokens = 0
|
||
_t0 = time.perf_counter()
|
||
for i, image in enumerate(cqdm(frames)):
|
||
if mm.processing_interrupted():
|
||
raise mm.InterruptProcessingException()
|
||
data = image2base64(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
|
||
for item in user_content:
|
||
if item.get("type") == "image_url":
|
||
item["image_url"]["url"] = f"data:image/jpeg;base64,{data}"
|
||
break
|
||
output = LLAMA_CPP_STORAGE.llm.create_chat_completion(messages=messages, seed=seed, **_parameters)
|
||
text = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out2.append(text)
|
||
if len(frames) > 1:
|
||
tmp_list.append(f"====== Image {i + 1} ======")
|
||
tmp_list.append(text)
|
||
try:
|
||
_total_tokens += output.get("usage", {}).get("completion_tokens", 0)
|
||
except Exception:
|
||
pass
|
||
_elapsed = time.perf_counter() - _t0
|
||
_tps = _total_tokens / _elapsed if _elapsed > 0 else 0
|
||
print(f"[XB-llama] one by one 完成 → {_total_tokens} tokens / {_elapsed:.2f}s = {_tps:.1f} T/s")
|
||
|
||
out1 = "\n\n".join(tmp_list)
|
||
else:
|
||
for image in frames:
|
||
if len(frames) > 1:
|
||
data = image2base64(scale_image(image, max_size))
|
||
else:
|
||
data = image2base64(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
|
||
image_content = {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{data}"}}
|
||
user_content.append(image_content)
|
||
|
||
messages.append({"role": "user", "content": user_content})
|
||
output = self._timed_completion(LLAMA_CPP_STORAGE.llm, f"{inference_mode} mode", messages=messages, seed=seed, **_parameters)
|
||
out1 = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out2 = [line for line in out1.split('\n') if line.strip()]
|
||
else:
|
||
messages.append({"role": "user", "content": user_content})
|
||
if is_api:
|
||
output = self._timed_api_completion(api_cfg, "text-only", messages=messages)
|
||
else:
|
||
output = self._timed_completion(LLAMA_CPP_STORAGE.llm, "text-only", messages=messages, seed=seed, **_parameters)
|
||
out1 = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out2 = [line for line in out1.split('\n') if line.strip()]
|
||
|
||
if save_states:
|
||
print(f"[XB-llama] 保存状态 id={uid}...")
|
||
messages.append({"role": "assistant", "content": out1})
|
||
clear_message = self.sanitize_messages(messages)
|
||
LLAMA_CPP_STORAGE.messages[f"{uid}"] = clear_message
|
||
else:
|
||
if not LLAMA_CPP_STORAGE.messages.get(f"{uid}"):
|
||
LLAMA_CPP_STORAGE.sys_prompts.pop(f"{uid}", None)
|
||
|
||
if force_offload:
|
||
if not is_api:
|
||
LLAMA_CPP_STORAGE.clean()
|
||
else:
|
||
if not is_api and LLAMA_CPP_STORAGE.current_config["chat_handler"] in ["Qwen3.5", "Qwen3.5-Thinking"]:
|
||
LLAMA_CPP_STORAGE.llm.n_tokens = 0
|
||
LLAMA_CPP_STORAGE.llm._ctx.memory_clear(True)
|
||
if LLAMA_CPP_STORAGE.llm.is_hybrid and LLAMA_CPP_STORAGE.llm._hybrid_cache_mgr is not None:
|
||
LLAMA_CPP_STORAGE.llm._hybrid_cache_mgr.clear()
|
||
|
||
del messages
|
||
gc.collect()
|
||
return (out1, out2, uid)
|
||
|
||
|
||
class XB_llamaParameters:
|
||
"""Llama 推理参数"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"max_tokens": ("INT", {"default": 6144, "min": 0, "max": 8192, "step": 1,
|
||
"tooltip": "生成 Token 上限\n6144 确保 6-8 镜分镜脚本不会截断"}),
|
||
"top_k": ("INT", {"default": 40, "min": 0, "max": 1000, "step": 1,
|
||
"tooltip": "词汇库检索范围\n40 配合 0.60 温度，兼顾格式严谨与词汇多样"}),
|
||
"top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"min_p": ("FLOAT", {"default": 0.05, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"typical_p": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"temperature": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 2.0, "step": 0.01,
|
||
"tooltip": "温度\n0.60 确保 [SHOT_START] 格式严谨，减少幻觉"}),
|
||
"repeat_penalty": ("FLOAT", {"default": 1.12, "min": 0.0, "max": 10.0, "step": 0.01,
|
||
"tooltip": "重复惩罚\n1.12 避免多镜分镜运镜描述句式复读"}),
|
||
"frequency_penalty": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"present_penalty": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 2.0, "step": 0.01}),
|
||
"mirostat_mode": ("INT", {"default": 0, "min": 0, "max": 2, "step": 1}),
|
||
"mirostat_eta": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"mirostat_tau": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 10.0, "step": 0.01}),
|
||
"state_uid": ("INT", {
|
||
"default": -1, "min": -1, "max": 999999, "step": 1,
|
||
"tooltip": "使用特定 ID 保存对话状态 (-1 = 使用节点 unique_id)"
|
||
}),
|
||
}
|
||
}
|
||
RETURN_TYPES = ("LLAMACPPARAMS",)
|
||
RETURN_NAMES = ("parameters",)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def process(self, **kwargs):
|
||
return (kwargs,)
|
||
|
||
|
||
class XB_llamaModelLoaderPro:
|
||
"""Llama 模型加载器 Pro — 模型选择 + 全部推理参数合并，支持折叠高级选项
|
||
|
||
=「XB-llama - 📦 模型加载器」+「XB-llama - ⚙️ 推理参数」二合一（复刻自 JZL - 🚀 模型加载Pro）。
|
||
惰性加载：本节点只产出配置，真正加载由下游推理节点在需要时执行，
|
||
这样切换到「在线 API」时彻底不加载本地模型（不占显存、不打本地加载日志）。
|
||
"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
all_llms = folder_paths.get_filename_list("LLM")
|
||
model_list = [f for f in all_llms if "mmproj" not in f.lower()]
|
||
mmproj_list = [f for f in all_llms if "mmproj" in f.lower()] or ["None"]
|
||
|
||
return {
|
||
"required": {
|
||
"model": (model_list,),
|
||
"mmproj": (mmproj_list,),
|
||
"chat_handler": (chat_handlers, {"default": "None"}),
|
||
"advanced_settings": ("BOOLEAN", {
|
||
"default": False,
|
||
"label_on": "高级参数 ▾",
|
||
"label_off": "高级参数 ▸",
|
||
"tooltip": "开启后显示上下文长度、显存上限、图像token 及全部推理参数"
|
||
}),
|
||
"n_ctx": ("INT", {
|
||
"default": 32768,
|
||
"min": 1024, "max": 262144, "step": 128,
|
||
"tooltip": "上下文长度上限\nQwen3.5-9B 原生 262144（256K）；短篇 32768，56 段调到 131072-262144"
|
||
}),
|
||
"vram_limit": ("INT", {
|
||
"default": -1,
|
||
"min": -1, "max": 1024, "step": 1,
|
||
"tooltip": "显存使用上限(GB), -1=不限制\n参考值, 实际可能略超"
|
||
}),
|
||
"image_min_tokens": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 32}),
|
||
"image_max_tokens": ("INT", {"default": 0, "min": 0, "max": 4096, "step": 32}),
|
||
"max_tokens": ("INT", {"default": 8192, "min": 0, "max": 262144, "step": 1,
|
||
"tooltip": "生成 Token 上限（Qwen3.5-9B 上限 262144）\n6 段约 12K，56 段约 128K，请按段数调整"}),
|
||
"top_k": ("INT", {"default": 40, "min": 0, "max": 1000, "step": 1,
|
||
"tooltip": "词汇库检索范围\n40 配合 0.60 温度，兼顾格式严谨与词汇多样"}),
|
||
"top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"min_p": ("FLOAT", {"default": 0.05, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"typical_p": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"temperature": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 2.0, "step": 0.01,
|
||
"tooltip": "温度\n0.60 确保 [SHOT_START] 格式严谨，减少幻觉"}),
|
||
"repeat_penalty": ("FLOAT", {"default": 1.05, "min": 0.0, "max": 10.0, "step": 0.01,
|
||
"tooltip": "重复惩罚\n1.05 轻微防句式复读，同时保留台词（太高会让 LLM 省略已在故事里写过的台词）"}),
|
||
"frequency_penalty": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"present_penalty": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 2.0, "step": 0.01}),
|
||
"mirostat_mode": ("INT", {"default": 0, "min": 0, "max": 2, "step": 1}),
|
||
"mirostat_eta": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||
"mirostat_tau": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 10.0, "step": 0.01}),
|
||
"state_uid": ("INT", {
|
||
"default": -1, "min": -1, "max": 999999, "step": 1,
|
||
"tooltip": "使用特定 ID 保存对话状态 (-1 = 使用节点 unique_id)"
|
||
}),
|
||
"backend": (["llama-server", "llama-cpp-python"], {
|
||
"default": "llama-cpp-python",
|
||
"tooltip": ("本地推理后端。\n"
|
||
"llama-cpp-python：进程内后端（XB_ToolBox 当前实现，默认）\n"
|
||
"llama-server：子进程后端，需额外的服务端运行环境；"
|
||
"当前 XB 后端尚未接入，选它不会改变实际推理方式（仅记录到配置里）")
|
||
}),
|
||
"gpu_device": (["auto", "0", "1", "2", "3"], {
|
||
"default": "auto",
|
||
"tooltip": ("GPU 设备索引（仅 llama-server 子进程后端使用）。\n"
|
||
"auto = 自动跟随 ComfyUI 当前显卡；当前 XB 进程内后端始终自动跟随 ComfyUI")
|
||
}),
|
||
}
|
||
}
|
||
|
||
RETURN_TYPES = ("LLAMACPPMODEL", "LLAMACPPARAMS")
|
||
RETURN_NAMES = ("llama_model", "parameters")
|
||
FUNCTION = "loadmodel"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def loadmodel(self, model, mmproj, chat_handler, advanced_settings,
|
||
n_ctx, vram_limit, image_min_tokens, image_max_tokens,
|
||
max_tokens, top_k, top_p, min_p, typical_p,
|
||
temperature, repeat_penalty, frequency_penalty, present_penalty,
|
||
mirostat_mode, mirostat_eta, mirostat_tau, state_uid, backend, gpu_device):
|
||
custom_config = {
|
||
"model": model,
|
||
"mmproj": mmproj,
|
||
"chat_handler": chat_handler,
|
||
"n_ctx": n_ctx,
|
||
"vram_limit": vram_limit,
|
||
"image_min_tokens": image_min_tokens,
|
||
"image_max_tokens": image_max_tokens,
|
||
"backend": backend,
|
||
"gpu_device": gpu_device
|
||
}
|
||
|
||
parameters = {
|
||
"max_tokens": max_tokens,
|
||
"top_k": top_k,
|
||
"top_p": top_p,
|
||
"min_p": min_p,
|
||
"typical_p": typical_p,
|
||
"temperature": temperature,
|
||
"repeat_penalty": repeat_penalty,
|
||
"frequency_penalty": frequency_penalty,
|
||
"present_penalty": present_penalty,
|
||
"mirostat_mode": mirostat_mode,
|
||
"mirostat_eta": mirostat_eta,
|
||
"mirostat_tau": mirostat_tau,
|
||
"state_uid": state_uid,
|
||
}
|
||
|
||
# 惰性加载：本节点只产出配置，真正加载由下游推理节点（如「分镜词处理器」）
|
||
# 在需要时按需执行，这样选择「在线 API」时彻底不加载本地模型。
|
||
return (custom_config, parameters)
|
||
|
||
|
||
class XB_llamaCleanStates:
|
||
"""清理对话状态"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"any": (any_type,),
|
||
"state_uid": ("INT", {
|
||
"default": -1, "min": -1, "max": 999999, "step": 1,
|
||
"tooltip": "清除特定 ID 的状态 (-1 = 清除全部)"
|
||
}),
|
||
},
|
||
}
|
||
|
||
RETURN_TYPES = (any_type,)
|
||
RETURN_NAMES = ("any",)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def process(self, any, state_uid):
|
||
print(f"[XB-llama] 清理状态 state_uid={state_uid}...")
|
||
LLAMA_CPP_STORAGE.clean_state(state_uid)
|
||
return (any,)
|
||
|
||
|
||
class XB_llamaUnloadModel:
|
||
"""卸载 Llama 模型"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {"required": {"any": (any_type,)}}
|
||
|
||
RETURN_TYPES = (any_type,)
|
||
RETURN_NAMES = ("any",)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def process(self, any):
|
||
print("[XB-llama] 卸载 Llama 模型...")
|
||
LLAMA_CPP_STORAGE.clean()
|
||
return (any,)
|
||
|
||
|
||
class SEG:
|
||
"""SEG 数据类"""
|
||
def __init__(self, cropped_image, cropped_mask, confidence, crop_region, bbox, label, control_net_wrapper=None):
|
||
self.cropped_image = cropped_image
|
||
self.cropped_mask = cropped_mask
|
||
self.confidence = confidence
|
||
self.crop_region = crop_region
|
||
self.bbox = bbox
|
||
self.label = label
|
||
self.control_net_wrapper = control_net_wrapper
|
||
|
||
|
||
class XB_llamaBBoxes2BBox:
|
||
"""从 BBoxes 中提取单个 BBox"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"bboxes": ("BBOX",),
|
||
"image_index": ("INT", {"default": 0, "min": 0, "max": 1000000, "step": 1}),
|
||
"bbox_index": ("INT", {
|
||
"default": 0, "min": -998, "max": 999, "step": 1,
|
||
"tooltip": "图像中的 BBox 索引 (999 = 获取全部)"
|
||
}),
|
||
}
|
||
}
|
||
|
||
RETURN_TYPES = ("BBOX",)
|
||
RETURN_NAMES = ("bbox",)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def process(self, bboxes, image_index, bbox_index):
|
||
if bbox_index != 999:
|
||
return ([bboxes[image_index][bbox_index]],)
|
||
return (bboxes[image_index],)
|
||
|
||
|
||
def get_nested_value(data, dotted_key, default=None):
|
||
keys = dotted_key.split('.')
|
||
for key in keys:
|
||
if isinstance(data, str):
|
||
try:
|
||
data = json.loads(data)
|
||
except Exception:
|
||
return default
|
||
if isinstance(data, dict) and key in data:
|
||
data = data[key]
|
||
else:
|
||
return default
|
||
return data
|
||
|
||
|
||
class XB_llamaUnpackCodeBlock:
|
||
"""解包代码块"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"input": ("STRING", {"forceInput": True}),
|
||
},
|
||
"optional": {
|
||
"label": ("STRING",),
|
||
},
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING",)
|
||
RETURN_NAMES = ("output",)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def process(self, input, label=None):
|
||
if isinstance(input, str):
|
||
input = [input]
|
||
|
||
output = []
|
||
label_str = label if label else ""
|
||
for value in input:
|
||
output.append(value.strip().removeprefix(f"```{label_str}").removesuffix("```"))
|
||
if len(output) == 1:
|
||
return (output[0],)
|
||
return (output,)
|
||
|
||
|
||
class XB_llamaPromptEnhancer:
|
||
"""提示词增强预设"""
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"preset": ([
|
||
"Animagine XL [EN]", "Animagine XL [ZH]",
|
||
"Boogu T2I [EN]", "Boogu T2I [ZH]",
|
||
"ERNIE-Image T2I [EN]", "ERNIE-Image T2I [ZH]",
|
||
"Flux.2 I2I [EN]", "Flux.2 I2I [ZH]",
|
||
"Flux.2 Klein I2I [EN]", "Flux.2 Klein I2I [ZH]",
|
||
"Flux.2 Klein T2I [EN]", "Flux.2 Klein T2I [ZH]",
|
||
"Flux.2 T2I [EN]", "Flux.2 T2I [ZH]",
|
||
"Ideogram 4.0 T2I [EN]", "Ideogram 4.0 T2I [ZH]",
|
||
"Krea2 T2I [EN]", "Krea2 T2I [ZH]",
|
||
"LTX2.3 T2V [EN]", "LTX2.3 T2V [ZH]",
|
||
"Lumina-Next T2I [EN]", "Lumina-Next T2I [ZH]",
|
||
"Pony V6 XL [EN]", "Pony V6 XL [ZH]",
|
||
"Qwen-Image 2512 [EN]", "Qwen-Image 2512 [ZH]",
|
||
"Qwen-Image-Edit 2511 [EN]", "Qwen-Image-Edit 2511 [ZH]",
|
||
"Wan FLF2V [EN]", "Wan FLF2V [ZH]",
|
||
"Wan I2V [EN]", "Wan I2V [ZH]",
|
||
"Wan I2V Full-Auto [EN]", "Wan I2V Full-Auto [ZH]",
|
||
"Wan T2V [EN]", "Wan T2V [ZH]",
|
||
"Z-Image Turbo [ZH]", "Z-Image Turbo [EN]",
|
||
],),
|
||
}
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING",)
|
||
RETURN_NAMES = ("system_prompt",)
|
||
FUNCTION = "main"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def main(self, preset):
|
||
match preset:
|
||
case "Qwen-Image 2512 [EN]":
|
||
return (QWEN_IMAGE_2512_EN,)
|
||
case "Qwen-Image 2512 [ZH]":
|
||
return (QWEN_IMAGE_2512_ZH,)
|
||
case "Qwen-Image-Edit 2511 [EN]":
|
||
return (QWEN_IMAGE_EDIT_2511,)
|
||
case "Qwen-Image-Edit 2511 [ZH]":
|
||
return (QWEN_IMAGE_EDIT_2511_ZH,)
|
||
case "Z-Image Turbo [ZH]":
|
||
return (ZIMAGE_TURBO,)
|
||
case "Z-Image Turbo [EN]":
|
||
return (ZIMAGE_TURBO_EN,)
|
||
case "Flux.2 T2I [EN]":
|
||
return (FLUX2_T2I,)
|
||
case "Flux.2 T2I [ZH]":
|
||
return (FLUX2_T2I_ZH,)
|
||
case "Flux.2 I2I [EN]":
|
||
return (FLUX2_I2I,)
|
||
case "Flux.2 I2I [ZH]":
|
||
return (FLUX2_I2I_ZH,)
|
||
case "Wan T2V [EN]":
|
||
return (WAN_T2V_EN,)
|
||
case "Wan T2V [ZH]":
|
||
return (WAN_T2V_ZH,)
|
||
case "Wan I2V [EN]":
|
||
return (WAN_I2V_EN,)
|
||
case "Wan I2V [ZH]":
|
||
return (WAN_I2V_ZH,)
|
||
case "Wan I2V Full-Auto [EN]":
|
||
return (WAN_I2V_EMPTY_EN,)
|
||
case "Wan I2V Full-Auto [ZH]":
|
||
return (WAN_I2V_EMPTY_ZH,)
|
||
case "Wan FLF2V [EN]":
|
||
return (WAN_FLF2V_EN,)
|
||
case "Wan FLF2V [ZH]":
|
||
return (WAN_FLF2V_ZH,)
|
||
case "Krea2 T2I [EN]":
|
||
return (KREA2_T2I_EN,)
|
||
case "Krea2 T2I [ZH]":
|
||
return (KREA2_T2I_ZH,)
|
||
case "Boogu T2I [EN]":
|
||
return (BOOGU_T2I_EN,)
|
||
case "Boogu T2I [ZH]":
|
||
return (BOOGU_T2I_ZH,)
|
||
case "LTX2.3 T2V [EN]":
|
||
return (LTX2_3_T2V_EN,)
|
||
case "LTX2.3 T2V [ZH]":
|
||
return (LTX2_3_T2V_ZH,)
|
||
case "Animagine XL [EN]":
|
||
return (ANIMAGINE_T2I_EN,)
|
||
case "Animagine XL [ZH]":
|
||
return (ANIMAGINE_T2I_ZH,)
|
||
case "Lumina-Next T2I [EN]":
|
||
return (LUMINA_T2I_EN,)
|
||
case "Lumina-Next T2I [ZH]":
|
||
return (LUMINA_T2I_ZH,)
|
||
case "Pony V6 XL [EN]":
|
||
return (PONY_V6_T2I_EN,)
|
||
case "Pony V6 XL [ZH]":
|
||
return (PONY_V6_T2I_ZH,)
|
||
case "ERNIE-Image T2I [EN]":
|
||
return (ERNIE_IMAGE_T2I_EN,)
|
||
case "ERNIE-Image T2I [ZH]":
|
||
return (ERNIE_IMAGE_T2I_ZH,)
|
||
case "Ideogram 4.0 T2I [EN]":
|
||
return (IDEOGRAM_V4_T2I_EN,)
|
||
case "Ideogram 4.0 T2I [ZH]":
|
||
return (IDEOGRAM_V4_T2I_ZH,)
|
||
case "Flux.2 Klein T2I [EN]":
|
||
return (FLUX2_KLEIN_T2I_EN,)
|
||
case "Flux.2 Klein T2I [ZH]":
|
||
return (FLUX2_KLEIN_T2I_ZH,)
|
||
case "Flux.2 Klein I2I [EN]":
|
||
return (FLUX2_KLEIN_I2I_EN,)
|
||
case "Flux.2 Klein I2I [ZH]":
|
||
return (FLUX2_KLEIN_I2I_ZH,)
|
||
case _:
|
||
raise ValueError(f'未知预设: "{preset}"')
|
||
|
||
|
||
class XB_llamaMiniMaxPreset:
|
||
"""MiniMax H3 提示词预设 — T2VA/I2VA/FL2VA/L2VA + 动态参数注入"""
|
||
|
||
_STYLES = [
|
||
"不指定 / Unspecified",
|
||
# 🎬 电影感
|
||
"电影感 / Cinematic", "实拍 / Live-action",
|
||
"复古胶片 / Vintage film", "黑白电影 / Black & White",
|
||
"纪录片 / Documentary",
|
||
# 📷 商业摄影
|
||
"极简广告 / Minimalist commercial",
|
||
"微距摄影 / Macro photography",
|
||
"航拍 / Aerial drone",
|
||
# 🎨 动画
|
||
"二维动画 / 2D-animated", "三维CG / 3D CG",
|
||
"日系二次元 / Anime", "美式漫画 / American Comic",
|
||
"皮克斯3D / Pixar-style 3D", "定格动画 / Stop-motion",
|
||
"手绘发光 / Hand-drawn glow", "像素艺术 / Pixel art",
|
||
# 🚀 科幻前卫
|
||
"赛博朋克 / Cyberpunk", "蒸汽朋克 / Steampunk",
|
||
"故障艺术 / Glitch art",
|
||
# 🧶 特殊材质
|
||
"羊毛毡 / Wool felt", "折纸 / Origami",
|
||
# 🖌️ 美术
|
||
"水彩 / Watercolor", "粘土动画 / Claymation",
|
||
"水墨 / Ink wash", "油画 / Oil painting",
|
||
"纸艺拼贴 / Paper collage", "剪纸 / Paper cutout",
|
||
"铅笔素描 / Pencil sketch", "浮世绘 / Ukiyo-e",
|
||
# 🏮 中国风
|
||
"敦煌壁画 / Dunhuang Murals",
|
||
"青花瓷 / Blue-white Porcelain",
|
||
"工笔画 / Gongbi Painting",
|
||
"皮影戏 / Shadow Puppetry",
|
||
"中国风插画 / Chinese Illustration",
|
||
"年画 / New Year Painting",
|
||
# 🧵 手工布艺
|
||
"布艺 / Fabric Art",
|
||
"蜡笔画 / Crayon drawing",
|
||
"哥特萝莉 / Gothic Lolita",
|
||
]
|
||
|
||
_STYLE_HINTS = {
|
||
"电影感 / Cinematic": "cinematic lighting with shallow depth of field, film grain, and professional color grading",
|
||
"实拍 / Live-action": "photorealistic live-action footage with natural lighting and authentic set design",
|
||
"复古胶片 / Vintage film": "vintage film stock with warm color grading, subtle grain, and nostalgic atmosphere",
|
||
"黑白电影 / Black & White": "high-contrast black-and-white cinematography with dramatic shadows",
|
||
"纪录片 / Documentary": "observational documentary style with natural handheld camera work and candid framing",
|
||
"极简广告 / Minimalist commercial": "clean minimalist product cinematography with smooth dolly moves, soft even lighting, and uncluttered compositions",
|
||
"微距摄影 / Macro photography": "extreme close-up macro lens with razor-thin depth of field, revealing fine textures and details",
|
||
"航拍 / Aerial drone": "sweeping aerial drone shots with wide vistas, slow majestic reveals, and expansive landscape views",
|
||
"二维动画 / 2D-animated": "traditional 2D hand-drawn animation with expressive line art and fluid character motion",
|
||
"三维CG / 3D CG": "high-quality 3D rendering with realistic materials, global illumination, and smooth animation",
|
||
"日系二次元 / Anime": "Japanese anime cel-shading with vibrant saturated colors, clean linework, and expressive character designs",
|
||
"美式漫画 / American Comic": "American comic book style with bold black ink outlines, halftone dot shading, and dynamic compositions",
|
||
"皮克斯3D / Pixar-style 3D": "Pixar-quality 3D with smooth curved surfaces, rich vibrant colors, expressive character animation, and polished lighting",
|
||
"定格动画 / Stop-motion": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a stop-motion animation. Characters are handcrafted puppets with visible material textures, moving with tactile frame-by-frame stutter. Environments are miniature physical sets with real fabrics, painted backdrops, and practical lighting. EVERYTHING is a physical model under a camera.",
|
||
"手绘发光 / Hand-drawn glow": "GLOBAL MATERIAL OVERRIDE — the entire visual world is rough hand-drawn line art on dark paper. Characters and environments are sketched with glowing neon-colored outlines that flicker and pulse organically. Light trails follow movement like afterimages. The world itself is a living drawing, every line redrawn in real-time.",
|
||
"像素艺术 / Pixel art": "GLOBAL MATERIAL OVERRIDE — the entire visual world is built from visible pixel blocks. Characters, environments, water, fire, smoke, sky — EVERYTHING is composed of crisp square pixels with a limited retro color palette. Motion is frame-by-frame at low FPS with deliberate pixel-level changes. Particles are individual pixel dots. The pixel grid IS the universe.",
|
||
"赛博朋克 / Cyberpunk": "high-contrast neon-lit cyberpunk cityscape with rain-slicked streets, holographic displays, and chrome cybernetics",
|
||
"蒸汽朋克 / Steampunk": "intricate brass machinery and Victorian-era steam technology with copper pipes, gears, and sepia tones",
|
||
"故障艺术 / Glitch art": "digital glitch distortion with RGB color channel split, scan lines, data corruption artifacts, and VHS noise",
|
||
"羊毛毡 / Wool felt": "GLOBAL MATERIAL OVERRIDE — the entire visual world is handcrafted from fuzzy wool felt. Characters have soft felt textile bodies with visible fiber textures and stitched seams. Wind ripples through felt grass, felt water flows with fiber movement, felt clouds drift across a felt sky. Environments are sewn felt dioramas. DO NOT place felt toys in a real scene — everything IS felt.",
|
||
"折纸 / Origami": "GLOBAL MATERIAL OVERRIDE — the entire universe is constructed from folded paper. Characters are origami figures with sharp clean creases and geometric folded anatomy. Paper birds flap creased wings, paper water ripples in folded layers, paper fire crackles as curling sheets. The world itself is paper — all matter is folded, creased, and crisp.",
|
||
"水彩 / Watercolor": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D watercolor painting on paper. Characters are NOT real people — their bodies are translucent color washes, their faces are soft pigment blooms on wet paper, their edges dissolve into the paper grain. Hair is bleeding pigment streaks, skin is the white of paper with tinted wash. Rain falls as pigment droplets, light diffuses through layered washes. NO realistic skin, NO 3D — only wet pigment on paper.",
|
||
"粘土动画 / Claymation": "GLOBAL MATERIAL OVERRIDE — the entire visual world is hand-sculpted clay. Characters are NOT real people — their bodies are clay with rounded tactile surfaces, visible fingerprints, and tool marks. Hair is sculpted clay strands, skin is smooth plasticine, clothing is pressed clay sheets. Clay water splashes in sculpted droplets, clay smoke rolls in malleable puffs. NO real skin — only clay shaped by human hands.",
|
||
"水墨 / Ink wash": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D ink wash painting on xuan rice paper. Characters are NOT real people — their bodies are fluid black brushstrokes, their faces are ink lines on paper, their clothing is graded ink washes. Hair flows as sweeping brushstrokes, skin tone is the white of the paper itself with ink shading. Water splashes as flying ink drops, wind leaves brushstroke trails, mist is spreading ink on wet paper. NO realistic skin, NO realistic fabric, NO 3D — only ink and paper.",
|
||
"油画 / Oil painting": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D oil painting on canvas. Characters are NOT real people — their bodies are thick oil paint applied with palette knives, their faces are built from layered brushstrokes, their clothing is impasto pigment. Hair is swept paint, skin is blended oil color on canvas, eyes are precise brush dabs. Water ripples in heavy oil strokes, fire is palette-knife texture, clouds are smeared white paint. NO realistic skin, NO real fabric, NO 3D — only oil paint on canvas.",
|
||
"纸艺拼贴 / Paper collage": "GLOBAL MATERIAL OVERRIDE — the entire visual world is layered torn paper. Characters are NOT real people — their bodies are cut from textured paper with torn edges, their faces are printed paper fragments, their clothing is different paper types (newsprint, craft, tissue). Paper birds flap torn-edge wings, paper water ripples in layered sheets. NO real skin — only paper.",
|
||
"剪纸 / Paper cutout": "GLOBAL MATERIAL OVERRIDE — the entire visual world is Chinese paper cutout art. Characters are NOT real people — their bodies are intricate red paper silhouettes cut with symmetrical patterns, moving with articulated paper joints. Shadows cast dramatic shapes through the paper lattice. NO real skin — only cut paper.",
|
||
"铅笔素描 / Pencil sketch": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D graphite pencil drawing on textured paper. Characters are NOT real people — their bodies are graphite lines, hatching, and cross-hatching on paper. Faces are sketched pencil marks, hair is sweeping graphite strokes, skin tone is the white of paper with varying pencil pressure. Motion is lines erasing and redrawing. Eraser marks leave ghost trails. NO real skin, NO 3D — only pencil on paper.",
|
||
"浮世绘 / Ukiyo-e": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D Japanese ukiyo-e woodblock print. Characters are NOT real people — their bodies are flat color areas with bold black outlines printed on washi paper. Faces are printed woodblock features, hair is carved-line black ink, clothing is flat color blocks. NO real skin, NO 3D — only woodblock ink on paper.",
|
||
"敦煌壁画 / Dunhuang Murals": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D animated Dunhuang cave mural on a fresco wall. Characters are NOT real people — their bodies are mineral pigment paintings (ochre, turquoise, lapis lazuli) with weathered fresco cracks. Flying deities trail faded pigment ribbons. NO real skin, NO 3D — only ancient mural pigment on plaster.",
|
||
"青花瓷 / Blue-white Porcelain": "GLOBAL MATERIAL OVERRIDE — the entire universe is living 3D blue-and-white porcelain. Characters are NOT real people — their bodies are white-glazed porcelain with cobalt-blue hand-painted patterns flowing across their skin as features and clothing. Porcelain birds take flight with clicking ceramic wings, porcelain water flows as liquid glaze, porcelain trees bloom with cobalt flowers. NO real skin — only glazed ceramic.",
|
||
"工笔画 / Gongbi Painting": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D gongbi painting on flat silk. Characters are NOT real people — their bodies are ultra-fine brush outlines filled with flat mineral color washes on silk. Every hair and petal is individually painted. Silk fibers visible beneath the pigment. NO real skin, NO 3D — only brush and silk.",
|
||
"皮影戏 / Shadow Puppetry": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D shadow puppet theater on a translucent screen. Characters are NOT real people — their bodies are intricately carved leather silhouettes with articulated joints, illuminated by warm amber backlighting. NO real skin, NO 3D — only leather shadows on a screen.",
|
||
"中国风插画 / Chinese Illustration": "modern Chinese illustration blending traditional ink aesthetics with contemporary digital art, featuring elegant flowing lines, poetic composition, and dreamlike color harmony",
|
||
"年画 / New Year Painting": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D vibrant Chinese folk New Year woodblock print. Characters are NOT real people — their bodies are bold primary color blocks with thick black outlines on flat printed paper. Door gods step out of frames and walk, carp leap as printed patterns. NO real skin, NO 3D — only folk print on paper.",
|
||
"布艺 / Fabric Art": "GLOBAL MATERIAL OVERRIDE — the entire visual world is constructed from sewn fabric and textiles. Characters are NOT real people — they are cloth dolls with stitched seams, button eyes, embroidered facial features, yarn hair, and patchwork clothing. Environments are quilted fabric landscapes: grass is green felt, water is flowing blue silk, clouds are tufted cotton, trees are embroidered tapestry. Every surface shows visible thread, stitching, and fabric grain. NO realistic skin, NO real materials — only fabric and thread.",
|
||
"蜡笔画 / Crayon drawing": "GLOBAL MATERIAL OVERRIDE — the entire visual world is a 2D wax crayon drawing on textured paper. Characters are NOT real people — their bodies are waxy crayon strokes on paper with the paper grain visible through the wax. Bright colors have the distinctive grainy, slightly uneven crayon texture. Lines are thick and waxy with visible stroke direction. Paper texture shows through all color areas. NO 3D depth, NO CG, NO realistic skin — only crayon on paper.",
|
||
"哥特萝莉 / Gothic Lolita": "Gothic Lolita fashion and atmosphere — NOT a material override but a costume and world style. Characters wear elaborate dark Victorian-inspired Lolita clothing: lace-trimmed black dresses, ruffled petticoats, corsets, platform boots, ornate headpieces with ribbons and roses. Architecture is moody Gothic with pointed arches, stained glass, wrought iron. Dramatic chiaroscuro lighting with deep shadows. Color palette: black, deep purple, burgundy, ivory, silver accents. Atmosphere is darkly romantic and theatrical.",
|
||
}
|
||
|
||
_MUSIC = [
|
||
"禁止音乐 / No Music",
|
||
"不指定 / Unspecified",
|
||
# 🎹 器乐
|
||
"钢琴 / Piano", "管弦乐 / Orchestral",
|
||
"原声吉他 / Acoustic",
|
||
# 🎛️ 电子
|
||
"电子 / Electronic", "氛围 / Ambient",
|
||
"合成器浪潮 / Synthwave", "芯片音乐 / Chiptune",
|
||
"Lo-fi / Lo-fi",
|
||
# 🎞️ 叙事
|
||
"史诗 / Epic", "悬疑 / Suspense",
|
||
"浪漫弦乐 / Romantic Strings",
|
||
# 🥁 节奏
|
||
"摇滚 / Rock", "爵士 / Jazz",
|
||
"嘻哈 / Hip-Hop", "放克 / Funk",
|
||
# ⛪ 人声
|
||
"纯人声合唱 / Acapella Choir",
|
||
# 🔇 极简
|
||
"极简拟音 / Minimalist Foley",
|
||
# 🏮 中国风
|
||
"国风民乐 / Chinese Folk",
|
||
"戏曲 / Chinese Opera",
|
||
"古琴 / Guqin",
|
||
]
|
||
|
||
_MUSIC_HINTS = {
|
||
"禁止音乐 / No Music": "ABSOLUTELY NO background music of any kind. non_diegetic_music MUST be \"N/A\". Do not add any score, melody, or rhythm.",
|
||
"钢琴 / Piano": "a solo piano piece at a slow to moderate tempo, with sparse delicate notes and natural reverb",
|
||
"管弦乐 / Orchestral": "a majestic full orchestral arrangement with swelling strings and warm brass, maintaining a steady high-energy rhythm throughout",
|
||
"原声吉他 / Acoustic": "an acoustic guitar piece with gentle fingerpicking patterns and warm natural wood resonance",
|
||
"电子 / Electronic": "an electronic track with layered synthesizers, digital beats, and atmospheric pads",
|
||
"氛围 / Ambient": "a minimal ambient soundscape with long sustained tones, subtle textures, and no distinct rhythm",
|
||
"合成器浪潮 / Synthwave": "a pulsing Synthwave instrumental track with heavy analog bass, retro drum machines, neon-soaked pads, and a driving steady rhythm, no vocals, starting abruptly at full energy with zero intro",
|
||
"芯片音乐 / Chiptune": "a retro 8-bit instrumental chiptune track with square-wave melodies, simple waveforms, and nostalgic video game sound",
|
||
"Lo-fi / Lo-fi": "a lo-fi instrumental beat with vinyl crackle, mellow chords, soft drum loops, and a relaxed downtempo groove",
|
||
"史诗 / Epic": "an epic cinematic instrumental score with powerful brass, thundering percussion, soaring choir, and dramatic steady intensity, starting abruptly at full energy without any intro or build-up",
|
||
"悬疑 / Suspense": "a tense suspense instrumental score with low-frequency drones, sudden dissonant stabs, creeping tension, and unsettling silence",
|
||
"浪漫弦乐 / Romantic Strings": "a romantic instrumental string arrangement with lush violins, gentle cello, harp glissandos, and a tender sustained atmosphere",
|
||
"摇滚 / Rock": "an instrumental-only rock track with electric guitar riffs, driving drums, bass groove, and energetic dynamics, STRICTLY NO VOCALS, starting at full power with zero intro",
|
||
"爵士 / Jazz": "an instrumental jazz piece with walking bass, brushed drums, improvisational piano or saxophone, smoky club atmosphere, no vocals",
|
||
"嘻哈 / Hip-Hop": "an instrumental hip-hop beat with heavy 808 bass, crisp trap snares, hi-hat rolls, and a grooving rhythmic flow, STRICTLY NO VOCALS, dropping in at full energy with no intro",
|
||
"放克 / Funk": "an instrumental funk groove with a bouncy slap bassline, tight rhythm guitar, brass stabs, and an infectious syncopated rhythm, no vocals, kicking in immediately at full groove",
|
||
"纯人声合唱 / Acapella Choir": "a pure acapella choir with layered vocal harmonies and no instruments, evoking sacred, ethereal, or haunting atmosphere",
|
||
"极简拟音 / Minimalist Foley": "minimalist foley and ambient silence — only crisp physical sound effects like subtle clicks, soft whooshes, and spatial emptiness, with no melodic music at all",
|
||
"国风民乐 / Chinese Folk": "a traditional Chinese folk piece with guzheng, erhu, dizi bamboo flute, pipa, and flowing pentatonic melodies evoking ancient landscapes",
|
||
"戏曲 / Chinese Opera": "a stylized Chinese opera piece with clanging gongs, wooden clappers, piercing erhu, and dramatic vocal delivery in traditional theatrical style",
|
||
"古琴 / Guqin": "a solo guqin piece with deep resonant plucked silk strings, slow meditative pace, profound stillness, and subtle harmonic overtones",
|
||
}
|
||
|
||
_ASPECTS = [
|
||
"16:9", "9:16", "4:3", "3:4", "1:1", "21:9", "4:5", "5:4", "2:3", "3:2",
|
||
]
|
||
|
||
_ASPECT_HINTS = {
|
||
"16:9": "standard widescreen — use wide establishing shots, horizontal subject placement, cinematic scope",
|
||
"9:16": "vertical portrait — center subjects vertically, stack elements top-to-bottom, leave breathing room above and below",
|
||
"4:3": "classic Academy ratio — balanced framed composition, suited for dialogue and character-focused shots",
|
||
"3:4": "tall portrait — vertical emphasis, subjects fill the frame from top to bottom, dramatic low/high angles",
|
||
"1:1": "square format — symmetrical center-weighted composition, subjects centered in frame",
|
||
"21:9": "ultrawide cinematic — emphasize sweeping horizontal space, panoramic landscapes, subjects placed off-center with vast negative space",
|
||
"4:5": "portrait tall — slightly taller than 3:4, ideal for social media portrait, subjects framed with vertical breathing room",
|
||
"5:4": "landscape tall — slightly taller than standard, balanced composition with modest horizontal emphasis",
|
||
"2:3": "portrait photo — classic DSLR vertical framing, subjects stand tall in frame, strong vertical emphasis with generous headroom",
|
||
"3:2": "photo landscape — classic DSLR horizontal framing, natural wide composition, balanced foreground and background depth",
|
||
}
|
||
|
||
_CUTS = [
|
||
"不指定 / Unspecified",
|
||
"不切镜 / Single Shot",
|
||
"1 次切镜 / 1 Cut",
|
||
"2 次切镜 / 2 Cuts",
|
||
"3 次切镜 / 3 Cuts",
|
||
"4 次切镜 / 4 Cuts",
|
||
"5 次切镜 / 5 Cuts",
|
||
"6 次切镜 / 6 Cuts",
|
||
"7 次切镜 / 7 Cuts",
|
||
"8 次切镜 / 8 Cuts",
|
||
"9 次切镜 / 9 Cuts",
|
||
]
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"预设模式": ([
|
||
"纯文本生成音视频[英文]-T2VA [EN]", "纯文本生成音视频[中文]-T2VA [ZH]",
|
||
"首帧图生成音视频[英文]-I2VA [EN]", "首帧图生成音视频[中文]-I2VA [ZH]",
|
||
"首尾帧生成音视频[英文]-FL2VA [EN]", "首尾帧生成音视频[中文]-FL2VA [ZH]",
|
||
"尾帧图生成音视频[英文]-L2VA [EN]", "尾帧图生成音视频[中文]-L2VA [ZH]",
|
||
],),
|
||
"视频时长": ("INT", {"default": 8, "min": 4, "max": 60, "step": 1,
|
||
"tooltip": "视频时长 (秒), MiniMax H3 支持 4–60 秒"}),
|
||
"视觉风格": (s._STYLES, {"default": "不指定 / Unspecified"}),
|
||
"音乐风格": (s._MUSIC, {"default": "禁止音乐 / No Music"}),
|
||
"画面比例": (s._ASPECTS, {"default": "16:9"}),
|
||
"切镜次数": (s._CUTS, {"default": "不指定 / Unspecified"}),
|
||
}
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING",)
|
||
RETURN_NAMES = ("system_prompt",)
|
||
FUNCTION = "build"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def build(self, 预设模式, 视频时长, 视觉风格, 音乐风格, 画面比例, 切镜次数):
|
||
preset = 预设模式
|
||
duration = 视频时长
|
||
style = 视觉风格
|
||
music = 音乐风格
|
||
aspect = 画面比例
|
||
cuts = 切镜次数
|
||
match preset:
|
||
case "纯文本生成音视频[英文]-T2VA [EN]": base = MINIMAX_T2VA_EN
|
||
case "纯文本生成音视频[中文]-T2VA [ZH]": base = MINIMAX_T2VA_ZH
|
||
case "首帧图生成音视频[英文]-I2VA [EN]": base = MINIMAX_I2VA_EN
|
||
case "首帧图生成音视频[中文]-I2VA [ZH]": base = MINIMAX_I2VA_ZH
|
||
case "首尾帧生成音视频[英文]-FL2VA [EN]": base = MINIMAX_FL2VA_EN
|
||
case "首尾帧生成音视频[中文]-FL2VA [ZH]": base = MINIMAX_FL2VA_ZH
|
||
case "尾帧图生成音视频[英文]-L2VA [EN]": base = MINIMAX_L2VA_EN
|
||
case "尾帧图生成音视频[中文]-L2VA [ZH]": base = MINIMAX_L2VA_ZH
|
||
case _: raise ValueError(f'未知预设: "{preset}"')
|
||
|
||
is_zh = "[ZH]" in preset
|
||
params = []
|
||
|
||
if is_zh:
|
||
params.append("## 目标视频参数（必须严格遵守）")
|
||
params.append(f"- 视频时长：正好 {duration} 秒（镜头切分和时间戳必须精确落在此范围内，最后一个镜头必须在第{duration}秒前结束）")
|
||
if "不指定" not in style:
|
||
en_name = style.split(" / ")[0]
|
||
zh_name = style.split(" / ")[-1]
|
||
hint = self._STYLE_HINTS.get(style, "")
|
||
params.append(f"- 视觉风格：{zh_name} ({en_name}) — {hint}")
|
||
params.append(f" ⚠️ [Shot 1] 必须以 \"{en_name}\" 开头，然后立即用1-2句话详细描述{zh_name}在画面中的具体视觉呈现——材质、光影、色彩、动作特征。严禁只写风格名称就跳到下一句！必须写出该风格\"长什么样\"。")
|
||
params.append(f" 正确示例: \"[Shot 1] 粘土动画，画面中的角色呈现手工泥塑的圆润质感，表面可见细微指痕和工具刮痕，动作带有定格动画特有的逐帧卡顿节奏...\"")
|
||
params.append(f" 错误示例: \"[Shot 1] 粘土动画，中全景镜头...\"（只写了名称，没有视觉描述）")
|
||
if "不指定" not in music:
|
||
zh_name = music.split(" / ")[-1]
|
||
hint = self._MUSIC_HINTS.get(music, "")
|
||
params.append(f"- 背景音乐风格：{zh_name} — {hint}")
|
||
if "禁止音乐" in music:
|
||
params.append(f" ⚠️ 整个视频不得出现任何背景音乐。non_diegetic_music 必须严格输出 \"N/A\"。")
|
||
if "不指定" not in cuts:
|
||
if "不切镜" in cuts:
|
||
params.append(f"- 切镜：固定镜头，不切镜。整段视频只有 [Shot 1] 一个镜头。仅通过运镜（摇摄、俯仰、横移、变焦、跟拍、推拉）改变视角。严禁输出 [Shot N] 时间戳。")
|
||
else:
|
||
n = int(cuts.split(" ")[0])
|
||
max_ok = max(1, duration // 2)
|
||
if n > max_ok:
|
||
params.append(f"- ⚠️ 切镜次数 {n} 对于 {duration} 秒视频过多（合理上限 {max_ok} 次）。请根据实际可用时长自行削减到合理范围，保证每个镜头至少 2 秒。")
|
||
params.append(f"- 切镜：正好 {n} 次（共 {n+1} 个镜头）。所有镜头必须在 {duration} 秒内完成。切镜时机遵循叙事节奏——紧张段落切快、抒情段落切慢，不可均分。每次切镜必须以 [Shot N] At MM:SS.mmm 开头，时间戳严格递增。")
|
||
hint = self._ASPECT_HINTS.get(aspect, "")
|
||
params.append(f"- 画面比例：{aspect} — {hint}")
|
||
params.append(f" 所有镜头构图、主体位置、留白空间必须匹配 {aspect} 比例。")
|
||
else:
|
||
params.append("## Target Video Parameters (MUST follow exactly)")
|
||
params.append(f"- Duration: exactly {duration} seconds (all shot timestamps MUST fall within this range; the final shot must end before the {duration}-second mark)")
|
||
if "Unspecified" not in style:
|
||
en_name = style.split(" / ")[0]
|
||
hint = self._STYLE_HINTS.get(style, "")
|
||
params.append(f"- Visual style: {en_name} — {hint}")
|
||
params.append(f" ⚠️ [Shot 1] MUST begin with \"{en_name}\" and immediately elaborate with 1-2 sentences of concrete visual description — textures, lighting, colors, motion characteristics that define this style. Do NOT just name the style and move on. Show what the style actually looks like.")
|
||
params.append(f" Correct: \"[Shot 1] Claymation, the characters have the rounded tactile quality of hand-sculpted clay with visible fingerprints and tool marks, their movements carrying the distinctive frame-by-frame stutter of stop-motion...\"")
|
||
params.append(f" Wrong: \"[Shot 1] Claymation, a medium-wide shot...\" (name only, no visual description)")
|
||
if "Unspecified" not in music:
|
||
en_name = music.split(" / ")[0]
|
||
hint = self._MUSIC_HINTS.get(music, "")
|
||
params.append(f"- Background music style: {en_name} — {hint}")
|
||
if "No Music" in music:
|
||
params.append(f" ⚠️ The video must have absolutely no background music. non_diegetic_music MUST be \"N/A\".")
|
||
if "Unspecified" not in cuts:
|
||
if "Single Shot" in cuts:
|
||
params.append(f"- Cuts: Single continuous shot — NO cuts. The entire video is only [Shot 1]. Use camera movement only (pan, tilt, truck, zoom, tracking, push/pull) to change viewpoint. Do NOT output any [Shot N] timestamps.")
|
||
else:
|
||
n = int(cuts.split(" ")[0])
|
||
max_ok = max(1, duration // 2)
|
||
if n > max_ok:
|
||
params.append(f"- ⚠️ {n} cuts is excessive for a {duration}-second video (reasonable max: {max_ok}). Reduce to a feasible number, ensuring at least 2 seconds per shot.")
|
||
params.append(f"- Cuts: exactly {n} cut(s) (meaning {n+1} total shots). All shots must fit within {duration} seconds. Cut timing must follow narrative rhythm — faster cuts for tense/action moments, longer holds for calm/emotional moments. Do NOT space cuts evenly. Every cut MUST begin with [Shot N] At MM:SS.mmm with strictly increasing timestamps.")
|
||
hint = self._ASPECT_HINTS.get(aspect, "")
|
||
params.append(f"- Aspect ratio: {aspect} — {hint}")
|
||
params.append(f" All shot compositions, subject placement, and negative space must be framed for {aspect}.")
|
||
params.append("")
|
||
param_block = "\n".join(params)
|
||
|
||
marker = "## "
|
||
idx = base.find(marker)
|
||
if idx > 0:
|
||
result = base[:idx].rstrip() + "\n\n" + param_block + base[idx:]
|
||
else:
|
||
result = param_block + "\n" + base
|
||
|
||
return (result,)
|
||
|
||
|
||
class XB_llamaMiniMaxRef2vaPreset:
|
||
"""MiniMax H3 Ref2VA 全引用模式专用 — 多模态参考 + 风格策略"""
|
||
|
||
_STYLES = [
|
||
"保持统一风格 / Consistent Style",
|
||
"多种风格混搭 / Mixed Styles",
|
||
"多种风格转换 / Style Transformation",
|
||
]
|
||
|
||
_STYLE_HINTS = {
|
||
"保持统一风格 / Consistent Style": "STRICT STYLE CONSISTENCY — all characters, environments, and visual elements must share the exact same visual style derived from the reference images. Every frame must look like it belongs to a single unified visual universe. No character should look like they came from a different artwork.",
|
||
"多种风格混搭 / Mixed Styles": "STYLE MIXING — different characters or elements may retain their distinct visual styles from their respective references. Examples: a live-action person interacting with a 2D-animated character; two pixel-art characters walking through a photorealistic background; a clay figure next to an origami figure. CRITICAL: each reference element must remain visually STABLE — a real person must stay real, an anime character must stay anime, clay must stay clay throughout the video. The challenge is making them coexist naturally in the same space.",
|
||
"多种风格转换 / Style Transformation": "STYLE TRANSFORMATION — the ENTIRE frame undergoes a smooth, visible transition from one visual style to another over the course of the video. Examples: live-action gradually becomes 2D-animated; claymation transforms into origami; watercolor washes over a realistic scene. ALL shapes, proportions, and spatial relationships must be preserved during the transformation — only the rendering style changes. The transformation must be smooth and continuous, not an abrupt switch.",
|
||
}
|
||
|
||
_CUTS = XB_llamaMiniMaxPreset._CUTS
|
||
_MUSIC = XB_llamaMiniMaxPreset._MUSIC
|
||
_MUSIC_HINTS = XB_llamaMiniMaxPreset._MUSIC_HINTS
|
||
_ASPECTS = XB_llamaMiniMaxPreset._ASPECTS
|
||
_ASPECT_HINTS = XB_llamaMiniMaxPreset._ASPECT_HINTS
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"预设模式": ([
|
||
"多参考生成音视频[英文]-Ref2VA [EN]",
|
||
"多参考生成音视频[中文]-Ref2VA [ZH]",
|
||
],),
|
||
"参考图片介绍": ("STRING", {"default": "", "multiline": True,
|
||
"placeholder": "输入参考图片描述，例：图一男主角特写，图二女主角全身，图三背景街道..."}),
|
||
"参考视频介绍": ("STRING", {"default": "", "multiline": True,
|
||
"placeholder": "输入参考视频描述，例：视频一运镜参考，视频二角色动作参考..."}),
|
||
"参考音频介绍": ("STRING", {"default": "", "multiline": True,
|
||
"placeholder": "输入参考音频描述，例：音频一男主音色参考，音频二女主音色参考..."}),
|
||
"视频时长": ("INT", {"default": 8, "min": 4, "max": 60, "step": 1,
|
||
"tooltip": "视频时长 (秒), MiniMax H3 支持 4–60 秒"}),
|
||
"视觉风格": (s._STYLES, {"default": "保持统一风格 / Consistent Style"}),
|
||
"音乐风格": (s._MUSIC, {"default": "禁止音乐 / No Music"}),
|
||
"画面比例": (s._ASPECTS, {"default": "16:9"}),
|
||
"切镜次数": (s._CUTS, {"default": "不指定 / Unspecified"}),
|
||
}
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING",)
|
||
RETURN_NAMES = ("system_prompt",)
|
||
FUNCTION = "build"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def build(self, 预设模式, 参考图片介绍, 参考视频介绍, 参考音频介绍, 视频时长, 视觉风格, 音乐风格, 画面比例, 切镜次数):
|
||
preset = 预设模式
|
||
imgs = 参考图片介绍
|
||
vids = 参考视频介绍
|
||
auds = 参考音频介绍
|
||
duration = 视频时长
|
||
style = 视觉风格
|
||
music = 音乐风格
|
||
aspect = 画面比例
|
||
cuts = 切镜次数
|
||
|
||
match preset:
|
||
case "多参考生成音视频[英文]-Ref2VA [EN]": base = MINIMAX_REF2VA_EN
|
||
case "多参考生成音视频[中文]-Ref2VA [ZH]": base = MINIMAX_REF2VA_ZH
|
||
case _: raise ValueError(f'未知预设: "{preset}"')
|
||
|
||
is_zh = "[ZH]" in preset
|
||
params = []
|
||
|
||
# ── 参考素材注入 ──
|
||
if is_zh:
|
||
params.append("## 目标视频参数（必须严格遵守）")
|
||
if imgs.strip():
|
||
params.append(f"- 参考图片说明：{imgs.strip()}")
|
||
if vids.strip():
|
||
params.append(f"- 参考视频说明：{vids.strip()}")
|
||
if auds.strip():
|
||
params.append(f"- 参考音频说明：{auds.strip()}")
|
||
params.append(f"- 视频时长：正好 {duration} 秒（镜头切分和时间戳必须精确落在此范围内，最后一个镜头必须在第{duration}秒前结束）")
|
||
hint = self._STYLE_HINTS.get(style, "")
|
||
params.append(f"- 视觉风格策略：{style.split('/')[-1].strip()} ({style.split('/')[0].strip()}) — {hint}")
|
||
if "不指定" not in music:
|
||
zh_name = music.split(" / ")[-1]
|
||
hint_m = self._MUSIC_HINTS.get(music, "")
|
||
params.append(f"- 背景音乐风格：{zh_name} — {hint_m}")
|
||
if "禁止音乐" in music:
|
||
params.append(f" ⚠️ 整个视频不得出现任何背景音乐。non_diegetic_music 必须严格输出 \"N/A\"。")
|
||
if "不指定" not in cuts:
|
||
if "不切镜" in cuts:
|
||
params.append(f"- 切镜：固定镜头，不切镜。整段视频只有 [Shot 1] 一个镜头。仅通过运镜改变视角。严禁输出 [Shot N] 时间戳。")
|
||
else:
|
||
n = int(cuts.split(" ")[0])
|
||
max_ok = max(1, duration // 2)
|
||
if n > max_ok:
|
||
params.append(f"- ⚠️ 切镜次数 {n} 对于 {duration} 秒视频过多（合理上限 {max_ok} 次）。请自行削减。")
|
||
params.append(f"- 切镜：正好 {n} 次（共 {n+1} 个镜头）。切镜时机遵循叙事节奏，不可均分。每次切镜必须以 [Shot N] At MM:SS.mmm 开头。")
|
||
hint_a = self._ASPECT_HINTS.get(aspect, "")
|
||
params.append(f"- 画面比例：{aspect} — {hint_a}")
|
||
params.append(f" 所有镜头构图、主体位置、留白空间必须匹配 {aspect} 比例。")
|
||
else:
|
||
params.append("## Target Video Parameters (MUST follow exactly)")
|
||
if imgs.strip():
|
||
params.append(f"- Reference image notes: {imgs.strip()}")
|
||
if vids.strip():
|
||
params.append(f"- Reference video notes: {vids.strip()}")
|
||
if auds.strip():
|
||
params.append(f"- Reference audio notes: {auds.strip()}")
|
||
params.append(f"- Duration: exactly {duration} seconds (all shot timestamps MUST fit within this range; the final shot must end before the {duration}-second mark)")
|
||
hint = self._STYLE_HINTS.get(style, "")
|
||
params.append(f"- Visual style strategy: {style.split('/')[0].strip()} — {hint}")
|
||
if "Unspecified" not in music:
|
||
en_name = music.split(" / ")[0]
|
||
hint_m = self._MUSIC_HINTS.get(music, "")
|
||
params.append(f"- Background music style: {en_name} — {hint_m}")
|
||
if "No Music" in music:
|
||
params.append(f" ⚠️ The video must have absolutely no background music. non_diegetic_music MUST be \"N/A\".")
|
||
if "Unspecified" not in cuts:
|
||
if "Single Shot" in cuts:
|
||
params.append(f"- Cuts: Single continuous shot — NO cuts. Only [Shot 1]. Use camera movement only.")
|
||
else:
|
||
n = int(cuts.split(" ")[0])
|
||
max_ok = max(1, duration // 2)
|
||
if n > max_ok:
|
||
params.append(f"- ⚠️ {n} cuts is excessive for a {duration}-second video (reasonable max: {max_ok}). Reduce.")
|
||
params.append(f"- Cuts: exactly {n} cut(s) (meaning {n+1} total shots). Cut timing follows narrative rhythm — do NOT space evenly. Every cut MUST begin with [Shot N] At MM:SS.mmm.")
|
||
hint_a = self._ASPECT_HINTS.get(aspect, "")
|
||
params.append(f"- Aspect ratio: {aspect} — {hint_a}")
|
||
params.append(f" All shot compositions must be framed for {aspect}.")
|
||
params.append("")
|
||
param_block = "\n".join(params)
|
||
|
||
marker = "## "
|
||
idx = base.find(marker)
|
||
if idx > 0:
|
||
result = base[:idx].rstrip() + "\n\n" + param_block + base[idx:]
|
||
else:
|
||
result = param_block + "\n" + base
|
||
|
||
return (result,)
|
||
|
||
|
||
class XB_llamaStoryboardEnhancer:
|
||
"""分镜增强预设 — 模块化动态拼装 System Prompt"""
|
||
|
||
_MODEL_KEYS = list(MODEL_RULES.keys())
|
||
_STYLE_KEYS = list(STYLE_RULES.keys())
|
||
_CAMERA_KEYS = list(CAMERA_LOGIC.keys())
|
||
_LANG_KEYS = ["英文[EN]", "中文[ZH]"]
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"model_target": (s._MODEL_KEYS, {
|
||
"default": s._MODEL_KEYS[0],
|
||
"tooltip": "选择目标生成模型, 后台自动注入对应的语法规则。\nFlux2-Klein: 自然语言散文流\nQwen-2511: 结构化 DiT 流\nWan/LTX: 视频运镜流\nAnimagine/Pony: 二次元标签流"
|
||
}),
|
||
"story_style": (s._STYLE_KEYS, {
|
||
"default": s._STYLE_KEYS[0],
|
||
"tooltip": "强制绑定全局美术风格、光影基调和色彩方案, 防止风格跑偏"
|
||
}),
|
||
"camera_logic": (s._CAMERA_KEYS, {
|
||
"default": s._CAMERA_KEYS[0],
|
||
"tooltip": "镜头组接逻辑: 控制空间变换方式和景别推进节奏\n固定场景: 单一空间, 景别收缩\n动态跟随: 锁定主角, 空间流动\n史诗跳跃: 大尺度场景跳跃\n情感对话: 心理距离驱动"
|
||
}),
|
||
"character_anchor": ("STRING", {
|
||
"default": "",
|
||
"multiline": True,
|
||
"placeholder": "主角锁定描述 (强烈建议填写!)\n如: 一只穿着蓝色背带裤、戴着草帽的白毛胖兔子"
|
||
}),
|
||
"frame_count": ("INT", {
|
||
"default": 6,
|
||
"min": 2, "max": 16, "step": 1,
|
||
"tooltip": "生成的分镜帧数 (2~16)\n四格漫画用4帧, 短视频用6~8帧, 长视频预演用12~16帧"
|
||
}),
|
||
"language": (s._LANG_KEYS, {
|
||
"default": s._LANG_KEYS[0],
|
||
"tooltip": "输出提示词的语言\nEnglish: 英文输出 (兼容所有模型)\n中文: 中文输出 (适合 Qwen-Image / ERNIE-Image 等中文原生模型)"
|
||
}),
|
||
"user_story": ("STRING", {
|
||
"default": "",
|
||
"multiline": True,
|
||
"placeholder": "输入你的故事、场景或灵感片段...\n支持中文/英文, 一段话即可"
|
||
}),
|
||
}
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING",)
|
||
RETURN_NAMES = ("system_prompt",)
|
||
FUNCTION = "main"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def main(self, model_target, story_style, camera_logic, character_anchor, frame_count, language, user_story):
|
||
if not user_story.strip():
|
||
raise ValueError("请在 user_story 输入框中填写你的故事或场景描述!")
|
||
|
||
final_prompt = build_storyboard_prompt(
|
||
character_anchor=character_anchor,
|
||
frame_count=frame_count,
|
||
model_target=model_target,
|
||
story_style=story_style,
|
||
camera_logic=camera_logic,
|
||
language=language,
|
||
user_story=user_story,
|
||
)
|
||
return (final_prompt,)
|
||
|
||
|
||
class XB_llamaStoryboardInstruct:
|
||
"""分镜推理 — 融合分镜增强与指令推理的一体化节点"""
|
||
|
||
# 仅分镜推理节点使用的简化模型列表
|
||
_MODEL_KEYS = ["Flux2-Klein", "Qwen-Edit"]
|
||
_STYLE_KEYS = list(STYLE_RULES.keys())
|
||
_CAMERA_KEYS = list(CAMERA_LOGIC.keys())
|
||
_LANG_KEYS = ["英文[EN]", "中文[ZH]"]
|
||
_PRESET_MODES = ["分镜模式 (Storyboard)", "故事模式 (Story Writing)"]
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(s):
|
||
return {
|
||
"required": {
|
||
"llama_model": ("LLAMACPPMODEL",),
|
||
"preset_mode": (s._PRESET_MODES, {
|
||
"default": s._PRESET_MODES[0],
|
||
"tooltip": "分镜模式: 生成N帧分镜提示词序列, 每帧一行, 可直连 CLIPTextEncode 批量出图\n故事模式: 根据设定创作配有分镜插图的短篇故事"
|
||
}),
|
||
"model_target": (s._MODEL_KEYS, {
|
||
"default": s._MODEL_KEYS[0],
|
||
"tooltip": "仅在分镜模式下生效 — 选择目标生成模型的对齐语法"
|
||
}),
|
||
"story_style": (s._STYLE_KEYS, {
|
||
"default": s._STYLE_KEYS[0],
|
||
"tooltip": "全局美术风格与光影基调\n选'自定义'则不施加风格约束"
|
||
}),
|
||
"camera_logic": (s._CAMERA_KEYS, {
|
||
"default": s._CAMERA_KEYS[0],
|
||
"tooltip": "镜头组接与空间变换逻辑"
|
||
}),
|
||
"frame_count": ("INT", {
|
||
"default": 6, "min": 2, "max": 16, "step": 1,
|
||
"tooltip": "分镜模式: 生成的分镜帧数\n故事模式: 故事配图数量"
|
||
}),
|
||
"language": (s._LANG_KEYS, {
|
||
"default": s._LANG_KEYS[0],
|
||
"tooltip": "输出提示词/故事的语言"
|
||
}),
|
||
"character_anchor": ("STRING", {
|
||
"default": "", "multiline": True,
|
||
"placeholder": "👤 角色设定\n如: 一只穿着蓝色背带裤、戴着草帽的白毛胖兔子"
|
||
}),
|
||
"background_setting": ("STRING", {
|
||
"default": "", "multiline": True,
|
||
"placeholder": "🏞️ 背景设定\n如: 魔法森林深处, 住着会说话的蘑菇和小精灵"
|
||
}),
|
||
"story_synopsis": ("STRING", {
|
||
"default": "", "multiline": True,
|
||
"placeholder": "📝 故事梗概\n如: 小兔子在拔萝卜时发现了一颗会发光的魔法萝卜..."
|
||
}),
|
||
"inference_mode": (["one by one", "images", "video"], {
|
||
"default": "images",
|
||
"tooltip": "one by one: 逐张读取 (多图时每张单独推理)\nimages: 一次性读取所有图片\nvideo: 将输入图像视为视频帧"
|
||
}),
|
||
"max_frames": ("INT", {
|
||
"default": 24, "min": 2, "max": 1024, "step": 1,
|
||
"tooltip": "从输入视频中均匀采样的帧数 (仅 video 模式)"
|
||
}),
|
||
"max_size": ("INT", {
|
||
"default": 384, "min": 128, "max": 16384, "step": 64,
|
||
"tooltip": "images/video 模式下输入图像的最大尺寸"
|
||
}),
|
||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "step": 1}),
|
||
"force_offload": ("BOOLEAN", {
|
||
"default": False,
|
||
"tooltip": "推理后卸载模型以释放显存"
|
||
}),
|
||
"save_states": ("BOOLEAN", {
|
||
"default": False,
|
||
"tooltip": "在内存中保留此对话的上下文"
|
||
}),
|
||
},
|
||
"optional": {
|
||
"parameters": ("LLAMACPPARAMS",),
|
||
"images": ("IMAGE",),
|
||
},
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING", "STRING")
|
||
RETURN_NAMES = ("output", "output_list")
|
||
OUTPUT_IS_LIST = (False, True)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
# ── 故事模式 System Prompt (增强版) ──
|
||
STORY_MODE_SKELETON = '''# Role: 短篇故事作家 & 视觉叙事大师
|
||
You are a master storyteller who crafts immersive short stories. Your task is to write a complete short story based on the provided character, setting, and synopsis. The story will be accompanied by {Frame_Count} vivid illustration scenes placed at key narrative moments.
|
||
|
||
## 风格基调
|
||
{Style_Rules}
|
||
|
||
## 输出语言
|
||
{Language_Rule}
|
||
|
||
## 写作要求
|
||
1. 写一个完整的短篇故事，有开头、发展、高潮和结尾。故事长度控制在800~1500字。
|
||
2. 在故事的关键转折点、精彩场景或情绪高峰处，插入 {Frame_Count} 个配图标记。
|
||
3. 每个配图标记独占一行，格式为: {Frame_Prefix}一段完整的、可直接用于AI绘图的视觉化场景描述
|
||
4. 配图描述需要包含: 景别、主体外貌动作、环境细节、光影氛围、艺术风格。必须是一段流畅的自然语言。末尾必须附带两个标签: 先写"背景A"+当前场景的背景描述，再写每个出场角色的完整描述（如"角色A..."、"角色B..."）。谁在画面里就标注谁，不出场的不标。背景标签始终保留。
|
||
5. 故事正文和配图标记交织呈现: 先写一段故事，遇到精彩场景就插入配图标记，然后继续写故事。
|
||
6. 开头的第1个配图标记用于确立世界观和主角形象，最后的配图标记用于故事收尾的经典画面。
|
||
7. 不输出任何开场白、解释或结尾总结。直接从故事正文开始。
|
||
|
||
## 人物设定
|
||
{Character_Anchor}
|
||
|
||
## 背景设定
|
||
{Background_Setting}
|
||
|
||
## 故事梗概
|
||
{Story_Synopsis}
|
||
'''
|
||
|
||
@staticmethod
|
||
def _timed_completion(llm, label, **kwargs):
|
||
import time
|
||
t0 = time.perf_counter()
|
||
output = llm.create_chat_completion(**kwargs)
|
||
t1 = time.perf_counter()
|
||
elapsed = t1 - t0
|
||
try:
|
||
tokens = output.get("usage", {}).get("completion_tokens", 0)
|
||
except Exception:
|
||
tokens = 0
|
||
tps = tokens / elapsed if elapsed > 0 else 0
|
||
print(f"[XB-llama] {label} → {tokens} tokens / {elapsed:.2f}s = {tps:.1f} T/s")
|
||
return output
|
||
|
||
def _build_system_prompt(self, preset_mode, model_target, story_style, camera_logic,
|
||
frame_count, language, character_anchor, background_setting, story_synopsis):
|
||
"""根据模式构建系统提示词"""
|
||
lang_rule = LANGUAGE_RULES.get(language, LANGUAGE_RULES["英文[EN]"])
|
||
safe_anchor = character_anchor.strip() or "the main subject"
|
||
safe_bg = background_setting.strip() or "an undefined setting"
|
||
safe_synopsis = story_synopsis.strip() or "Please create a narrative based on the given character and setting."
|
||
|
||
if preset_mode == "分镜模式 (Storyboard)":
|
||
prompt = build_storyboard_prompt(
|
||
character_anchor=safe_anchor,
|
||
frame_count=frame_count,
|
||
model_target=model_target,
|
||
story_style=story_style,
|
||
camera_logic=camera_logic,
|
||
language=language,
|
||
user_story=f"Character: {safe_anchor}\nSetting: {safe_bg}\nSynopsis: {safe_synopsis}",
|
||
)
|
||
# 追加输出格式指令 — 使用 Scene_NN / 分镜图_NN 格式
|
||
if language == "中文[ZH]":
|
||
fmt = (
|
||
f"\n\n## 8. 输出格式修正 (Output Format Override)\n"
|
||
f"上述第8条中的XML标签格式作废。请使用以下新格式:\n"
|
||
f"- 每帧独占一行, 格式为: 分镜图_NN：提示词文本\n"
|
||
f"- NN 为两位数字 (01, 02, ..., {frame_count:02d})\n"
|
||
f"- 不要使用 <frame_N> XML 标签\n"
|
||
f"- 不要输出任何额外文字或解释\n"
|
||
f"示例:\n"
|
||
f"分镜图_01：全景镜头, 一只白毛胖兔子戴着草帽站在魔法森林入口...\n"
|
||
f"分镜图_02：中景镜头, 兔子蹲在一颗发光的金色萝卜前..."
|
||
)
|
||
else:
|
||
fmt = (
|
||
f"\n\n## 8. Output Format Override\n"
|
||
f"The XML tag format in section 8 above is VOID. Use this new format instead:\n"
|
||
f"- One frame per line, format: Scene_NN: prompt text\n"
|
||
f"- NN is a two-digit number (01, 02, ..., {frame_count:02d})\n"
|
||
f"- Do NOT use <frame_N> XML tags\n"
|
||
f"- Do NOT output any extra text or explanations\n"
|
||
f"Example:\n"
|
||
f"Scene_01: Wide shot, a white fluffy rabbit wearing a straw hat standing at the entrance of a magical forest...\n"
|
||
f"Scene_02: Medium shot, the rabbit crouching before a glowing golden carrot..."
|
||
)
|
||
return prompt + fmt
|
||
|
||
else: # 故事模式
|
||
style_rule = STYLE_RULES.get(story_style, STYLE_RULES["治愈绘本 (Fairy Tale Illustration)"])
|
||
frame_prefix = "分镜图_NN：" if language == "中文[ZH]" else "Scene_NN:"
|
||
prompt = self.STORY_MODE_SKELETON.format(
|
||
Frame_Count=str(frame_count),
|
||
Style_Rules=style_rule,
|
||
Language_Rule=lang_rule,
|
||
Frame_Prefix=frame_prefix,
|
||
Character_Anchor=safe_anchor,
|
||
Background_Setting=safe_bg,
|
||
Story_Synopsis=safe_synopsis,
|
||
)
|
||
return prompt
|
||
|
||
def process(self, llama_model, preset_mode, model_target, story_style, camera_logic,
|
||
frame_count, language, character_anchor, background_setting, story_synopsis,
|
||
inference_mode, max_frames, max_size, seed, force_offload, save_states,
|
||
parameters=None, images=None):
|
||
# ── 模型加载 ──
|
||
if not LLAMA_CPP_STORAGE.llm:
|
||
LLAMA_CPP_STORAGE.load_model(llama_model)
|
||
|
||
if parameters is None:
|
||
parameters = {}
|
||
_params = parameters.copy()
|
||
_params.pop("state_uid", None)
|
||
|
||
# ── 构建 System Prompt ──
|
||
system_prompt = self._build_system_prompt(
|
||
preset_mode, model_target, story_style, camera_logic,
|
||
frame_count, language, character_anchor, background_setting, story_synopsis,
|
||
)
|
||
|
||
video_input = inference_mode == "video"
|
||
if video_input and images is not None:
|
||
system_prompt = "请将输入的图片序列当做视频而不是静态帧序列。\n" + system_prompt
|
||
|
||
messages = [{"role": "system", "content": system_prompt}]
|
||
|
||
# ── 构建用户消息 ──
|
||
user_content = []
|
||
parts = []
|
||
if character_anchor.strip():
|
||
parts.append(f"Character: {character_anchor.strip()}")
|
||
if background_setting.strip():
|
||
parts.append(f"Setting: {background_setting.strip()}")
|
||
if story_synopsis.strip():
|
||
parts.append(f"Story: {story_synopsis.strip()}")
|
||
user_text = "\n".join(parts) if parts else "Please generate based on the system instructions."
|
||
user_content.append({"type": "text", "text": user_text})
|
||
|
||
# ── 图像处理 ──
|
||
has_images = images is not None
|
||
if has_images:
|
||
_ch = LLAMA_CPP_STORAGE.chat_handler
|
||
if _ch is None:
|
||
raise ValueError("检测到图像输入, 但 chat_handler=None (纯文本模型)!\n请选择支持视觉的 chat_handler。")
|
||
if hasattr(_ch, "clip_model_path") and _ch.clip_model_path is None:
|
||
raise ValueError("检测到图像输入, 但 mmproj 未正确加载!\n请确认在模型加载器中选择了正确的 mmproj 文件。")
|
||
|
||
frames = images
|
||
if video_input:
|
||
indices = np.linspace(0, len(images) - 1, max_frames, dtype=int)
|
||
frames = [images[i] for i in indices]
|
||
|
||
if inference_mode == "one by one":
|
||
image_content = {"type": "image_url", "image_url": {"url": ""}}
|
||
user_content.append(image_content)
|
||
messages.append({"role": "user", "content": user_content})
|
||
print(f"[XB-llama] 开始处理 {len(frames)} 张图像 (one by one)")
|
||
|
||
import time
|
||
_total_tokens = 0
|
||
_t0 = time.perf_counter()
|
||
tmp_list = []
|
||
out2 = []
|
||
for i, image in enumerate(cqdm(frames)):
|
||
if mm.processing_interrupted():
|
||
raise mm.InterruptProcessingException()
|
||
data = image2base64(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
|
||
for item in user_content:
|
||
if item.get("type") == "image_url":
|
||
item["image_url"]["url"] = f"data:image/jpeg;base64,{data}"
|
||
break
|
||
output = LLAMA_CPP_STORAGE.llm.create_chat_completion(messages=messages, seed=seed, **_params)
|
||
text = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out2.append(text)
|
||
if len(frames) > 1:
|
||
tmp_list.append(f"====== Image {i + 1} ======")
|
||
tmp_list.append(text)
|
||
try:
|
||
_total_tokens += output.get("usage", {}).get("completion_tokens", 0)
|
||
except Exception:
|
||
pass
|
||
_elapsed = time.perf_counter() - _t0
|
||
_tps = _total_tokens / _elapsed if _elapsed > 0 else 0
|
||
print(f"[XB-llama] one by one 完成 → {_total_tokens} tokens / {_elapsed:.2f}s = {_tps:.1f} T/s")
|
||
out1 = "\n\n".join(tmp_list)
|
||
else:
|
||
for image in frames:
|
||
if len(frames) > 1:
|
||
data = image2base64(np.clip(255.0 * scale_image(image, max_size), 0, 255).astype(np.uint8))
|
||
else:
|
||
data = image2base64(np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
|
||
image_content = {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{data}"}}
|
||
user_content.append(image_content)
|
||
|
||
messages.append({"role": "user", "content": user_content})
|
||
output = self._timed_completion(LLAMA_CPP_STORAGE.llm, f"{inference_mode} mode", messages=messages, seed=seed, **_params)
|
||
out1 = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out1 = _merge_frame_lines(out1)
|
||
out1 = _re.sub(r'(背景|角色)\s+([A-E])', r'\1\2', out1)
|
||
out2 = [line for line in out1.split('\n') if line.strip()]
|
||
else:
|
||
messages.append({"role": "user", "content": user_content})
|
||
output = self._timed_completion(LLAMA_CPP_STORAGE.llm, "text-only", messages=messages, seed=seed, **_params)
|
||
out1 = output['choices'][0]['message']['content'].removeprefix(": ").lstrip()
|
||
out1 = _merge_frame_lines(out1)
|
||
out1 = _re.sub(r'(背景|角色)\s+([A-E])', r'\1\2', out1)
|
||
out2 = [line for line in out1.split('\n') if line.strip()]
|
||
|
||
# ── 后处理 ──
|
||
if force_offload:
|
||
LLAMA_CPP_STORAGE.clean()
|
||
else:
|
||
if LLAMA_CPP_STORAGE.current_config.get("chat_handler", "") in ["Qwen3.5", "Qwen3.5-Thinking"]:
|
||
try:
|
||
LLAMA_CPP_STORAGE.llm.n_tokens = 0
|
||
LLAMA_CPP_STORAGE.llm._ctx.memory_clear(True)
|
||
if LLAMA_CPP_STORAGE.llm.is_hybrid and LLAMA_CPP_STORAGE.llm._hybrid_cache_mgr is not None:
|
||
LLAMA_CPP_STORAGE.llm._hybrid_cache_mgr.clear()
|
||
except Exception:
|
||
pass
|
||
|
||
del messages
|
||
gc.collect()
|
||
return (out1, out2)
|
||
|
||
|
||
# =============================================================================
|
||
# 分镜输出标准化: 将多行分镜合并为每帧一行
|
||
# =============================================================================
|
||
|
||
import re as _re
|
||
|
||
def _merge_frame_lines(text: str) -> str:
|
||
"""将分镜提示词中的换行合并：以分镜标记为锚点，每帧 = 一行"""
|
||
if not text:
|
||
return text
|
||
# 分镜标记: 分镜图_NN： / 分镜图_NN: / Scene_NN: / <frame_N>
|
||
frame_pattern = _re.compile(r'^(分镜图_\d+[：:]|Scene_\d+:|<frame_\d+>)')
|
||
|
||
lines = text.split('\n')
|
||
merged = []
|
||
buf = ""
|
||
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if not stripped:
|
||
continue
|
||
if frame_pattern.match(stripped):
|
||
if buf:
|
||
merged.append(buf)
|
||
buf = stripped
|
||
else:
|
||
if buf:
|
||
buf += stripped # 接在前一帧后面
|
||
else:
|
||
buf = stripped # 没有前导标记的行, 可能是 XML 标签外的文本, 跳过或保留
|
||
|
||
if buf:
|
||
merged.append(buf)
|
||
|
||
return '\n'.join(merged)
|
||
|
||
|
||
class XB_llamaStoryboardProcessor:
|
||
"""分镜词处理器 — Show Text 显示 + 抽卡模式切换 + 分镜范围选择"""
|
||
|
||
_cached_text = ""
|
||
_cached_frames = []
|
||
|
||
@classmethod
|
||
def _parse_frames(cls, text):
|
||
import re
|
||
frames = []
|
||
for line in text.split('\n'):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
m = re.match(r'(分镜图_\d+)', line)
|
||
if m:
|
||
frames.append((m.group(1), line))
|
||
continue
|
||
m = re.match(r'(Scene_\d+)', line)
|
||
if m:
|
||
frames.append((m.group(1), line))
|
||
return frames
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
draw_options = ["全部"] + [f"分镜图_{i:02d}" for i in range(1, 17)]
|
||
return {
|
||
"required": {
|
||
"draw_mode": ("BOOLEAN", {
|
||
"default": False,
|
||
"label_on": "开启", "label_off": "关闭",
|
||
}),
|
||
"draw_range": (draw_options, {"default": "全部"}),
|
||
},
|
||
"optional": {
|
||
"text_input": ("*", {"force_input": True}),
|
||
},
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING",)
|
||
RETURN_NAMES = ("output_list",)
|
||
OUTPUT_IS_LIST = (True,)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
def process(self, draw_mode, draw_range, text_input=None):
|
||
# ── 缓存 & 显示文本 ──
|
||
if text_input:
|
||
frames = self._parse_frames(text_input)
|
||
if frames:
|
||
XB_llamaStoryboardProcessor._cached_frames = frames
|
||
XB_llamaStoryboardProcessor._cached_text = text_input
|
||
display_text = text_input
|
||
else:
|
||
# 上游被 mute 时无新输入, 使用缓存文本保持预览
|
||
display_text = XB_llamaStoryboardProcessor._cached_text or ""
|
||
|
||
if not draw_mode:
|
||
# 关闭模式: text_input → 逐行列表输出
|
||
lines = [line for line in display_text.split('\n') if line.strip()]
|
||
if not lines:
|
||
lines = ["[等待输入...]"]
|
||
return {"ui": {"text": [display_text]}, "result": (lines,)}
|
||
|
||
# 开启模式: 绕过上游, 从缓存中按 draw_range 取词
|
||
if not XB_llamaStoryboardProcessor._cached_frames:
|
||
return {"ui": {"text": [display_text]}, "result": (["[抽卡] 暂无缓存, 请先关闭抽卡模式运行一次"],)}
|
||
|
||
if draw_range == "全部":
|
||
result = [prompt for _, prompt in XB_llamaStoryboardProcessor._cached_frames]
|
||
else:
|
||
result = []
|
||
for name, prompt in XB_llamaStoryboardProcessor._cached_frames:
|
||
if name == draw_range:
|
||
result.append(prompt)
|
||
break
|
||
if not result:
|
||
result = [f"生成一张纯色背景图片，正中央用大号白色粗体文字写着：未找到{draw_range}"]
|
||
return {"ui": {"text": [display_text]}, "result": (result,)}
|
||
|
||
|
||
class XB_llamaStoryboardProcessorPro:
|
||
"""分镜词处理器Pro — 完全复刻原版 + 背景/角色嗅探输出"""
|
||
|
||
_cached_text = ""
|
||
_cached_frames = []
|
||
|
||
@classmethod
|
||
def _parse_frames(cls, text):
|
||
import re
|
||
frames = []
|
||
for line in text.split('\n'):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
m = re.match(r'(分镜图_\d+)', line)
|
||
if m:
|
||
frames.append((m.group(1), line))
|
||
continue
|
||
m = re.match(r'(Scene_\d+)', line)
|
||
if m:
|
||
frames.append((m.group(1), line))
|
||
return frames
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
draw_options = ["全部"] + [f"分镜图_{i:02d}" for i in range(1, 17)]
|
||
return {
|
||
"required": {
|
||
"draw_mode": ("BOOLEAN", {
|
||
"default": False,
|
||
"label_on": "开启", "label_off": "关闭",
|
||
}),
|
||
"draw_range": (draw_options, {"default": "全部"}),
|
||
},
|
||
"optional": {
|
||
"text_input": ("*", {"force_input": True}),
|
||
},
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING", "STRING", "STRING", "STRING", "STRING")
|
||
RETURN_NAMES = ("output_list", "背景", "角色_1", "角色_2", "角色_3")
|
||
OUTPUT_IS_LIST = (True, False, False, False, False)
|
||
FUNCTION = "process"
|
||
CATEGORY = "XB-llama"
|
||
|
||
@staticmethod
|
||
def _sniff_all(prompt, pattern):
|
||
"""嗅探提示词中所有匹配项, 按出现顺序返回截断到10字的列表"""
|
||
import re
|
||
matches = re.findall(pattern, prompt)
|
||
return [(m[:10] if len(m) > 10 else m) for m in matches]
|
||
|
||
# 非贪婪正则: 匹配到下一个标签或字符串末尾为止
|
||
_RE_BG = r'背景[ABCDE].*?(?=角色[ABCDE]|背景[ABCDE]|$)'
|
||
_RE_CH = r'角色[ABCDE].*?(?=角色[ABCDE]|背景[ABCDE]|$)'
|
||
|
||
def process(self, draw_mode, draw_range, text_input=None):
|
||
# ── 完全复刻原版逻辑 ──
|
||
if text_input:
|
||
frames = self._parse_frames(text_input)
|
||
if frames:
|
||
XB_llamaStoryboardProcessorPro._cached_frames = frames
|
||
XB_llamaStoryboardProcessorPro._cached_text = text_input
|
||
display_text = text_input
|
||
else:
|
||
display_text = XB_llamaStoryboardProcessorPro._cached_text or ""
|
||
|
||
if not draw_mode:
|
||
lines = [line for line in display_text.split('\n') if line.strip()]
|
||
if not lines:
|
||
lines = ["[等待输入...]"]
|
||
result = lines
|
||
elif not XB_llamaStoryboardProcessorPro._cached_frames:
|
||
result = ["[抽卡] 暂无缓存, 请先关闭抽卡模式运行一次"]
|
||
elif draw_range == "全部":
|
||
result = [prompt for _, prompt in XB_llamaStoryboardProcessorPro._cached_frames]
|
||
else:
|
||
result = []
|
||
for name, prompt in XB_llamaStoryboardProcessorPro._cached_frames:
|
||
if name == draw_range:
|
||
result.append(prompt)
|
||
break
|
||
if not result:
|
||
result = [f"生成一张纯色背景图片，正中央用大号白色粗体文字写着：未找到{draw_range}"]
|
||
|
||
# ── 嗅探当前帧 (先归一化空格: 背景 A → 背景A) ──
|
||
target_prompt = result[0] if result else ""
|
||
import re as _re2
|
||
target_prompt = _re2.sub(r'(背景|角色)\s+([A-E])', r'\1\2', target_prompt)
|
||
bg_matches = self._sniff_all(target_prompt, XB_llamaStoryboardProcessorPro._RE_BG)
|
||
ch_raw = self._sniff_all(target_prompt, XB_llamaStoryboardProcessorPro._RE_CH)
|
||
|
||
# 去重: 只保留首次出现的角色 (如 角色A/角色C/角色A → 角色A/角色C)
|
||
seen = set()
|
||
ch_matches = []
|
||
for c in ch_raw:
|
||
key = c[:3] # "角色A" / "角色B" 等前3字为唯一标识
|
||
if key not in seen:
|
||
seen.add(key)
|
||
ch_matches.append(c)
|
||
|
||
bg_str = bg_matches[0] if bg_matches else "全部关闭"
|
||
ch1_str = ch_matches[0] if len(ch_matches) > 0 else "全部关闭"
|
||
ch2_str = ch_matches[1] if len(ch_matches) > 1 else "全部关闭"
|
||
ch3_str = ch_matches[2] if len(ch_matches) > 2 else "全部关闭"
|
||
|
||
return {
|
||
"ui": {
|
||
"text": [display_text],
|
||
"sniff_bg": bg_str,
|
||
"sniff_ch1": ch1_str,
|
||
"sniff_ch2": ch2_str,
|
||
"sniff_ch3": ch3_str,
|
||
},
|
||
"result": (result, bg_str, ch1_str, ch2_str, ch3_str)
|
||
}
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# 灵活输入类型
|
||
# ═══════════════════════════════════════════════════════════════
|
||
|
||
class _AnyType(str):
|
||
def __ne__(self, other):
|
||
return False
|
||
|
||
class _FlexibleInputType(dict):
|
||
def __init__(self, type_):
|
||
self.type_ = type_
|
||
def __getitem__(self, key):
|
||
return (self.type_,)
|
||
def __contains__(self, key):
|
||
return True
|
||
|
||
|
||
class XB_RoleSceneDispatcher:
|
||
"""角色场景调度器 — 逐帧配对提示词+背景+角色, ComfyUI Zip 对齐"""
|
||
|
||
_cached_text = ""
|
||
_cached_frames = []
|
||
_RE_BG = r'背景[ABCDE].*?(?=角色[ABCDE]|背景[ABCDE]|$)'
|
||
_RE_CH = r'角色[ABCDE].*?(?=角色[ABCDE]|背景[ABCDE]|$)'
|
||
|
||
@classmethod
|
||
def _parse_frames(cls, text):
|
||
import re
|
||
frames = []
|
||
for line in text.split('\n'):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
m = re.match(r'(分镜图_\d+)', line)
|
||
if m:
|
||
frames.append((m.group(1), line))
|
||
continue
|
||
m = re.match(r'(Scene_\d+)', line)
|
||
if m:
|
||
frames.append((m.group(1), line))
|
||
return frames
|
||
|
||
@staticmethod
|
||
def _sniff_labels(prompt, pattern):
|
||
import re
|
||
raw = re.findall(pattern, prompt)
|
||
seen = set()
|
||
result = []
|
||
for m in raw:
|
||
key = m[:3]
|
||
if key not in seen:
|
||
seen.add(key)
|
||
result.append(key)
|
||
return result
|
||
|
||
@classmethod
|
||
def INPUT_TYPES(cls):
|
||
draw_options = ["全部"] + [f"分镜图_{i:02d}" for i in range(1, 17)]
|
||
return {
|
||
"required": {
|
||
"text_input": ("*", {"force_input": True}),
|
||
"draw_mode": ("BOOLEAN", {
|
||
"default": False,
|
||
"label_on": "开启", "label_off": "关闭",
|
||
}),
|
||
"draw_range": (draw_options, {"default": "全部"}),
|
||
},
|
||
"optional": _FlexibleInputType("IMAGE"),
|
||
}
|
||
|
||
RETURN_TYPES = ("STRING", "IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE")
|
||
RETURN_NAMES = ("提示词", "背景图", "角色01", "角色02", "角色03", "角色04")
|
||
OUTPUT_IS_LIST = (True, True, True, True, True, True)
|
||
FUNCTION = "execute"
|
||
CATEGORY = "XB-llama"
|
||
|
||
@staticmethod
|
||
def _normalize(text):
|
||
"""归一化标签空格: 背景 A → 背景A, 角色 B → 角色B"""
|
||
import re
|
||
return re.sub(r'(背景|角色)\s+([A-E])', r'\1\2', text)
|
||
|
||
def execute(self, draw_mode, draw_range, text_input=None, **kwargs):
|
||
import torch
|
||
text_input = text_input or ""
|
||
text_input = self._normalize(text_input)
|
||
|
||
# ── 抽卡模式: 缓存 ──
|
||
if text_input:
|
||
frames = self._parse_frames(text_input)
|
||
if frames:
|
||
XB_RoleSceneDispatcher._cached_frames = frames
|
||
XB_RoleSceneDispatcher._cached_text = text_input
|
||
|
||
if not draw_mode:
|
||
prompts = [line for line in text_input.split('\n') if line.strip()]
|
||
elif not XB_RoleSceneDispatcher._cached_frames:
|
||
prompts = ["[抽卡] 暂无缓存, 请先关闭抽卡模式运行一次"]
|
||
elif draw_range == "全部":
|
||
prompts = [p for _, p in XB_RoleSceneDispatcher._cached_frames]
|
||
else:
|
||
prompts = []
|
||
for name, p in XB_RoleSceneDispatcher._cached_frames:
|
||
if name == draw_range:
|
||
prompts.append(p)
|
||
break
|
||
if not prompts:
|
||
prompts = [f"生成一张纯色背景图片，正中央用大号白色粗体文字写着：未找到{draw_range}"]
|
||
|
||
n = len(prompts)
|
||
if n == 0:
|
||
return ([], [], [], [], [], [])
|
||
|
||
# 生成占位空图 (严禁传 None)
|
||
empty_image = None
|
||
for v in kwargs.values():
|
||
if v is not None and isinstance(v, torch.Tensor):
|
||
empty_image = torch.zeros_like(v)
|
||
break
|
||
if empty_image is None:
|
||
empty_image = torch.zeros((1, 64, 64, 3), dtype=torch.float32, device="cpu")
|
||
|
||
bg_out, c1_out, c2_out, c3_out, c4_out = [], [], [], [], []
|
||
|
||
for prompt in prompts:
|
||
bg_labels = self._sniff_labels(prompt, self._RE_BG)
|
||
bg_tensor = empty_image
|
||
for lbl in bg_labels:
|
||
if lbl in kwargs and kwargs[lbl] is not None:
|
||
bg_tensor = kwargs[lbl]
|
||
break
|
||
|
||
ch_labels = self._sniff_labels(prompt, self._RE_CH)
|
||
ch_tensors = []
|
||
for lbl in ch_labels:
|
||
if lbl in kwargs and kwargs[lbl] is not None:
|
||
ch_tensors.append(kwargs[lbl])
|
||
|
||
bg_out.append(bg_tensor)
|
||
c1_out.append(ch_tensors[0] if len(ch_tensors) > 0 else empty_image)
|
||
c2_out.append(ch_tensors[1] if len(ch_tensors) > 1 else empty_image)
|
||
c3_out.append(ch_tensors[2] if len(ch_tensors) > 2 else empty_image)
|
||
c4_out.append(ch_tensors[3] if len(ch_tensors) > 3 else empty_image)
|
||
|
||
display_text = text_input or XB_RoleSceneDispatcher._cached_text or ""
|
||
return {"ui": {"text": [display_text]}, "result": (prompts, bg_out, c1_out, c2_out, c3_out, c4_out)}
|
||
|
||
|
||
# =============================================================================
|
||
# 节点注册
|
||
# =============================================================================
|
||
|
||
NODE_CLASS_MAPPINGS = {
|
||
"XB_llamaModelLoader": XB_llamaModelLoader,
|
||
"XB_llamaInstruct": XB_llamaInstruct,
|
||
"XB_llamaParameters": XB_llamaParameters,
|
||
"XB_llamaUnloadModel": XB_llamaUnloadModel,
|
||
"XB_llamaCleanStates": XB_llamaCleanStates,
|
||
"XB_llamaBBoxes2BBox": XB_llamaBBoxes2BBox,
|
||
"XB_llamaUnpackCodeBlock": XB_llamaUnpackCodeBlock,
|
||
"XB_llamaPromptEnhancer": XB_llamaPromptEnhancer,
|
||
"XB_llamaMiniMaxPreset": XB_llamaMiniMaxPreset,
|
||
"XB_llamaMiniMaxRef2va": XB_llamaMiniMaxRef2vaPreset,
|
||
"XB_llamaStoryboardEnhancer": XB_llamaStoryboardEnhancer,
|
||
"XB_llamaStoryboardInstruct": XB_llamaStoryboardInstruct,
|
||
"XB_llamaStoryboardProcessor": XB_llamaStoryboardProcessor,
|
||
"XB_llamaStoryboardProcessorPro": XB_llamaStoryboardProcessorPro,
|
||
"XB_RoleSceneDispatcher": XB_RoleSceneDispatcher,
|
||
}
|
||
|
||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||
"XB_llamaModelLoader": "XB-llama - 📦 模型加载器",
|
||
"XB_llamaInstruct": "XB-llama - 💬 指令推理",
|
||
"XB_llamaParameters": "XB-llama - ⚙️ 推理参数",
|
||
"XB_llamaUnloadModel": "XB-llama - 🗑️ 卸载模型",
|
||
"XB_llamaCleanStates": "XB-llama - 🧹 清理状态",
|
||
"XB_llamaBBoxes2BBox": "XB-llama - 🔍 BBoxes取BBox",
|
||
"XB_llamaUnpackCodeBlock": "XB-llama - 📝 解包代码块",
|
||
"XB_llamaPromptEnhancer": "XB-llama - ✨ 提示词增强预设",
|
||
"XB_llamaMiniMaxPreset": "XB-llama - ✨ MiniMax-fl2va提示词预设",
|
||
"XB_llamaMiniMaxRef2va": "XB-llama - ✨ MiniMax-ref2va提示词预设",
|
||
"XB_llamaStoryboardEnhancer": "XB-llama - ✨ 分镜增强预设",
|
||
"XB_llamaStoryboardInstruct": "XB-llama - 💬 分镜推理",
|
||
"XB_llamaStoryboardProcessor": "XB-llama - 🎞️ 分镜词处理器",
|
||
"XB_llamaStoryboardProcessorPro": "XB-llama - 🎞️ 分镜词处理器Pro",
|
||
"XB_RoleSceneDispatcher": "XB-llama - 🎬 角色场景调度器",
|
||
}
|