# Copyright (c) 2026 exportAnything. All rights reserved.
# SPDX-License-Identifier: MIT
from __future__ import annotations
import copy
import gc
import json
import logging
import math
import os
import re
import shlex
import subprocess
import threading
import time
import unicodedata
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
CATEGORY = "prompt/diffusiongemma"
COMPATIBILITY_CATEGORY = f"{CATEGORY}/compatibility"
OPTIONAL_CATEGORY = f"{CATEGORY}/optional"
EXPERIMENTAL_MOTION_CATEGORY = f"{CATEGORY}/experimental-motion-planning"
MODEL_TYPE = "DG_MODEL_CONFIG"
MEDIA_TYPE = "DG_MEDIA_CONTEXT"
CONTEXT_TYPE = "DG_CONTEXT"
TARGET_PROFILE_TYPE = "DG_TARGET_PROFILE_CONFIG"
DEFAULT_GGUF_MODEL_PATH = r"Z:\models\LLM\diffusiongemma-26B-A4B-it-GGUF\diffusiongemma-26B-A4B-it-Q4_K_M.gguf"
DEFAULT_GGUF_CLI_PATH = r"C:\tools\llama.cpp-diffusiongemma\build-clang-cuda\bin\llama-diffusion-cli.exe"
DEFAULT_GGUF_EXTRA_ARGS = "-ngl all -cmoe -sm none -mg 0 --diffusion-eb auto --diffusion-kv-cache auto --diffusion-gpu-sampling auto --diffusion-eb-max-steps 24 --diffusion-steps 24 --temp 0.45 -no-cnv --log-verbosity 1"
DEFAULT_CUDA_RUNTIME_BIN = r"C:\tools\cuda-13.3-conda\Library\bin"
DEFAULT_NVFP4_MODEL_DIR = str(Path.cwd() / "models" / "LLM" / "diffusiongemma-26B-A4B-it-NVFP4")
DEFAULT_TRANSFORMERS_MODEL_DIR = str(Path.cwd() / "models" / "LLM" / "diffusiongemma-26B-A4B-it")
DEFAULT_ADVANCED_MODEL_PATH = DEFAULT_NVFP4_MODEL_DIR if Path(DEFAULT_NVFP4_MODEL_DIR).exists() else DEFAULT_TRANSFORMERS_MODEL_DIR
DEFAULT_MODEL_PATH = DEFAULT_GGUF_MODEL_PATH if Path(DEFAULT_GGUF_MODEL_PATH).exists() else DEFAULT_TRANSFORMERS_MODEL_DIR
FINAL_JSON_OPEN = "<|final_json|>"
FINAL_JSON_CLOSE = "<|end_final_json|>"
FINAL_JSON_OPEN_MARKERS = (FINAL_JSON_OPEN, "<|answer|>", "<|final|>")
FINAL_JSON_CLOSE_MARKERS = (FINAL_JSON_CLOSE, "<|/final_json|>", "<|end_final|>", "<|end|>")
MIN_MODELOPT_NVFP4_BRIDGE_MEMORY_GB = 18.0
MEMORY_PRESET_SAFE_20 = "20.0 GB - safe practical"
MEMORY_PRESET_MIN_18 = "18.0 GB - minimum supported"
MEMORY_PRESET_FULL_GPU = "0 GB - full GPU"
MEMORY_PRESET_CHOICES = [MEMORY_PRESET_SAFE_20, MEMORY_PRESET_MIN_18, MEMORY_PRESET_FULL_GPU]
RESOLUTION_SELECTOR_ASPECT_LABELS = {
"1:1": "1:1 (Square)",
"2:3": "2:3 (Portrait Photo)",
"3:2": "3:2 (Photo)",
"3:4": "3:4 (Portrait Standard)",
"4:3": "4:3 (Standard)",
"9:16": "9:16 (Portrait Widescreen)",
"16:9": "16:9 (Widescreen)",
"21:9": "21:9 (Ultrawide)",
}
RESOLUTION_SELECTOR_RATIOS = {
"1:1": (1, 1),
"2:3": (2, 3),
"3:2": (3, 2),
"3:4": (3, 4),
"4:3": (4, 3),
"9:16": (9, 16),
"16:9": (16, 9),
"21:9": (21, 9),
}
REFERENCE_PREP_ASPECT_CHOICES = ["source", "1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3", "21:9"]
MEDIA_SYNTHESIS_MODE_CHOICES = ["video_recreation", "image_identity_video_control"]
DEFAULT_MASTER_PROMPT = """You are a local ComfyUI prompt architect. Treat the user's text as a content brief: it only needs to say what the video or image should be about. Do not require the user to ask for prompt-writing boilerplate such as detailed subject, background, lighting, camera movement, pacing, composition, or style; infer useful generator-ready detail when it is missing, while preserving every explicit user constraint.
Use the user's text, any backend-consumable pixels, and any supplied visual_description to produce final, practical prompts for generation models. If the active backend is metadata-only for image/video, do not pretend you saw pixels; use only the user's text, the visual_description, and factual media metadata. Preserve requested subjects, exact text, composition, motion, style, camera behavior, visual capture language, audio mode, audio guidance, and constraints. In image_identity_video_control workflows, treat a request to replace the video subject with the image subject as a strict identity-and-appearance transfer: face, hair, body type, wardrobe, accessories, styling, and distinguishing traits come from the reference image, while the video contributes pose, action, blocking, depth/canny/edge structure, camera choreography, timing, composition, and scene geometry. Do not copy the control-video subject's wardrobe, hair, face, body type, accessories, or styling into the final prompt unless the user explicitly asks for those video-subject traits. For LTX source-video work, explicitly observe camera choreography before writing the final prompt: first-frame and ending framing, shot size, camera height/angle, camera support or stability, camera path, zoom/dolly/truck/boom/pan/tilt/roll/orbit/rotation, subject-camera relationship, focus behavior, depth/parallax, and visible lens/capture changes. Transcribe observed camera work precisely instead of reducing it to generic phrases like camera movement, zoom out, or pan left. Visual capture language includes exposure level, overexposed or blown highlights, underexposure, flash/direct light, bloom, halation, lens blur, bokeh, depth of field, focal length impression, perspective compression, lens distortion, motion blur, grain/noise, white balance, contrast, dynamic range, and camera angle. Do not describe invisible intent. Every prompt field must contain the finished prompt text or finished prompt JSON, never instructions about how to write a prompt. Return only valid JSON with ltx_prompt, ideogram_prompt, minimax_h3_prompt, negative_prompt, scene_segments, and metadata."""
@dataclass
class RuntimeConfig:
model_path: str
backend: str
dtype: str
quantization: str
local_files_only: bool
unload_policy: str
max_memory_gb: float
cli_path: str = ""
extra_args: str = ""
temperature: float = 0.45
fallback_backend: str = "template"
status: dict[str, Any] = field(default_factory=dict)
@dataclass
class MediaContext:
images: Any = None
source: str = "none"
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass
class GemmaContext:
user_prompt: str = ""
images: Any = None
source: str = "none"
media_metadata: dict[str, Any] = field(default_factory=dict)
visual_description: str = ""
warnings: list[str] = field(default_factory=list)
@dataclass
class TargetProfileConfig:
target_profile: str = "ltx"
audio_mode: str = "auto_scene_audio"
audio_guidance: str = ""
target_duration_seconds: float = 0.0
ltx_style: str = ""
ideogram_aspect_ratio: str = "1:1"
ideogram_render_style: str = ""
ideogram_exact_text: str = ""
ideogram_json_output: bool = True
negative_prompt_mode: str = "auto"
negative_prompt_guidance: str = ""
minimax_h3_mode: str = "t2va"
_MODEL_CACHE: dict[str, tuple[Any, Any]] = {}
_NVFP4_SMOKE_CACHE: dict[str, dict[str, Any]] = {}
def _dg_log(message: str, *args: Any) -> None:
logging.info("[DiffusionGemma] " + message, *args)
def _progress_enabled() -> bool:
return os.environ.get("DG_PROGRESS", "1").strip().lower() not in {"0", "false", "off", "no"}
def _release_transformers_runtime() -> None:
_MODEL_CACHE.clear()
gc.collect()
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
try:
torch.cuda.ipc_collect()
except Exception:
pass
except Exception:
pass
def _json_dumps(data: Any) -> str:
return json.dumps(data, indent=2, sort_keys=True, ensure_ascii=True)
def _prompt_json_dumps(data: Any) -> str:
return json.dumps(data, indent=2, ensure_ascii=False)
def _ideogram_json_dumps(data: Any) -> str:
return json.dumps(data, separators=(",", ":"), ensure_ascii=False)
def _safe_float(value: Any, default: float = 0.0) -> float:
try:
return float(value)
except Exception:
return default
def _memory_preset_to_gb(value: Any) -> float:
if isinstance(value, (int, float)):
return float(value)
text = str(value or "").strip()
lowered = text.lower()
if "full" in lowered:
return 0.0
if "minimum" in lowered:
return MIN_MODELOPT_NVFP4_BRIDGE_MEMORY_GB
if "safe" in lowered:
return 20.0
match = re.search(r"[-+]?\d+(?:\.\d+)?", text)
if match:
return _safe_float(match.group(0), 20.0)
return 20.0
def _run_blocking_with_progress(
label: str,
fn: Any,
*,
node_id: str | None = None,
estimated_seconds: float | None = None,
) -> Any:
if not _progress_enabled():
return fn()
estimate = max(5.0, _safe_float(os.environ.get("DG_PROGRESS_ESTIMATE_SECONDS", ""), estimated_seconds or 120.0))
estimate_units = int(math.ceil(estimate))
interval = max(0.25, _safe_float(os.environ.get("DG_PROGRESS_INTERVAL_SECONDS", ""), 1.0))
result_box: dict[str, Any] = {}
def worker() -> None:
try:
result_box["value"] = fn()
except BaseException as exc:
result_box["error"] = exc
progress_bar = None
tqdm_bar = None
try:
from comfy.utils import ProgressBar
progress_bar = ProgressBar(estimate_units, node_id=node_id)
progress_bar.update_absolute(0, estimate_units)
except Exception:
progress_bar = None
try:
from tqdm.auto import tqdm
tqdm_bar = tqdm(
total=estimate_units,
desc=label,
unit="s",
leave=True,
bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]",
)
except Exception:
tqdm_bar = None
thread = threading.Thread(target=worker, name=f"{label} worker", daemon=True)
start = time.perf_counter()
last_reported = 0
displayed_total = estimate_units
thread.start()
try:
while thread.is_alive():
elapsed = time.perf_counter() - start
displayed_total = max(displayed_total, int(math.ceil(elapsed + interval)))
current = min(int(elapsed), displayed_total)
if progress_bar is not None:
progress_bar.update_absolute(current, displayed_total)
if tqdm_bar is not None:
if displayed_total != tqdm_bar.total:
tqdm_bar.total = displayed_total
tqdm_bar.refresh()
delta = max(0, current - last_reported)
if delta:
tqdm_bar.update(delta)
last_reported += delta
time.sleep(interval)
thread.join()
elapsed = time.perf_counter() - start
total = max(1, last_reported, int(math.ceil(elapsed)))
if progress_bar is not None:
progress_bar.update_absolute(total, total)
if tqdm_bar is not None:
if total != tqdm_bar.total:
tqdm_bar.total = total
remaining = max(0.0, total - tqdm_bar.n)
if remaining:
tqdm_bar.update(remaining)
finally:
if tqdm_bar is not None:
tqdm_bar.close()
if "error" in result_box:
raise result_box["error"]
return result_box.get("value")
def _normalize_aspect_ratio_key(aspect_ratio: Any) -> str:
text = str(aspect_ratio or "").strip()
match = re.search(r"\b(\d+)\s*:\s*(\d+)\b", text)
if not match:
return "1:1"
key = f"{int(match.group(1))}:{int(match.group(2))}"
return key if key in RESOLUTION_SELECTOR_RATIOS else "1:1"
def _resolution_selector_preset(aspect_ratio: Any) -> str:
return RESOLUTION_SELECTOR_ASPECT_LABELS[_normalize_aspect_ratio_key(aspect_ratio)]
def _known_aspect_ratio_label_from_dimensions(width: int, height: int) -> str:
src_w = max(1, int(width))
src_h = max(1, int(height))
source_ratio = src_w / float(src_h)
best_key = ""
best_delta = float("inf")
for key, (w_ratio, h_ratio) in RESOLUTION_SELECTOR_RATIOS.items():
delta = abs(source_ratio - (w_ratio / float(h_ratio)))
if delta < best_delta:
best_delta = delta
best_key = key
if best_key and best_delta <= 0.02:
return RESOLUTION_SELECTOR_ASPECT_LABELS[best_key]
return f"Custom / source ({src_w}x{src_h})"
def _resolution_selector_dimensions(aspect_ratio: Any, megapixels: Any = 2.0, multiple: Any = 8) -> tuple[int, int]:
key = _normalize_aspect_ratio_key(aspect_ratio)
w_ratio, h_ratio = RESOLUTION_SELECTOR_RATIOS[key]
mp = max(0.1, min(16.0, _safe_float(megapixels, 2.0)))
mult = int(max(8, min(128, _safe_float(multiple, 8))))
mult = max(1, int(round(mult / 4)) * 4)
total_pixels = mp * 1024 * 1024
scale = math.sqrt(total_pixels / (w_ratio * h_ratio))
width = max(mult, round(w_ratio * scale / mult) * mult)
height = max(mult, round(h_ratio * scale / mult) * mult)
return int(width), int(height)
def _round_dimension_to_multiple(value: float, multiple: int) -> int:
mult = int(max(1, multiple))
return max(mult, int(round(float(value) / mult) * mult))
def _dimensions_for_long_edge(width: int, height: int, long_edge: int, multiple: int = 8) -> tuple[int, int]:
src_w = max(1, int(width))
src_h = max(1, int(height))
edge = max(64, int(long_edge))
scale = edge / float(max(src_w, src_h))
return (
_round_dimension_to_multiple(src_w * scale, multiple),
_round_dimension_to_multiple(src_h * scale, multiple),
)
def _aspect_dimensions_for_long_edge(aspect_ratio: Any, long_edge: int, multiple: int = 8) -> tuple[int, int]:
key = _normalize_aspect_ratio_key(aspect_ratio)
w_ratio, h_ratio = RESOLUTION_SELECTOR_RATIOS[key]
edge = max(64, int(long_edge))
if w_ratio >= h_ratio:
width = edge
height = max(1, int(round(edge * (h_ratio / float(w_ratio)))))
else:
height = edge
width = max(1, int(round(edge * (w_ratio / float(h_ratio)))))
return (_round_dimension_to_multiple(width, multiple), _round_dimension_to_multiple(height, multiple))
def _center_crop_dimensions_for_aspect(width: int, height: int, aspect_ratio: Any) -> tuple[int, int]:
key = _normalize_aspect_ratio_key(aspect_ratio)
target_w, target_h = RESOLUTION_SELECTOR_RATIOS[key]
src_w = max(1, int(width))
src_h = max(1, int(height))
src_ratio = src_w / float(src_h)
dst_ratio = target_w / float(target_h)
if abs(src_ratio - dst_ratio) < 1e-6:
return (src_w, src_h)
if src_ratio > dst_ratio:
crop_w = max(1, int(round(src_h * dst_ratio)))
return (min(crop_w, src_w), src_h)
crop_h = max(1, int(round(src_w / dst_ratio)))
return (src_w, min(crop_h, src_h))
def _center_crop_image_batch(image: Any, crop_width: int, crop_height: int) -> Any:
try:
import torch
tensor = image if isinstance(image, torch.Tensor) else torch.as_tensor(image)
if tensor.ndim == 3:
tensor = tensor.unsqueeze(0)
_, src_h, src_w, _ = tensor.shape
crop_w = max(1, min(int(crop_width), int(src_w)))
crop_h = max(1, min(int(crop_height), int(src_h)))
x0 = max(0, (int(src_w) - crop_w) // 2)
y0 = max(0, (int(src_h) - crop_h) // 2)
return tensor[:, y0 : y0 + crop_h, x0 : x0 + crop_w, :]
except Exception:
return image
def _resize_image_batch(image: Any, width: int, height: int) -> Any:
import torch
tensor = image if isinstance(image, torch.Tensor) else torch.as_tensor(image)
if tensor.ndim == 3:
tensor = tensor.unsqueeze(0)
original_dtype = tensor.dtype
original_device = tensor.device
work = tensor
if not torch.is_floating_point(work):
work = work.float()
nchw = work.permute(0, 3, 1, 2).contiguous()
try:
resized = torch.nn.functional.interpolate(
nchw,
size=(max(1, int(height)), max(1, int(width))),
mode="bilinear",
align_corners=False,
antialias=True,
)
except TypeError:
resized = torch.nn.functional.interpolate(
nchw,
size=(max(1, int(height)), max(1, int(width))),
mode="bilinear",
align_corners=False,
)
result = resized.permute(0, 2, 3, 1).contiguous()
if torch.is_floating_point(tensor) and result.dtype != original_dtype:
result = result.to(dtype=original_dtype)
return result.to(device=original_device)
def _read_json_file(path: Path) -> dict[str, Any]:
try:
data = json.loads(path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except Exception:
return {}
def _pad_nvfp4_block_scale_rows(block_scale: Any, output_rows: int) -> Any:
import torch
rounded_rows = ((int(output_rows) + 127) // 128) * 128
if int(block_scale.shape[0]) >= rounded_rows:
return block_scale
pad_shape = (rounded_rows - int(block_scale.shape[0]), int(block_scale.shape[1]))
padding = torch.zeros(pad_shape, dtype=block_scale.dtype, device=block_scale.device)
return torch.cat([block_scale, padding], dim=0)
def _modelopt_to_comfy_fp4_packing(qdata: Any) -> Any:
return ((qdata & 0x0F) << 4) | ((qdata & 0xF0) >> 4)
def _modelopt_to_comfy_block_scale(block_scale: Any) -> Any:
from comfy_kitchen.float_utils import to_blocked
return to_blocked(block_scale, flatten=False)
def _safetensors_index(model_path: str) -> dict[str, str]:
index_path = Path(model_path) / "model.safetensors.index.json"
index_data = _read_json_file(index_path)
weight_map = index_data.get("weight_map")
return weight_map if isinstance(weight_map, dict) else {}
def _nvfp4_bridge_smoke(model_path: str) -> dict[str, Any]:
cache_key = str(Path(model_path))
cached = _NVFP4_SMOKE_CACHE.get(cache_key)
if cached is not None:
return cached.copy()
result: dict[str, Any] = {"passed": False, "error": "", "projection": ""}
try:
import torch
from safetensors import safe_open
from comfy.quant_ops import QuantizedTensor, get_layout_class
if not torch.cuda.is_available():
raise RuntimeError("CUDA is not available.")
root = Path(model_path)
weight_map = _safetensors_index(model_path)
prefix = "model.decoder.layers.0.experts.0.gate_proj"
required = [f"{prefix}.{name}" for name in ("weight", "weight_scale", "weight_scale_2", "input_scale")]
if not all(key in weight_map for key in required):
raise RuntimeError("Expected layer-0 expert gate_proj NVFP4 tensors were not found.")
handles: dict[str, Any] = {}
def get_tensor(key: str) -> Any:
shard = weight_map[key]
handle = handles.get(shard)
if handle is None:
handle = safe_open(str(root / shard), framework="pt", device="cpu")
handles[shard] = handle
return handle.get_tensor(key)
try:
qdata = _modelopt_to_comfy_fp4_packing(get_tensor(f"{prefix}.weight")).to("cuda", non_blocking=True)
block_scale = _modelopt_to_comfy_block_scale(get_tensor(f"{prefix}.weight_scale")).to("cuda", non_blocking=True)
tensor_scale = get_tensor(f"{prefix}.weight_scale_2").to("cuda", non_blocking=True)
input_scale = get_tensor(f"{prefix}.input_scale").to("cuda", non_blocking=True)
finally:
handles.clear()
out_features = int(qdata.shape[0])
in_features = int(qdata.shape[1]) * 2
layout_cls = get_layout_class("TensorCoreNVFP4Layout")
weight_params = layout_cls.Params(
scale=tensor_scale,
block_scale=block_scale,
orig_dtype=torch.bfloat16,
orig_shape=(out_features, in_features),
)
weight = QuantizedTensor(qdata.to(torch.uint8), "TensorCoreNVFP4Layout", weight_params)
x = torch.zeros((2, in_features), dtype=torch.bfloat16, device="cuda")
qx = QuantizedTensor.from_float(x, "TensorCoreNVFP4Layout", scale=input_scale)
y = torch.nn.functional.linear(qx, weight, None)
torch.cuda.synchronize()
if not bool(torch.isfinite(y.float()).all()):
raise RuntimeError("NVFP4 bridge smoke produced non-finite output.")
result.update(
{
"passed": True,
"projection": prefix,
"output_shape": list(y.shape),
"output_dtype": str(y.dtype).replace("torch.", ""),
}
)
except Exception as exc:
result["error"] = f"{type(exc).__name__}: {exc}"
finally:
gc.collect()
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_NVFP4_SMOKE_CACHE[cache_key] = result.copy()
return result
def _model_path_info(model_path: str) -> dict[str, Any]:
path = Path(model_path or "")
info: dict[str, Any] = {
"path_kind": "missing",
"is_gguf_file": False,
"is_hf_repo": False,
"model_type": "",
"architecture": "",
"quant_algo": "",
"quant_method": "",
"quant_producer": "",
"quant_producer_version": "",
"transformers_version_required": "",
"has_generation_config": False,
"has_modelopt_state": False,
"runtime_hint": "",
}
if not path.exists():
return info
if path.is_file():
info["path_kind"] = "gguf_file" if path.suffix.lower() == ".gguf" else "file"
info["is_gguf_file"] = path.suffix.lower() == ".gguf"
if info["is_gguf_file"]:
info["runtime_hint"] = "llama-diffusion-cli GGUF"
return info
if not path.is_dir():
info["path_kind"] = "unknown"
return info
config = _read_json_file(path / "config.json")
hf_quant_config = _read_json_file(path / "hf_quant_config.json")
quant_config = config.get("quantization_config") if isinstance(config.get("quantization_config"), dict) else {}
if not quant_config:
quant_config = hf_quant_config.get("quantization") if isinstance(hf_quant_config.get("quantization"), dict) else {}
producer = quant_config.get("producer") if isinstance(quant_config.get("producer"), dict) else {}
if not producer:
producer = hf_quant_config.get("producer") if isinstance(hf_quant_config.get("producer"), dict) else {}
architectures = config.get("architectures") if isinstance(config.get("architectures"), list) else []
info.update(
{
"is_hf_repo": bool(config),
"model_type": str(config.get("model_type", "")),
"architecture": str(architectures[0]) if architectures else "",
"quant_algo": str(quant_config.get("quant_algo", "")),
"quant_method": str(quant_config.get("quant_method", config.get("quantization_config", {}).get("quant_method", "")) if isinstance(config.get("quantization_config"), dict) else ""),
"quant_producer": str(producer.get("name", "")),
"quant_producer_version": str(producer.get("version", "")),
"transformers_version_required": str(config.get("transformers_version", "")),
"has_generation_config": (path / "generation_config.json").exists(),
"has_modelopt_state": (path / "modelopt_state.pth").exists(),
}
)
if info["quant_algo"].upper() == "NVFP4":
info["path_kind"] = "nvfp4_hf_repo"
info["runtime_hint"] = "NVIDIA ModelOpt NVFP4 checkpoint"
elif config:
info["path_kind"] = "hf_repo"
info["runtime_hint"] = "Transformers whole-repo model"
else:
info["path_kind"] = "directory"
return info
def _runtime_status(config: RuntimeConfig) -> dict[str, Any]:
model_info = _model_path_info(config.model_path)
status: dict[str, Any] = {
"backend": config.backend,
"model_path": config.model_path,
"model_path_exists": Path(config.model_path).exists(),
**model_info,
"local_files_only": config.local_files_only,
"max_memory_gb": config.max_memory_gb,
"temperature": config.temperature,
"transformers_importable": False,
"diffusion_gemma_module": False,
"diffusion_gemma_class": False,
"auto_processor": False,
"vllm_importable": False,
"modelopt_importable": False,
"comfy_kitchen_importable": False,
"fp_quant_importable": False,
"qutlass_importable": False,
"tensorrt_llm_importable": False,
"torch_cuda_available": False,
"torch_cuda_capability": "",
"transformers_version": "",
"processor_class": "",
"modelopt_nvfp4_supported": False,
"comfy_nvfp4_bridge_supported": False,
"nvfp4_bridge_min_memory_gb": MIN_MODELOPT_NVFP4_BRIDGE_MEMORY_GB,
"nvfp4_bridge_memory_guard_passed": True,
"nvfp4_bridge_smoke": {},
"ready": False,
"supports_pixels": False,
"supports_video": False,
"supports_thinking": False,
"media_policy": "unknown",
"notes": [],
}
try:
import importlib.util
status["transformers_importable"] = importlib.util.find_spec("transformers") is not None
status["diffusion_gemma_module"] = importlib.util.find_spec("transformers.models.diffusion_gemma") is not None
status["vllm_importable"] = importlib.util.find_spec("vllm") is not None
status["modelopt_importable"] = bool(
importlib.util.find_spec("modelopt") or importlib.util.find_spec("nvidia_modelopt")
)
status["comfy_kitchen_importable"] = importlib.util.find_spec("comfy_kitchen") is not None
status["fp_quant_importable"] = importlib.util.find_spec("fp_quant") is not None
status["qutlass_importable"] = importlib.util.find_spec("qutlass") is not None
status["tensorrt_llm_importable"] = importlib.util.find_spec("tensorrt_llm") is not None
try:
import transformers
status["transformers_version"] = str(getattr(transformers, "__version__", ""))
except Exception:
pass
try:
import torch
status["torch_cuda_available"] = bool(torch.cuda.is_available())
if torch.cuda.is_available():
status["torch_cuda_capability"] = ".".join(str(part) for part in torch.cuda.get_device_capability())
except Exception as exc:
if config.backend == "transformers_inprocess":
status["notes"].append(f"Torch CUDA probe failed: {exc}")
try:
from transformers import AutoProcessor # noqa: F401
status["auto_processor"] = True
if config.backend == "transformers_inprocess" and status["model_path_exists"] and status["is_hf_repo"]:
try:
processor = AutoProcessor.from_pretrained(config.model_path, local_files_only=True)
status["processor_class"] = type(processor).__name__
except Exception as exc:
status["notes"].append(f"AutoProcessor load failed: {exc}")
except Exception as exc:
if config.backend == "transformers_inprocess":
status["notes"].append(f"AutoProcessor import failed: {exc}")
try:
from transformers import DiffusionGemmaForBlockDiffusion # noqa: F401
status["diffusion_gemma_class"] = True
except Exception as exc:
if config.backend == "transformers_inprocess":
status["notes"].append(f"DiffusionGemmaForBlockDiffusion import failed: {exc}")
except Exception as exc:
if config.backend == "transformers_inprocess":
status["notes"].append(f"Transformers probe failed: {exc}")
if config.backend == "transformers_inprocess":
modelopt_nvfp4 = status["path_kind"] == "nvfp4_hf_repo" and status["quant_method"] == "modelopt"
low_bridge_memory = bool(
modelopt_nvfp4
and config.quantization == "modelopt_nvfp4"
and 0 < float(config.max_memory_gb) < MIN_MODELOPT_NVFP4_BRIDGE_MEMORY_GB
)
status["nvfp4_bridge_memory_guard_passed"] = not low_bridge_memory
if modelopt_nvfp4 and config.quantization == "modelopt_nvfp4":
smoke = _nvfp4_bridge_smoke(config.model_path)
status["nvfp4_bridge_smoke"] = smoke
status["comfy_nvfp4_bridge_supported"] = bool(
smoke.get("passed")
and status["comfy_kitchen_importable"]
and status["torch_cuda_available"]
and status["diffusion_gemma_class"]
and status["auto_processor"]
)
status["modelopt_nvfp4_supported"] = bool(
modelopt_nvfp4
and (
status["comfy_nvfp4_bridge_supported"]
or (
status["modelopt_importable"]
and status["tensorrt_llm_importable"]
and status["has_modelopt_state"]
)
)
)
status["ready"] = bool(
status["model_path_exists"]
and status["auto_processor"]
and status["diffusion_gemma_class"]
and not (modelopt_nvfp4 and not status["modelopt_nvfp4_supported"])
and not low_bridge_memory
)
status["supports_pixels"] = bool(status["ready"])
status["supports_video"] = bool(status["ready"])
status["supports_thinking"] = bool(status["ready"])
status["media_policy"] = "pixels_when_ready"
if modelopt_nvfp4 and status["comfy_nvfp4_bridge_supported"]:
status["notes"].append(
"This NVFP4 repo uses quant_method=modelopt, which vanilla Transformers does not deserialize. "
"The node will use the local Comfy NVFP4 bridge: Transformers supplies DiffusionGemma structure "
"and Comfy/comfy_kitchen supplies packed NVFP4 expert matmul."
)
if low_bridge_memory:
status["notes"].append(
f"The local NVFP4 bridge does not offload by max_memory_gb yet. Set max_memory_gb to "
f"{MIN_MODELOPT_NVFP4_BRIDGE_MEMORY_GB:g} or higher, or 0 for an explicit full-GPU load. "
"Lower values are blocked to avoid a partial CUDA load that can require a ComfyUI restart."
)
elif modelopt_nvfp4 and not status["modelopt_nvfp4_supported"]:
status["notes"].append(
"This NVFP4 repo uses quant_method=modelopt. Transformers 5.12.1 exposes DiffusionGemma, "
"but it does not register modelopt as an in-process quantizer. The local ModelOpt package "
"also lacks a modelopt_state.pth restore file for this repo, TensorRT-LLM is not available, "
"and the Comfy NVFP4 bridge smoke did not pass. "
"Blocking before full weight load to avoid a CPU-memory hang."
)
elif config.backend == "gguf_subprocess":
status["cli_path_exists"] = bool(config.cli_path and Path(config.cli_path).exists())
status["cuda_runtime_bin_exists"] = Path(DEFAULT_CUDA_RUNTIME_BIN).exists()
status["ready"] = bool(status["model_path_exists"] and status["cli_path_exists"] and status["is_gguf_file"])
status["supports_pixels"] = False
status["supports_video"] = False
status["supports_thinking"] = bool(status["ready"])
status["media_policy"] = "metadata_only"
if not status["cli_path_exists"]:
status["notes"].append("GGUF subprocess backend needs a foreground llama-diffusion-cli path.")
if status["model_path_exists"] and not status["is_gguf_file"]:
if status["path_kind"] == "nvfp4_hf_repo":
status["notes"].append(
"This path is an NVFP4 Hugging Face/vLLM repo, not a .gguf file. "
"The clean Model Loader runs llama-diffusion-cli and needs the GGUF model path."
)
elif status["is_hf_repo"]:
status["notes"].append(
"This path is a Hugging Face repo folder, not a .gguf file. "
"Use Model Loader (Advanced) with transformers_inprocess only after proof gates pass."
)
else:
status["notes"].append("GGUF subprocess backend needs model_path to point directly to a .gguf file.")
elif config.backend in {"qwen_vl_placeholder", "minicpm_v_placeholder"}:
status["ready"] = False
status["notes"].append(
"Fallback backend lane is reserved for comparative benchmarking; no Qwen/MiniCPM runner is loaded by this pack yet."
)
else:
status["ready"] = config.backend == "template"
status["supports_thinking"] = False
status["media_policy"] = "template"
if config.backend == "transformers_inprocess" and not status["ready"]:
if not status["diffusion_gemma_class"]:
status["notes"].append(
"This ComfyUI venv can keep the node pack loaded, but it cannot run DiffusionGemma until "
"a compatible Transformers build exposes DiffusionGemmaForBlockDiffusion."
)
if status["path_kind"] == "nvfp4_hf_repo" and not status["comfy_nvfp4_bridge_supported"]:
status["notes"].append(
"NVFP4 was detected. Use this path only after a supported in-process ModelOpt NVFP4 loader is "
"available; the current local foreground fallback remains the GGUF llama-diffusion-cli route."
)
return status
def _strip_markdown(text: str) -> str:
text = re.sub(r"```(?:json|text|markdown)?\s*", "", text, flags=re.IGNORECASE)
text = text.replace("```", "")
text = re.sub(r"^\s{0,3}#{1,6}\s+", "", text, flags=re.MULTILINE)
return text.strip()
def _extract_delimited_final_payload(text: str) -> tuple[str, bool]:
value = str(text or "")
if not value:
return "", False
lowered = value.lower()
starts: list[tuple[int, str]] = []
for marker in FINAL_JSON_OPEN_MARKERS:
index = lowered.find(marker.lower())
if index >= 0:
starts.append((index, marker))
if not starts:
return value, False
start_index, open_marker = min(starts, key=lambda item: item[0])
payload_start = start_index + len(open_marker)
payload_end = len(value)
for marker in FINAL_JSON_CLOSE_MARKERS:
index = lowered.find(marker.lower(), payload_start)
if index >= 0:
payload_end = min(payload_end, index)
return value[payload_start:payload_end].strip(), True
def _strip_thinking(text: str) -> str:
text = re.sub(r".*?", "", text, flags=re.IGNORECASE | re.DOTALL)
text = re.sub(r"<\|thought\|>.*?<\|end_thought\|>", "", text, flags=re.IGNORECASE | re.DOTALL)
text = re.sub(r"<\|thought\|>.*?(?=<\|(?:final_json|answer|final)\|>)", "", text, flags=re.IGNORECASE | re.DOTALL)
text = re.sub(r"<\|channel\|>thought.*?(?=<\|channel\|>|$)", "", text, flags=re.IGNORECASE | re.DOTALL)
text = re.sub(r"<\|?channel\|?>\s*thought\s*", "", text, flags=re.IGNORECASE)
text = re.sub(r"<\|?channel\|?>", "", text, flags=re.IGNORECASE)
text = re.sub(r"<\|[^>]+?\|>", "", text)
return text.strip()
def _strip_cli_runtime_lines(text: str) -> str:
kept: list[str] = []
for line in text.splitlines():
if re.match(r"\s*(total time:|throughput:|diffusion step:)", line, flags=re.IGNORECASE):
continue
if re.match(r"\s*\d+\.\d+\.\d+\.\d+\s+[IWE]\s+", line):
continue
kept.append(line)
return "\n".join(kept).strip()
def _strip_generation_artifacts(text: str) -> str:
value = str(text or "")
if not value:
return ""
replacements = {
"\ufeff": "",
"\ufffd": "",
"": "",
"ç´‹": " ",
}
for artifact, replacement in replacements.items():
value = value.replace(artifact, replacement)
value = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]", " ", value)
value = re.sub(r"(?:(?<=\s)|(?<=[,.;:!?]))(?:[ÃÂâ][^\sA-Za-z0-9]{1,8}|[ÃÂ][A-Za-z0-9]{1,4})(?=\s|[,.;:!?]|$)", " ", value)
value = re.sub(r"\s*,\s+to\s+(him|her|them)\b", r" near \1", value, flags=re.IGNORECASE)
value = re.sub(r"\s+([,.!?;:])", r"\1", value)
value = re.sub(r"([,.!?;:])(?=[^\s,.!?;:])", r"\1 ", value)
return re.sub(r"\s+", " ", value).strip()
def _split_extra_args(extra_args: str) -> list[str]:
if not extra_args.strip():
return []
if os.name != "nt":
return shlex.split(extra_args)
return [part.strip("\"'") for part in shlex.split(extra_args, posix=False)]
def _format_temperature(value: Any, default: float = 0.2) -> str:
temp = max(0.0, min(2.0, _safe_float(value, default)))
return f"{temp:.3f}".rstrip("0").rstrip(".")
def _extra_args_with_temperature(extra_args: str, temperature: float) -> str:
value = (extra_args or "").strip()
temp = _format_temperature(temperature)
if not value:
return f"--temp {temp}"
pattern = r"(? dict[str, str]:
env = os.environ.copy()
path_key = "Path" if "Path" in env else "PATH"
existing_path = env.get(path_key, "")
candidates = [
str(Path(cli_path).parent) if cli_path else "",
DEFAULT_CUDA_RUNTIME_BIN,
]
prefix = [path for path in candidates if path and Path(path).exists()]
env[path_key] = os.pathsep.join([*prefix, existing_path]) if existing_path else os.pathsep.join(prefix)
cuda_root = Path(DEFAULT_CUDA_RUNTIME_BIN).parent
if cuda_root.exists():
env.setdefault("CUDA_PATH", str(cuda_root))
return env
def _sanitize_prompt_text(
text: str,
fallback: str = "",
max_chars: int = 4000,
strip_thinking: bool = True,
strip_markdown: bool = True,
) -> str:
value = (text or "").strip()
if strip_thinking:
value = _strip_thinking(value)
if strip_markdown:
value = _strip_markdown(value)
value = _strip_generation_artifacts(value)
value = re.sub(r"\s+", " ", value).strip()
if not value:
value = (fallback or "").strip()
if max_chars > 0 and len(value) > max_chars:
value = value[:max_chars].rsplit(" ", 1)[0].strip() or value[:max_chars].strip()
return value
def _sanitize_structured_prompt_text(
text: str,
fallback: str = "",
max_chars: int = 8000,
strip_thinking: bool = True,
strip_markdown: bool = True,
) -> str:
value = str(text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
if strip_thinking:
value = _strip_thinking(value)
if strip_markdown:
value = _strip_markdown(value)
replacements = {
"\ufeff": "",
"\ufffd": "",
"": "",
}
for artifact, replacement in replacements.items():
value = value.replace(artifact, replacement)
value = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]", " ", value)
lines = []
for line in value.splitlines():
line = re.sub(r"[ \t]+", " ", line).strip()
lines.append(line)
value = "\n".join(lines)
value = re.sub(r"\n{3,}", "\n\n", value).strip()
if not value:
value = str(fallback or "").strip()
if max_chars > 0 and len(value) > max_chars:
value = value[:max_chars].rstrip()
return value
_MINIMAX_H3_REF_FIELDS = (
"subject_definitions",
"summary",
"retention_analysis",
"detailed_description",
"overall_soundscape",
"non_diegetic_music",
)
def _minimax_h3_ref_sections(prompt: str) -> dict[str, str] | None:
pattern = r"(?ms)^" + r"\s*\n+\s*".join(
rf"{re.escape(field_name)}\s*:\s*(.*?)" for field_name in _MINIMAX_H3_REF_FIELDS[:-1]
) + rf"\s*\n+\s*{re.escape(_MINIMAX_H3_REF_FIELDS[-1])}\s*:\s*(.*)$"
match = re.fullmatch(pattern, prompt)
if not match:
return None
return {
field_name: match.group(index + 1).strip()
for index, field_name in enumerate(_MINIMAX_H3_REF_FIELDS)
}
def _rebuild_minimax_h3_ref_prompt(sections: dict[str, str]) -> str:
return "\n\n".join(f"{field_name}:\n{sections.get(field_name, '').strip()}" for field_name in _MINIMAX_H3_REF_FIELDS)
def _normalize_minimax_h3_prompt(text: str, minimax_h3_mode: str = "t2va") -> str:
prompt = _sanitize_structured_prompt_text(text, "", 0)
if not prompt:
return ""
if _normalize_minimax_h3_mode(minimax_h3_mode) == "ref2va":
prompt = re.sub(
r"<\s*(subject|picture|image|video|audio)\s+(\d+)\s*>",
lambda match: (
f""
if match.group(1).casefold() == "subject"
else f"<{'Picture' if match.group(1).casefold() in {'picture', 'image'} else match.group(1).title()} {int(match.group(2))}>"
),
prompt,
flags=re.IGNORECASE,
)
for field_name in _MINIMAX_H3_REF_FIELDS:
prompt = re.sub(
rf"(?im)^\s*{re.escape(field_name)}\s*:",
f"{field_name}:",
prompt,
)
sections = _minimax_h3_ref_sections(prompt)
return _rebuild_minimax_h3_ref_prompt(sections) if sections else prompt
fields = (
"integrated_multimodal_description",
"overall_soundscape",
"non_diegetic_music",
)
for field_name in fields:
prompt = re.sub(
rf"(?im)^\s*{re.escape(field_name)}\s*:",
f"{field_name}:",
prompt,
)
field_matches = {
field_name: re.search(rf"(?m)^{re.escape(field_name)}\s*:", prompt)
for field_name in fields
}
if not any(field_matches.values()) and re.match(r"^\[Shot 1\](?=\s|$)", prompt):
blocks = [block.strip() for block in re.split(r"\n\s*\n", prompt) if block.strip()]
if len(blocks) >= 3:
integrated = "\n\n".join(blocks[:-2])
prompt = (
f"integrated_multimodal_description: {integrated}\n\n"
f"overall_soundscape: {blocks[-2]}\n\n"
f"non_diegetic_music: {blocks[-1]}"
)
elif (
not field_matches["integrated_multimodal_description"]
and field_matches["overall_soundscape"]
and field_matches["non_diegetic_music"]
and re.match(r"^\[Shot 1\](?=\s|$)", prompt)
):
prompt = f"integrated_multimodal_description: {prompt}"
structured_match = re.fullmatch(
r"(?ms)integrated_multimodal_description\s*:\s*(.*?)\s*\n+\s*overall_soundscape\s*:\s*(.*?)\s*\n+\s*non_diegetic_music\s*:\s*(.*)",
prompt,
)
if structured_match:
integrated = structured_match.group(1).strip()
shot_labels = re.findall(r"\[Shot (\d+)\]", integrated, flags=re.IGNORECASE)
if shot_labels == ["1"]:
next_shot = 1
def label_unmarked_cut(match: re.Match[str]) -> str:
nonlocal next_shot
next_shot += 1
return f"[Shot {next_shot}] {match.group(0)}"
integrated = re.sub(
r"\bAt\s+\d{2,}:\d{2}\.\d{3},?\s+(?=the\s+(?:camera|shot|scene)\s+(?:(?:hard\s+)?cuts|transitions|changes|switches|cross-dissolves|fades|wipes)\s+(?:to|into)\b)",
label_unmarked_cut,
integrated,
flags=re.IGNORECASE,
)
prompt = (
f"integrated_multimodal_description: {integrated}\n\n"
f"overall_soundscape: {structured_match.group(2).strip()}\n\n"
f"non_diegetic_music: {structured_match.group(3).strip()}"
)
return re.sub(
r"(\[Shot [2-9]\d*\]\s+At\s+\d{2,}:\d{2}\.\d{3})(?!,)(?=\s+the\s+(?:camera|shot|scene)\b)",
r"\1,",
prompt,
flags=re.IGNORECASE,
)
def _minimax_h3_ref_definition_items(text: str) -> list[tuple[str, str]]:
return [
(f"<{kind.title()} {int(ordinal)}>", description.strip())
for kind, ordinal, description in re.findall(
r"(?im)^\s*<\s*(Subject|Picture|Video|Audio)\s+(\d+)\s*>\s*(?::|=|[-–—]|\bis\b)\s*(\S.*)$",
text or "",
)
]
def _minimax_h3_ref_summary_task_types(user_prompt: str, reference_manifest: str) -> list[str]:
manifest = _normalize_minimax_h3_reference_manifest(reference_manifest)
definitions = _minimax_h3_reference_definitions(manifest)
tags = [tag for tag, _description in definitions]
combined = f"{user_prompt} {manifest}"
task_types: list[str] = []
if _has_unnegated_prompt_phrase(
combined,
r"\b(?:keyframes?|first[-\s]+frames?|last[-\s]+frames?|first\s+and\s+last\s+frames?|interpolat(?:e|ion|ing))\b",
):
task_types.append("keyframe completion")
if any(tag.startswith((" str:
range_pattern = re.compile(
r"\[Shot (?P\d+)\]\s*\(?\s*"
r"(?P\d{1,2}):(?P\d{2})(?:\.(?P\d{1,3}))?\s*[-–—]\s*"
r"(?P\d{1,2}):(?P\d{2})(?:\.(?P\d{1,3}))?\s*\)?\s*:?\s*",
flags=re.IGNORECASE,
)
range_matches = list(range_pattern.finditer(detailed))
duration = max(0.0, _safe_float(duration_seconds, 0.0))
def range_time(match: re.Match[str], prefix: str, mode: str) -> float:
major = int(match.group(f"{prefix}_major"))
minor = int(match.group(f"{prefix}_minor"))
fraction_text = match.group(f"{prefix}_fraction") or ""
if mode == "minutes_seconds":
if minor >= 60:
return -1.0
fraction = int(fraction_text.ljust(3, "0")) / 1000.0 if fraction_text else 0.0
return major * 60.0 + minor + fraction
if fraction_text or minor >= 100:
return -1.0
return major + minor / 100.0
def valid_range_mode(mode: str) -> bool:
previous_end: float | None = None
for match in range_matches:
start = range_time(match, "start", mode)
end = range_time(match, "end", mode)
if start < 0 or end <= start:
return False
if int(match.group("shot")) == 1 and start != 0:
return False
if previous_end is not None and abs(start - previous_end) > 0.01:
return False
if duration > 0 and (start >= duration or end > duration + 0.05):
return False
previous_end = end
return bool(range_matches)
range_mode = ""
if valid_range_mode("minutes_seconds"):
range_mode = "minutes_seconds"
elif duration > 0 and valid_range_mode("seconds_hundredths"):
range_mode = "seconds_hundredths"
if range_mode:
def replace_range(match: re.Match[str]) -> str:
shot = int(match.group("shot"))
start = range_time(match, "start", range_mode)
if shot == 1:
return "[Shot 1] "
minutes = int(start // 60)
seconds = int(start % 60)
milliseconds = int(round((start - int(start)) * 1000))
if milliseconds == 1000:
seconds += 1
milliseconds = 0
if seconds == 60:
minutes += 1
seconds = 0
following = match.string[match.end() :]
article = "a " if re.match(
r"(?:close[- ]up|medium(?:[- ]wide)? shot|wide shot|long shot|high[- ]angle|low[- ]angle|bird['’]s[- ]eye)",
following,
flags=re.IGNORECASE,
) else ""
return f"[Shot {shot}] At {minutes:02d}:{seconds:02d}.{milliseconds:03d}, the camera cuts to {article}"
detailed = range_pattern.sub(replace_range, detailed)
detailed = re.sub(
r"(\[Shot [2-9]\d*\]\s+At\s+\d{2,}:\d{2}\.\d{3},\s*the camera cuts to (?:a )?)([A-Z])",
lambda match: f"{match.group(1)}{match.group(2).casefold()}",
detailed,
)
detailed = re.sub(
r"(\[Shot 1\])\s+At\s+00:00\.000,?\s*",
r"\1 ",
detailed,
flags=re.IGNORECASE,
)
detailed = re.sub(
r"(\[Shot [2-9]\d*\]\s+At\s+\d{2,}:\d{2}\.\d{3})(?!,)(?=\s+\S)",
r"\1,",
detailed,
flags=re.IGNORECASE,
)
detailed = re.sub(
r"(\[Shot [2-9]\d*\]\s+At\s+\d{2,}:\d{2}\.\d{3},\s*)the\s+camera\s+(?:quick|clean)\s+cuts?\s+to\s+",
r"\1the camera cuts to ",
detailed,
flags=re.IGNORECASE,
)
detailed = re.sub(
r"(\[Shot [2-9]\d*\]\s+At\s+\d{2,}:\d{2}\.\d{3},\s*)(?:a\s+)?(?:quick|hard|clean)?\s*cut\s+to\s+",
r"\1the camera cuts to ",
detailed,
flags=re.IGNORECASE,
)
detailed = re.sub(
r"(\[Shot [2-9]\d*\]\s+At\s+\d{2,}:\d{2}\.\d{3},)(?!\s*the\s+(?:camera|shot|scene)\s+(?:(?:hard\s+)?cuts|transitions|changes|switches|cross-dissolves|fades|wipes)\s+(?:to|into)\b)(\s*)",
r"\1\2the camera cuts to a static wide view as ",
detailed,
flags=re.IGNORECASE,
)
return re.sub(
r"(the camera cuts to a static wide view as )(The|An?)\b",
lambda match: f"{match.group(1)}{match.group(2).casefold()}",
detailed,
flags=re.IGNORECASE,
)
def _repair_minimax_h3_ref_subject_timeline_labels(sections: dict[str, str]) -> None:
detailed = sections["detailed_description"]
shot1_match = re.search(r"\[Shot 1\]", detailed, flags=re.IGNORECASE)
if not shot1_match:
return
retention = sections["retention_analysis"]
timeline = detailed[shot1_match.start() :]
repairs: list[tuple[str, str]] = []
for tag, description in _minimax_h3_ref_definition_items(sections["subject_definitions"]):
if not tag.startswith(" str:
prompt = _normalize_minimax_h3_prompt(prompt_text, "ref2va")
if not prompt:
return ""
timeline_duration = max(
0.0,
_safe_float(duration_seconds, 0.0) or _explicit_prompt_duration_seconds(user_prompt),
)
canonical_sections = _minimax_h3_ref_sections(prompt)
if canonical_sections is not None:
summary = canonical_sections["summary"].strip()
summary_match = re.match(r"^\[([^\]\n]+)\]\s*(.*)$", summary, flags=re.DOTALL)
summary_body = summary_match.group(2).strip() if summary_match else summary
actual_task_types = [item.strip() for item in summary_match.group(1).split("+")] if summary_match else []
expected_task_types = _minimax_h3_ref_summary_task_types(user_prompt, reference_manifest)
if actual_task_types != expected_task_types:
canonical_sections["summary"] = f"[{' + '.join(expected_task_types)}] {summary_body}".rstrip()
_repair_minimax_h3_ref_subject_timeline_labels(canonical_sections)
canonical_sections["detailed_description"] = _repair_minimax_h3_ref_detailed_timeline(
canonical_sections["detailed_description"],
timeline_duration,
)
repaired_prompt = _rebuild_minimax_h3_ref_prompt(canonical_sections)
return repaired_prompt if repaired_prompt != prompt else prompt
heading_pattern = re.compile(
r"(?im)^[ \t]*(subject[ _-]*definitions|summary(?:[ \t]*\[[^\]\n]*\])?|retention[ _-]*analysis|"
r"detail(?:ed)?[ _-]*description|overall[ _-]*soundscape|non[ _-]*diegetic[ _-]*music)"
r"[ \t]*(?P:[ \t]*|(?=\r?$))"
)
matches = list(heading_pattern.finditer(prompt))
if not matches:
return prompt
heading_names = [
re.sub(
r"[\s-]+",
"_",
re.sub(r"\s*\[[^\]\n]*\]\s*$", "", match.group(1)).strip().casefold(),
)
for match in matches
]
heading_names = ["detailed_description" if name == "detail_description" else name for name in heading_names]
prefix = prompt[: matches[0].start()].strip()
has_colonless_heading = any(not match.group("delimiter").startswith(":") for match in matches)
if has_colonless_heading and not (
(not prefix and heading_names == list(_MINIMAX_H3_REF_FIELDS))
or (
prefix
and heading_names
== ["retention_analysis", "detailed_description", "overall_soundscape", "non_diegetic_music"]
)
):
return prompt
if prefix and "subject_definitions" not in heading_names and "summary" not in heading_names:
if heading_names not in (
["retention_analysis", "detailed_description", "overall_soundscape", "non_diegetic_music"],
["overall_soundscape", "non_diegetic_music"],
):
return prompt
if (
not any(name.startswith("summary") for name in heading_names)
and "subject_definitions" not in heading_names
and not _minimax_h3_ref_definition_items(prefix)
):
return prompt
sections = {field_name: "" for field_name in _MINIMAX_H3_REF_FIELDS}
for index, match in enumerate(matches):
raw_name = re.sub(r"\s*\[[^\]\n]*\]\s*$", "", match.group(1))
field_name = re.sub(r"[\s-]+", "_", raw_name.strip().casefold())
if field_name == "detail_description":
field_name = "detailed_description"
value_end = matches[index + 1].start() if index + 1 < len(matches) else len(prompt)
value = prompt[match.end() : value_end].strip()
if field_name in sections:
sections[field_name] = "\n".join(item for item in (sections[field_name], value) if item).strip()
if has_colonless_heading and any(not sections[field_name] for field_name in heading_names):
return prompt
prefix_recovered = False
if prefix and heading_names in (
["retention_analysis", "detailed_description", "overall_soundscape", "non_diegetic_music"],
["overall_soundscape", "non_diegetic_music"],
):
prefix_blocks = [block.strip() for block in re.split(r"\n\s*\n", prefix) if block.strip()]
retention_markers = _MINIMAX_H3_REF_VISUAL_RETENTION | _MINIMAX_H3_REF_AUDIO_RETENTION
retention_pattern = re.compile(
r"(?im)^\s*<\s*(?:Subject|Picture|Video|Audio)\s+\d+\s*>"
rf"(?:\s*(?:\([^\n)]*\)|\[[^\n\]]*\]))*\s*:\s*(?:{'|'.join(sorted(retention_markers))})\s*-\s*\S"
)
def is_retention_block(block: str) -> bool:
tagged_items = _minimax_h3_ref_definition_items(block)
lines = [line for line in block.splitlines() if line.strip()]
return bool(tagged_items) and len(retention_pattern.findall(block)) == len(tagged_items) == len(lines)
def is_definition_block(block: str) -> bool:
lines = [line for line in block.splitlines() if line.strip()]
return bool(lines) and len(_minimax_h3_ref_definition_items(block)) == len(lines)
if prefix and heading_names == ["retention_analysis", "detailed_description", "overall_soundscape", "non_diegetic_music"]:
summary_index = next(
(
index
for index, block in enumerate(prefix_blocks)
if re.match(r"^\[(?!Shot\b)[^\]\n]+\]\s+\S", block, flags=re.IGNORECASE | re.DOTALL)
),
None,
)
definition_blocks = prefix_blocks[:summary_index] if summary_index is not None else []
if (
summary_index
and summary_index == len(prefix_blocks) - 1
and all(is_definition_block(block) and not retention_pattern.search(block) for block in definition_blocks)
and is_retention_block(sections["retention_analysis"])
and re.search(r"\[Shot 1\]", sections["detailed_description"], flags=re.IGNORECASE)
and sections["overall_soundscape"]
and sections["non_diegetic_music"]
):
sections["subject_definitions"] = "\n".join(definition_blocks)
sections["summary"] = prefix_blocks[summary_index]
prefix_recovered = True
else:
return prompt
if prefix and heading_names == ["overall_soundscape", "non_diegetic_music"]:
summary_index = next(
(
index
for index, block in enumerate(prefix_blocks)
if re.match(r"^\[(?!Shot\b)[^\]\n]+\]\s+\S", block, flags=re.IGNORECASE | re.DOTALL)
),
None,
)
definition_blocks = prefix_blocks[:summary_index] if summary_index is not None else []
retention_start = summary_index + 1 if summary_index is not None else 0
retention_end = retention_start
while retention_end < len(prefix_blocks) and is_retention_block(prefix_blocks[retention_end]):
retention_end += 1
detailed_blocks = prefix_blocks[retention_end:]
detailed_shot_index = next(
(
index
for index, block in enumerate(detailed_blocks)
if re.search(r"\[Shot 1\]", block, flags=re.IGNORECASE)
),
None,
)
stray_tagged_detail = bool(
detailed_shot_index is not None
and any(_minimax_h3_ref_definition_items(block) for block in detailed_blocks[:detailed_shot_index])
)
if (
summary_index
and all(is_definition_block(block) and not retention_pattern.search(block) for block in definition_blocks)
and retention_end > retention_start
and detailed_blocks
and not stray_tagged_detail
and re.search(r"\[Shot 1\]", "\n\n".join(detailed_blocks), flags=re.IGNORECASE)
):
sections["subject_definitions"] = "\n".join(definition_blocks)
sections["summary"] = prefix_blocks[summary_index]
sections["retention_analysis"] = "\n".join(prefix_blocks[retention_start:retention_end])
sections["detailed_description"] = "\n\n".join(detailed_blocks)
prefix_recovered = True
else:
return prompt
if prefix and not prefix_recovered:
if not sections["subject_definitions"]:
sections["subject_definitions"] = prefix
elif _minimax_h3_ref_definition_items(prefix):
sections["subject_definitions"] = f"{prefix}\n{sections['subject_definitions']}".strip()
definition_items: list[tuple[str, str]] = []
for tag, description in _minimax_h3_ref_definition_items(sections["subject_definitions"]):
if tag not in {item_tag for item_tag, _item_description in definition_items}:
definition_items.append((tag, description.strip(" []")))
definition_text = "\n".join(f"{tag}: {description}" for tag, description in definition_items)
for tag, description in _minimax_h3_reference_definitions(reference_manifest):
if tag not in _minimax_h3_reference_tags(definition_text):
definition_items.append((tag, description.strip(" []")))
definition_text = "\n".join(f"{item_tag}: {item_description}" for item_tag, item_description in definition_items)
if definition_items:
sections["subject_definitions"] = definition_text
summary = sections["summary"].strip()
summary_match = re.match(r"^\[([^\]\n]+)\]\s*(.*)$", summary, flags=re.DOTALL)
if summary_match:
summary = summary_match.group(2).strip()
task_prefix = " + ".join(_minimax_h3_ref_summary_task_types(user_prompt, reference_manifest))
sections["summary"] = f"[{task_prefix}] {summary}".rstrip()
retention = sections["retention_analysis"]
retention_lines: list[str] = []
for tag, definition in definition_items:
tag_match = re.fullmatch(r"<(Subject|Picture|Video|Audio) (\d+)>", tag)
if not tag_match:
continue
kind, ordinal = tag_match.groups()
existing_match = re.search(
rf"(?is)(?:<|\[)\s*{kind}\s+{ordinal}\s*(?:>|\])(?:\s*(?:\([^\n)]*\)|\[[^\n\]]*\]))*\s*:\s*"
r"(.*?)(?=(?:<|\[)\s*(?:Subject|Picture|Video|Audio)\s+\d+\s*(?:>|\])|$)",
retention,
)
existing = existing_match.group(1).strip().strip("[]").strip() if existing_match else ""
marker_match = re.match(
r"^(fully_preserved|partially_preserved|attribute_transfer|weak_reference|fully_copy|partially_copy|reference)\s*-\s*(.*)$",
existing,
flags=re.IGNORECASE | re.DOTALL,
)
if marker_match:
marker = marker_match.group(1).casefold()
retention_detail = marker_match.group(2).strip().strip("[]").strip()
else:
marker = "fully_preserved" if kind == "Subject" else ("reference" if kind == "Audio" else "attribute_transfer")
retention_detail = re.sub(r"^[^:]{0,48}:\s*", "", existing).strip().strip("[]").strip() or definition
allowed_markers = _MINIMAX_H3_REF_AUDIO_RETENTION if kind == "Audio" else _MINIMAX_H3_REF_VISUAL_RETENTION
if marker not in allowed_markers:
marker = "reference" if kind == "Audio" else ("fully_preserved" if kind == "Subject" else "attribute_transfer")
retention_lines.append(f"{tag}: {marker} - {retention_detail}")
if retention_lines:
sections["retention_analysis"] = "\n".join(retention_lines)
_repair_minimax_h3_ref_subject_timeline_labels(sections)
sections["detailed_description"] = _repair_minimax_h3_ref_detailed_timeline(
sections["detailed_description"],
timeline_duration,
)
return _rebuild_minimax_h3_ref_prompt(sections)
def _extract_json_object(text: str) -> dict[str, Any] | None:
if not text:
return None
payload, _had_boundary = _extract_delimited_final_payload(text)
cleaned = _strip_markdown(_strip_thinking(payload))
candidates = [cleaned]
candidates.append(re.sub(r"^\s*\{\s*(?=\{)", "", cleaned).strip())
first = cleaned.find("{")
last = cleaned.rfind("}")
if first >= 0 and last > first:
candidates.append(cleaned[first : last + 1])
candidates.append(re.sub(r"^\s*\{\s*(?=\{)", "", cleaned[first : last + 1]).strip())
for candidate in candidates:
try:
data = json.loads(candidate)
if isinstance(data, dict):
return data
except Exception:
continue
return None
def _single_paragraph(text: str) -> str:
return _strip_generation_artifacts(re.sub(r"\s+", " ", (text or "").replace("\n", " ")).strip())
def _ensure_terminal_period(text: str) -> str:
value = _single_paragraph(text)
if not value:
return ""
if value[-1] in ".!?":
return value
return f"{value}."
def _sentence_case(text: str) -> str:
value = _single_paragraph(text)
if value and value[0].islower():
return value[0].upper() + value[1:]
return value
def _clean_prompt_request_text(text: str) -> str:
value = _single_paragraph(text)
if _is_media_reference_only_request(value):
return ""
patterns = [
r"^(?:create|write|generate|make)\s+(?:an?\s+)?(?:ltx(?:[- ]?2(?:\.3)?)?\s+)?(?:script|prompt)\s+for\s+(?:image|video)\s+generation\s+(?:featuring|with|of|about)\s+",
r"^(?:create|write|generate|make)\s+(?:an?\s+)?(?:ltx(?:[- ]?2(?:\.3)?)?\s+)?(?:script|prompt)\s+(?:featuring|with|of|about)\s+",
r"^(?:create|write|generate|make)\s+(?:an?\s+)?(?:image|video)\s+(?:generation\s+)?(?:script|prompt)\s+(?:featuring|with|of|about)\s+",
]
cleaned = value
for pattern in patterns:
cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE).strip()
cleaned = re.sub(r"\.{2,}", ".", cleaned)
cleaned = re.sub(r"\s+([,.!?])", r"\1", cleaned).strip()
return _normalize_hex_text(_sentence_case(cleaned or value))
def _clean_visual_description(text: str, max_chars: int = 3000) -> str:
value = _strip_generation_artifacts(_strip_markdown(_strip_thinking(str(text or ""))))
value = re.sub(r"\s+", " ", value).strip()
if max_chars > 0 and len(value) > max_chars:
value = value[:max_chars].rsplit(" ", 1)[0].strip() or value[:max_chars].strip()
return value
def _visual_description_from_metadata(media_metadata: dict[str, Any] | None) -> str:
if not isinstance(media_metadata, dict):
return ""
return _clean_visual_description(str(media_metadata.get("visual_description", "")))
def _normalize_media_synthesis_mode(value: Any) -> str:
text = str(value or "video_recreation").strip().lower().replace("-", "_").replace(" ", "_")
if text in {"image_identity_video_control", "identity_image_video_control", "image_identity", "identity_reference"}:
return "image_identity_video_control"
return "video_recreation"
def _is_image_identity_video_control(media_metadata: dict[str, Any] | None) -> bool:
return _normalize_media_synthesis_mode(
media_metadata.get("media_synthesis_mode") if isinstance(media_metadata, dict) else ""
) == "image_identity_video_control"
def _is_minimax_h3_reference_context(media_metadata: dict[str, Any] | None) -> bool:
if not isinstance(media_metadata, dict):
return False
value = str(media_metadata.get("minimax_h3_mode", "")).strip().lower().replace("-", "_")
return value in {"ref2va", "ref2v", "r2v", "reference", "reference_to_video", "reference_to_audio_video"}
def _synthesis_contract_text(media_metadata: dict[str, Any] | None) -> str:
if _is_image_identity_video_control(media_metadata):
return (
"Use the reference image as the identity and appearance source for the output subject, including face, hair, body type, wardrobe, accessories, styling, and distinguishing visual traits. "
"Use the video frames only as control structure for pose, depth, canny/edge layout, composition, action, timing, camera motion, blocking, and scene geometry. "
"Do not copy the control-video subject's clothing, hair, face, body type, accessories, or style unless the user explicitly asks for those video-subject traits."
)
return "Recreate the source video or media evidence directly unless the user prompt asks for a different transformation."
def _copy_synthesis_metadata(target: dict[str, Any], media_metadata: dict[str, Any]) -> None:
for key in (
"media_synthesis_mode",
"image_role",
"video_role",
"reference_image_count",
"reference_image_backend_attached",
"reference_image_resized_for_video_control",
"reference_image_resize_mode",
"reference_image_original_width",
"reference_image_original_height",
"reference_image_backend_width",
"reference_image_backend_height",
"video_sampled_frame_count",
"synthesis_contract",
):
if key in media_metadata:
target[key] = media_metadata[key]
def _media_source_has_pixels(media_metadata: dict[str, Any] | None) -> bool:
if not isinstance(media_metadata, dict):
return False
source = str(media_metadata.get("source", "none")).lower()
return source in {"image", "video", "image+video"} or bool(media_metadata.get("sampled_frame_count"))
def _media_pixels_sent_to_backend(media_metadata: dict[str, Any] | None) -> bool:
return bool(isinstance(media_metadata, dict) and media_metadata.get("pixels_sent_to_backend"))
def _media_grounding_mode(media_metadata: dict[str, Any] | None) -> str:
if not _media_source_has_pixels(media_metadata):
return "none"
if _media_pixels_sent_to_backend(media_metadata):
return "pixels"
if _visual_description_from_metadata(media_metadata):
return "visual_description"
return "metadata_only"
def _video_transport_mode() -> str:
value = os.environ.get("DG_VIDEO_TRANSPORT", "sampled_frame_images").strip().lower()
value = value.replace("-", "_").replace(" ", "_")
if value in {"video", "video_token", "video_tokens", "native_video"}:
return "video_tokens"
return "sampled_frame_images"
def _video_frame_timecodes(media_metadata: dict[str, Any] | None, frame_count: int) -> list[float]:
if frame_count <= 0:
return []
if not isinstance(media_metadata, dict):
return [float(i) for i in range(frame_count)]
indices = media_metadata.get("sampled_indices")
source_fps = _safe_float(media_metadata.get("source_fps"), 0.0)
if isinstance(indices, list) and source_fps > 0:
values: list[float] = []
for index in indices[:frame_count]:
try:
values.append(float(index) / source_fps)
except Exception:
values.append(float(len(values)))
if len(values) == frame_count:
return values
duration = _safe_float(media_metadata.get("duration_seconds"), 0.0)
if duration > 0 and frame_count > 1:
step = duration / max(1, frame_count - 1)
return [idx * step for idx in range(frame_count)]
sample_fps = max(0.001, _safe_float(media_metadata.get("sample_fps"), 1.0))
return [idx / sample_fps for idx in range(frame_count)]
def _display_video_frame_count(media_metadata: dict[str, Any] | None, frame_count: int) -> int:
if not isinstance(media_metadata, dict):
return frame_count
video_count = int(max(0, _safe_float(media_metadata.get("video_sampled_frame_count"), 0.0)))
if video_count > 0:
return min(frame_count, video_count)
return frame_count
def _sampled_video_frame_sequence_intro(media_metadata: dict[str, Any] | None, frame_count: int) -> str:
reference_count = int(max(0, _safe_float(media_metadata.get("reference_image_count"), 0.0))) if isinstance(media_metadata, dict) else 0
video_frame_count = _display_video_frame_count(media_metadata, max(0, frame_count - reference_count))
timecodes = _video_frame_timecodes(media_metadata, video_frame_count)
shown = ", ".join(f"{idx + 1}={seconds:.2f}s" for idx, seconds in enumerate(timecodes[:24]))
if video_frame_count > 24:
shown = f"{shown}, ..."
source = str(media_metadata.get("source", "video")) if isinstance(media_metadata, dict) else "video"
if _is_minimax_h3_reference_context(media_metadata):
picture_intro = (
f"The first {reference_count} attached image(s) are ordered MiniMax H3 reference pictures through . "
if reference_count
else "No reference pictures are attached for pixel analysis. "
)
return (
f"{picture_intro}The remaining images are ordered sampled frames from