"""
Alice T2V Prompt Refiner Node
Rewrites rough text ideas (and optionally analyzes a reference image) into
structured, cinematically-precise prompts optimised for the Alice T2V 14B MoE
diffusion transformer model.
Alice T2V was trained on GPT-4 prose descriptions. It responds to natural
language sentences - NOT Stable Diffusion tag lists. This node handles
the translation from rough idea → model-ready prompt.
Backends:
lm_studio : OpenAI-compatible HTTP API (default http://localhost:1234/v1)
Load your GGUF in LM Studio before running.
ollama : Ollama HTTP API (default http://localhost:11434)
Author: Eric Hiss (GitHub: EricRollei)
"""
import base64
import io
import json
import traceback
from typing import Optional, Tuple
import numpy as np
import torch
from PIL import Image
from .wildcard_support import (
enforce_verbatim,
is_changed_token,
llm_block,
normalise_seed,
report_lines,
resolve_wildcards,
wildcard_optional_inputs,
)
# ---------------------------------------------------------------------------
# Default model file paths (shown in the node UI as editable strings).
# These are reminders for which file to load in LM Studio - the actual
# API calls go to the endpoint URL, not these paths directly.
# ---------------------------------------------------------------------------
DEFAULT_MODEL_PATH = r"H:\Captioning\Qwen3-VL-8B-Instruct-abliterated-v2.0.Q8_0.gguf"
DEFAULT_MMPROJ_PATH = r"H:\Captioning\Qwen3-VL-8B-Instruct-abliterated-v2.0.mmproj-f16.gguf"
# ---------------------------------------------------------------------------
# Style preset catalogue
# ---------------------------------------------------------------------------
STYLE_PRESETS = {
"cinematic": (
"Target a theatrical film aesthetic. Use precise cinematography vocabulary: "
"shot scale (ECU/CU/MS/WS), camera movement (dolly, crane, handheld tracking), "
"depth of field, colour grading, and lens character. "
"The scene should feel intentional and emotionally grounded."
),
"nature_documentary": (
"Target a high-end nature documentary aesthetic (BBC/NatGeo quality). "
"Describe naturalistic animal or environmental motion, ambient sound design, "
"and the kind of patient, observational camerawork that reveals behaviour. "
"Lighting should be natural - golden hour, dappled canopy light, or overcast diffusion."
),
"portrait_character": (
"Focus on the human (or creature) subject. Prioritise facial nuance, micro-expression, "
"gesture, and posture as emotional storytelling devices. "
"Avoid abstract emotional labels - show, don't tell. "
"Camera should favour medium close-up to close-up framing."
),
"architecture_urban": (
"Emphasise structural geometry, material texture, and the relationship between "
"built environment and human scale. Camera movement should reveal spatial depth - "
"slow reveals, parallax, or aerial perspective. "
"Environmental flow (people, light shafts, weather) adds life without overwhelming."
),
"action_dynamic": (
"Prioritise kinetic energy, physical impact, and spatial velocity. "
"Use motion blur, speed ramping, and aggressive camera angles to convey force. "
"Keep the scene readable - chaotic framing undermines impact. "
"Sound design (impacts, air displacement, crowd) should be implied in the description."
),
"abstract_artistic": (
"Allow visual metaphor and surreal logic. Motion can defy physics if it serves "
"a thematic purpose. Colour, texture, and light are narrative elements. "
"The scene should feel like a visual poem - evocative rather than literal. "
"Avoid cluttered compositions; simplicity amplifies strangeness."
),
"custom": "", # replaced at runtime with custom_style_notes
}
# ---------------------------------------------------------------------------
# Core system prompt
# ---------------------------------------------------------------------------
_BASE_SYSTEM_PROMPT = """\
You are an expert cinematographer and AI video prompt engineer specialising in \
Alice T2V 14B MoE - a diffusion transformer video model trained on prose \
descriptions generated by GPT-4.
CRITICAL: Alice does NOT respond to Stable Diffusion tag lists or comma-separated \
keyword strings. It requires natural, flowing prose sentences.
YOUR JOB
Transform the user's rough idea (and optional image analysis) into a precise, \
cinematic video generation prompt that Alice will execute reliably.
OUTPUT STRUCTURE (4-6 sentences, 150-250 words)
1. Shot type + camera position + opening atmosphere
2. Primary subject - appearance, pose, action (physical cues, not emotional labels)
3. Setting - environment, key objects, spatial depth
4. Lighting - quality, direction, colour temperature, source
5. Motion - how the camera and/or subject move through the shot
6. (Optional) Closing texture - sound, weather, final mood reinforcement
HARD RULES
- Present tense, active voice throughout
- No tag lists, no parenthetical emphasis syntax like (keyword:1.5)
- No text, logos, or readable signage (the model cannot render these)
- No more than 2-3 distinct subjects in one scene
- No conflicting light sources (warm sunset AND cold fluorescents is invalid)
- No chaotic or physics-defying motion that can't be described in a straight line
- Do NOT add preamble, explanation, or labels - output the prompt ONLY
- Target 150-250 words. Longer prompts are truncated at 512 tokens inside Alice.
"""
_IMAGE_ANALYSIS_SYSTEM_PROMPT = """\
You are an expert visual analyst preparing image descriptions for AI video generation.
Describe the provided image concisely and precisely:
- Primary subject(s): appearance, pose, expression, clothing
- Setting/environment: location, background, key objects
- Lighting: quality (hard/soft), direction, colour temperature, mood
- Colour palette: dominant hues, saturation level
- Composition: how elements are arranged, depth, framing
- Atmosphere: overall emotional tone conveyed visually
Rules:
- Be factual and specific. No interpretation or narrative.
- Describe what IS visible, not what might happen.
- Output ONLY the description. No headers, no labels.
"""
# ---------------------------------------------------------------------------
# Helper: tensor → PIL
# ---------------------------------------------------------------------------
def _tensor_to_pil(image: torch.Tensor) -> Image.Image:
img_np = image[0].cpu().numpy()
img_np = np.clip(img_np * 255.0, 0, 255).astype(np.uint8)
return Image.fromarray(img_np)
def _pil_to_base64(pil_image: Image.Image) -> str:
buf = io.BytesIO()
pil_image.save(buf, format="PNG")
return base64.b64encode(buf.getvalue()).decode("utf-8")
# ---------------------------------------------------------------------------
# Strip ... blocks (Qwen3 reasoning tokens)
# ---------------------------------------------------------------------------
def _strip_thinking(text: str) -> Tuple[str, str]:
"""Returns (thinking_text, clean_response)."""
import re
think_match = re.search(r"(.*?)", text, re.DOTALL | re.IGNORECASE)
if think_match:
thinking = think_match.group(1).strip()
clean = text[think_match.end():].strip()
return thinking, clean
return "", text.strip()
# ---------------------------------------------------------------------------
# Auto-detect loaded model from LM Studio /v1/models
# ---------------------------------------------------------------------------
def _detect_lm_studio_model(endpoint: str) -> Optional[str]:
"""Query /v1/models and return the first model id, or None on failure."""
import requests
try:
r = requests.get(
f"{endpoint.rstrip('/')}/models", timeout=5
)
r.raise_for_status()
models = r.json().get("data", [])
if models:
return models[0].get("id")
except Exception:
pass
return None
# ---------------------------------------------------------------------------
# Extract usable text from LM Studio response message
# ---------------------------------------------------------------------------
def _extract_content(message: dict, show_debug: bool = False) -> str:
"""
LM Studio / OpenAI message dict → clean text string.
Qwen3 thinking models in LM Studio 0.4.x can return responses in
several layouts depending on server version and think-mode setting:
Layout A (think mode OFF or older LM Studio):
message = {"content": "...\nActual answer"}
Layout B (think mode ON, LM Studio 0.4.7+):
message = {"content": "Actual answer", "reasoning_content": "...thinking..."}
→ content may be empty if model front-loaded everything into reasoning
Layout C (rare edge case):
message = {"content": ""} ← model produced only reasoning tokens
Strategy:
1. Prefer `content` after stripping any inline block.
2. If that's empty/short, fall back to `reasoning_content` stripped of thinking.
3. Log the raw structure so the user can diagnose unexpected layouts.
"""
if show_debug:
# Print keys and lengths (not full content - could be huge)
summary = {k: (len(v) if isinstance(v, str) else v)
for k, v in message.items()}
print(f"[AlicePromptRefiner] Raw message keys/lengths: {summary}")
# Primary: content field
raw_content = message.get("content") or ""
_, clean_content = _strip_thinking(raw_content)
if len(clean_content) >= 30:
return clean_content
# Fallback: reasoning_content field (Qwen3 think-mode layout)
reasoning = message.get("reasoning_content") or ""
if reasoning:
if show_debug:
print(
f"[AlicePromptRefiner] content was short ({len(clean_content)} chars), "
f"falling back to reasoning_content ({len(reasoning)} chars)"
)
# reasoning_content is the raw thinking; strip any nested think tags
_, clean_reasoning = _strip_thinking(reasoning)
if len(clean_reasoning) >= 30:
return clean_reasoning
# Last resort: return the raw reasoning text
if len(reasoning) >= 30:
return reasoning.strip()
# Nothing useful found
if show_debug:
print(
f"[AlicePromptRefiner] Both content and reasoning_content were short. "
f"Full message dict:\n{json.dumps(message, ensure_ascii=False, indent=2)}"
)
return clean_content # return whatever we have (may be empty)
# ---------------------------------------------------------------------------
# HTTP helpers
# ---------------------------------------------------------------------------
def _call_lm_studio(
endpoint: str,
system_prompt: str,
user_content, # str OR list (vision content blocks)
temperature: float,
max_tokens: int,
show_debug: bool = False,
) -> dict:
"""
OpenAI-compatible chat/completions call to LM Studio.
Handles Qwen3 thinking-mode response layouts robustly.
"""
import requests
# Auto-detect model id so LM Studio doesn't have to guess
model_id = _detect_lm_studio_model(endpoint) or "local-model"
if show_debug:
print(f"[AlicePromptRefiner] Using model id: {model_id!r}")
url = f"{endpoint.rstrip('/')}/chat/completions"
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_content},
]
payload = {
"model": model_id,
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
"stream": False,
}
try:
r = requests.post(url, json=payload, timeout=180)
r.raise_for_status()
data = r.json()
if show_debug:
# Log top-level keys and finish reason
finish = data.get("choices", [{}])[0].get("finish_reason", "?")
usage = data.get("usage", {})
print(
f"[AlicePromptRefiner] Response: finish_reason={finish!r} "
f"usage={usage}"
)
message = data["choices"][0]["message"]
content = _extract_content(message, show_debug=show_debug)
return {"success": True, "response": content, "error": None}
except Exception as exc:
if show_debug:
print(f"[AlicePromptRefiner] HTTP error: {traceback.format_exc()}")
return {"success": False, "response": "", "error": str(exc)}
def _call_ollama(
endpoint: str,
system_prompt: str,
user_text: str,
img_base64: Optional[str],
temperature: float,
max_tokens: int,
) -> dict:
"""Ollama generate API (supports optional inline image)."""
import requests
url = f"{endpoint.rstrip('/')}/api/generate"
model_name = "llava"
try:
tags = requests.get(f"{endpoint.rstrip('/')}/api/tags", timeout=5).json()
models = tags.get("models", [])
if models:
model_name = models[0].get("name", model_name)
except Exception:
pass
payload: dict = {
"model": model_name,
"prompt": f"{system_prompt}\n\n{user_text}",
"stream": False,
"options": {"temperature": temperature, "num_predict": max_tokens},
}
if img_base64:
payload["images"] = [img_base64]
try:
r = requests.post(url, json=payload, timeout=180)
r.raise_for_status()
content = r.json().get("response", "")
_, clean = _strip_thinking(content)
return {"success": True, "response": clean or content, "error": None}
except Exception as exc:
return {"success": False, "response": "", "error": str(exc)}
# ---------------------------------------------------------------------------
# Node class
# ---------------------------------------------------------------------------
class AlicePromptRefiner:
"""
Refines rough text and/or reference images into structured Alice T2V prompts.
Workflow
--------
Text-only: rough idea → LLM rewrite → Alice-ready prompt
Image+text: image → vision analysis → combine with hint → LLM rewrite → prompt
Image-only: image → vision analysis → LLM invent motion → prompt
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"backend": (["lm_studio", "ollama"], {
"default": "lm_studio",
"tooltip": (
"lm_studio: calls http://localhost:1234/v1 (OpenAI-compatible). "
"ollama: calls http://localhost:11434."
),
}),
"endpoint": ("STRING", {
"default": "http://localhost:1234/v1",
"multiline": False,
"tooltip": "API endpoint URL.",
}),
"model_path": ("STRING", {
"default": DEFAULT_MODEL_PATH,
"multiline": False,
"tooltip": (
"REMINDER - load this GGUF in LM Studio before running. "
"Not used directly by the node."
),
}),
"mmproj_path": ("STRING", {
"default": DEFAULT_MMPROJ_PATH,
"multiline": False,
"tooltip": (
"REMINDER - load this mmproj alongside the GGUF in LM Studio "
"for vision support."
),
}),
"text_prompt": ("STRING", {
"default": "",
"multiline": True,
"tooltip": "Rough idea for the video. Leave empty for image-only mode.",
}),
"style_preset": (list(STYLE_PRESETS.keys()), {
"default": "cinematic",
}),
"custom_style_notes": ("STRING", {
"default": "",
"multiline": True,
"tooltip": "Used only when style_preset = 'custom'.",
}),
"temperature": ("FLOAT", {
"default": 0.7,
"min": 0.1,
"max": 1.5,
"step": 0.05,
}),
"max_tokens": ("INT", {
"default": 512,
"min": 128,
"max": 1024,
"step": 64,
}),
"show_thinking": ("BOOLEAN", {
"default": False,
"tooltip": (
"Print Qwen3 reasoning chain to console. "
"Also enables detailed response debug logging."
),
}),
},
"optional": {
"image": ("IMAGE", {
"tooltip": (
"Optional reference image. Requires a vision model in LM Studio."
),
}),
"custom_system_prompt": ("STRING", {
"default": "",
"multiline": True,
"tooltip": "Fully overrides the built-in system prompt when non-empty.",
}),
# New inputs are only ever APPENDED below this line, so saved workflows keep their widget order.
**wildcard_optional_inputs(),
},
}
@classmethod
def IS_CHANGED(cls, wildcard_seed: int = -1, **kwargs):
"""wildcard_seed = -1 means a fresh roll each run, which must defeat ComfyUI's cache."""
return is_changed_token(wildcard_seed)
RETURN_TYPES = ("STRING", "STRING", "STRING")
RETURN_NAMES = ("refined_prompt", "image_description", "status")
FUNCTION = "refine"
CATEGORY = "Eric Prompt Enhancers"
def refine(
self,
backend: str,
endpoint: str,
model_path: str,
mmproj_path: str,
text_prompt: str,
style_preset: str,
custom_style_notes: str,
temperature: float,
max_tokens: int,
show_thinking: bool,
image: Optional[torch.Tensor] = None,
custom_system_prompt: str = "",
wildcard_seed: int = -1,
wildcard_llm_block: bool = True,
wildcard_instructions: str = "",
) -> Tuple[str, str, str]:
image_description = ""
try:
# ----------------------------------------------------------
# STEP 0 - Resolve wildcards in the rough idea (core/wildcards)
# ----------------------------------------------------------
wildcard_resolution = resolve_wildcards(
text_prompt or "", seed=normalise_seed(wildcard_seed), legacy=True,
upstream_instructions=wildcard_instructions,
)
text_prompt = wildcard_resolution.text
# ----------------------------------------------------------
# STEP 1 - Analyse reference image if provided
# ----------------------------------------------------------
if image is not None:
pil_img = _tensor_to_pil(image)
img_b64 = _pil_to_base64(pil_img)
print("[AlicePromptRefiner] Analysing reference image via vision model...")
vision_result = self._analyse_image(
backend, endpoint, img_b64, temperature, show_debug=show_thinking
)
if not vision_result["success"]:
return (
text_prompt or "",
"",
f"❌ Vision analysis failed: {vision_result['error']}",
)
raw_desc = vision_result["response"].strip()
if show_thinking:
thinking, image_description = _strip_thinking(raw_desc)
if thinking:
print(f"[AlicePromptRefiner] Vision thinking:\n{thinking}\n")
else:
_, image_description = _strip_thinking(raw_desc)
print(
f"[AlicePromptRefiner] Image description "
f"({len(image_description)} chars): "
f"{image_description[:120]}..."
)
# ----------------------------------------------------------
# STEP 2 - Build system prompt
# ----------------------------------------------------------
if custom_system_prompt and custom_system_prompt.strip():
system_prompt = custom_system_prompt.strip()
else:
system_prompt = _BASE_SYSTEM_PROMPT.strip()
style_guidance = STYLE_PRESETS.get(style_preset, "")
if style_preset == "custom":
style_guidance = custom_style_notes.strip()
if style_guidance:
system_prompt += (
f"\n\nSTYLE GUIDANCE - "
f"{style_preset.upper().replace('_', ' ')}\n{style_guidance}"
)
# ----------------------------------------------------------
# STEP 3 - Build user message
# ----------------------------------------------------------
user_parts: list[str] = []
if image_description:
user_parts.append(f"REFERENCE IMAGE ANALYSIS:\n{image_description}")
if text_prompt and text_prompt.strip():
label = "MOTION / SCENE HINT:" if image_description else "ROUGH IDEA:"
user_parts.append(f"{label}\n{text_prompt.strip()}")
elif not image_description:
return ("", "", "❌ No input: provide text_prompt and/or connect an image.")
if not text_prompt.strip() and image_description:
user_parts.append(
"TASK: Based only on the image analysis above, invent the most "
"cinematically compelling motion that could naturally occur in this "
"scene. Write the Alice T2V prompt."
)
else:
user_parts.append(
"TASK: Using the above as context, write the Alice T2V prompt."
)
# Tell the LLM which elements were picked at random, so it keeps them
selected_elements = llm_block(wildcard_resolution, wildcard_llm_block)
if selected_elements:
user_parts.append(selected_elements)
user_message = "\n\n".join(user_parts)
# ----------------------------------------------------------
# STEP 4 - Call the LLM for prompt rewrite
# ----------------------------------------------------------
print(
f"[AlicePromptRefiner] Calling {backend} at {endpoint} "
f"(temp={temperature}, max_tokens={max_tokens})..."
)
if backend == "lm_studio":
result = _call_lm_studio(
endpoint, system_prompt, user_message,
temperature, max_tokens,
show_debug=show_thinking,
)
elif backend == "ollama":
result = _call_ollama(
endpoint, system_prompt, user_message,
None, temperature, max_tokens,
)
else:
return ("", image_description, f"❌ Unknown backend: {backend}")
if not result["success"]:
return (
text_prompt or "",
image_description,
f"❌ LLM call failed: {result['error']}",
)
raw_response = result["response"]
# ----------------------------------------------------------
# STEP 5 - Strip any remaining thinking, clean output
# ----------------------------------------------------------
thinking, refined_prompt = _strip_thinking(raw_response)
if show_thinking and thinking:
print(f"[AlicePromptRefiner] Model thinking:\n{thinking}\n")
refined_prompt = refined_prompt.strip()
if len(refined_prompt) < 30:
msg = (
f"⚠️ Short response ({len(refined_prompt)} chars). "
f"Enable show_thinking=True for full debug output. "
f"Raw: {repr(raw_response[:200])}"
)
print(f"[AlicePromptRefiner] {msg}")
return (text_prompt or "", image_description, msg)
# Verbatim wildcard picks (__=name__) must survive the rewrite word for word
refined_prompt, _restored = enforce_verbatim(refined_prompt, wildcard_resolution)
word_count = len(refined_prompt.split())
print(
f"[AlicePromptRefiner] Done. {word_count} words, "
f"{len(refined_prompt)} chars."
)
status = (
f"✅ Refined | style={style_preset} | {word_count} words | "
f"image={'yes' if image_description else 'no'}"
)
wildcard_report = report_lines(wildcard_resolution)
if wildcard_report:
status = f"{status}\n{wildcard_report}"
return (refined_prompt, image_description, status)
except Exception as exc:
err = traceback.format_exc()
print(f"[AlicePromptRefiner] ERROR:\n{err}")
return (text_prompt or "", image_description, f"❌ {exc}")
def _analyse_image(
self,
backend: str,
endpoint: str,
img_b64: str,
temperature: float,
show_debug: bool = False,
) -> dict:
system_prompt = _IMAGE_ANALYSIS_SYSTEM_PROMPT.strip()
user_text = "Describe this image in detail for video generation purposes."
if backend == "lm_studio":
user_content = [
{"type": "text", "text": user_text},
{"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{img_b64}"}},
]
return _call_lm_studio(
endpoint, system_prompt, user_content,
temperature, max_tokens=768, show_debug=show_debug,
)
elif backend == "ollama":
return _call_ollama(
endpoint, system_prompt, user_text,
img_b64, temperature, max_tokens=768,
)
return {"success": False, "response": "", "error": f"Unknown backend: {backend}"}