574 lines
27 KiB
Python
574 lines
27 KiB
Python
"""
|
|
ComfyUI-VLMPrompt: Muse nodes — LLM-driven creative image prompt generation.
|
|
Invents vivid scenes from genre/mood/artist constraints rather than describing existing images.
|
|
"""
|
|
|
|
import json
|
|
import random
|
|
import re
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
|
|
from .vlm_gguf import (
|
|
MODELS_CATALOG,
|
|
VLMEngine,
|
|
_filter_kwargs,
|
|
_pick_device,
|
|
_sample_video_frames,
|
|
_tensor_to_base64_png,
|
|
)
|
|
|
|
NODE_DIR = Path(__file__).parent
|
|
MUSE_DATA_PATH = NODE_DIR / "muse_data.json"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Data loader
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _load_muse_data() -> dict:
|
|
if not MUSE_DATA_PATH.exists():
|
|
return {}
|
|
try:
|
|
with open(MUSE_DATA_PATH, "r", encoding="utf-8") as f:
|
|
return json.load(f) or {}
|
|
except Exception as exc:
|
|
print(f"[VLMPrompt/Muse] muse_data.json load failed: {exc}")
|
|
return {}
|
|
|
|
|
|
MUSE_DATA = _load_muse_data()
|
|
|
|
|
|
def _get_list(key: str) -> list[str]:
|
|
raw = MUSE_DATA.get(key) or []
|
|
# Strip separator entries (start with ──)
|
|
return [x for x in raw if not x.startswith("──")]
|
|
|
|
|
|
def _selectable(key: str) -> list[str]:
|
|
"""Full list including separator labels (for dropdown display)."""
|
|
return MUSE_DATA.get(key) or []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Random resolution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _resolve(value: str, pool: list[str], rng: random.Random) -> str | None:
|
|
"""If value is 'Random', pick a random non-separator, non-None entry."""
|
|
if value != "Random":
|
|
return None if value in ("None", "") else value
|
|
candidates = [x for x in pool if x not in ("Random", "None", "") and not x.startswith("──")]
|
|
return rng.choice(candidates) if candidates else None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Muse prompt builder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_MUSE_SYSTEM = """\
|
|
You are a visionary creative director and worldbuilder with encyclopedic knowledge of photography, illustration, painting, cinema, anime, manga, and the literary canon of science fiction and fantasy (Le Guin, Banks, Miyazaki, Tolkien, Miéville, Gibson, Otomo, Pratchett, Sanderson, VanderMeer, Murakami, etc.). You dream up original, specific, emotionally resonant image concepts — never generic, always alive.
|
|
|
|
When given constraints (genre, mood, artist, subject, era, etc.) you weave them into a coherent scene that feels like a single frame plucked from a larger novel or film — a story already in motion. You invent particular characters with names and history, specific places, precise moments. If a Heroine or protagonist is requested, give her real presence: distinctive features, signature gear, body language that tells you something about who she is and what she has just survived. You think in terms of narrative weight: what just happened? What is about to happen? You think about what a viewer will FEEL.
|
|
|
|
Your output must be ONLY valid JSON — no preamble, no explanation, no markdown fences. Output exactly this structure:
|
|
{
|
|
"title": "short evocative title, 3-7 words",
|
|
"subject": "the main subject or focal point, 1-2 sentences",
|
|
"scene": "the setting, environment, spatial context, 1-2 sentences",
|
|
"lighting": "light source, quality, direction, color temperature, mood effect",
|
|
"camera": "shot type, angle, lens feel, depth of field",
|
|
"color_palette": "key colors, their relationships, emotional effect",
|
|
"style": "visual medium, technique, aesthetic references",
|
|
"artist_ref": "specific artist(s) this evokes and why",
|
|
"mood": "the emotional atmosphere in 3-5 evocative words",
|
|
"details": "3-5 specific tactile or visual micro-details that make it feel real",
|
|
"prompt": "a single flowing paragraph (120-200 words) assembling all the above into a complete image generation prompt, written in present tense, no bullet points, no JSON syntax"
|
|
}
|
|
"""
|
|
|
|
_MUSE_SYSTEM_PROMPT_ONLY = """\
|
|
You are a visionary creative director with encyclopedic knowledge of photography, illustration, painting, cinema, and digital art. You dream up original, specific, emotionally resonant image concepts — never generic, always alive.
|
|
|
|
When given constraints (genre, mood, artist, era, etc.) you weave them into a coherent scene that feels like it was conceived by a master. You invent particular characters, specific places, precise moments.
|
|
|
|
Output ONLY a single flowing paragraph image generation prompt (120-200 words). Present tense. No preamble, no commentary, no bullet points, no JSON — just the prompt.
|
|
"""
|
|
|
|
_REFERENCE_IMAGE_PREFACE = """\
|
|
|
|
You are also given a REFERENCE IMAGE. Treat it as STYLE/AESTHETIC reference only — NOT as the literal subject of your output. Study it for: lighting quality and direction, color palette and tonal relationships, compositional language, texture and grain, level of detail and abstraction, mood, era/medium signals. Then INVENT A COMPLETELY NEW SCENE that captures the same visual language and emotional register. The new scene should feel like it was made by the same artist or shot on the same set, but it is a different moment, different subject, different story. Do not describe the reference image. Do not reuse its specific subject. Translate its DNA into something new.
|
|
"""
|
|
|
|
_AESTHETIC_BIASES = {
|
|
"None": "",
|
|
|
|
"Midjourney v6/v7 — Painterly Realism": """
|
|
|
|
AESTHETIC BIAS — Midjourney v6/v7:
|
|
Write the final prompt with the visual DNA of high-end Midjourney v6/v7 outputs. Lean into: painterly photorealism (grounded but slightly stylized, never plastic), cinematic dramatic lighting (rim light, golden hour, volumetric god rays, atmospheric haze, soft bokeh), rich harmonious color palettes (teal/orange complementary, jewel tones, controlled saturation with desaturated highlights), intentional composition (rule of thirds, leading lines, strong subject-foreground-background separation, depth), tactile material textures (skin pores, fabric weave, weathered metal, dust motes), and a "single magazine cover frame" sense of grandeur. Camera language is specific: lens, aperture, shot distance. Avoid generic "8k masterpiece" filler — instead use evocative descriptors that show, not tell. Subject should feel designed, deliberate, like a film still.""",
|
|
|
|
"Midjourney Niji — Anime/Manga": """
|
|
|
|
AESTHETIC BIAS — Midjourney Niji (Anime/Manga):
|
|
Write the prompt as if for Niji — clean anime/manga linework, expressive eyes, dramatic poses, cel-shaded color blocking with soft gradients, dynamic motion lines or particle effects when warranted. Reference specific anime visual languages (90s cel, modern Kyoto Animation polish, Ghibli warmth, Shinkai light). Keep human anatomy slightly stylized — long eyelashes, signature hair silhouettes, deliberate fashion. Backgrounds should have painterly anime depth (matte-painted skies, detailed environments). Mood and emotion are foregrounded.""",
|
|
|
|
"Stable Diffusion / Tag Style": """
|
|
|
|
AESTHETIC BIAS — Stable Diffusion Tag Style:
|
|
Write the prompt as a comma-separated stack of weighted tags and short phrases rather than flowing prose. Order: subject, descriptors, environment, lighting, style tags, artist refs, quality tags. Use parenthetical emphasis sparingly when needed. Keep individual tags short and specific. Skip narrative sentences.""",
|
|
|
|
"Photoreal Cinematic": """
|
|
|
|
AESTHETIC BIAS — Photoreal Cinematic:
|
|
Write the prompt as a cinematographer would describe a film still: specific camera body, lens (e.g., 35mm, 85mm), aperture (f/1.4 — f/8), film stock or digital sensor, lighting setup (key/fill/practical, color temperature in Kelvin), grain or clarity. Real materials, real physics, real bodies. No fantastical embellishment. Mood emerges from light and framing, not from adjectives.""",
|
|
|
|
"Concept Art / Painterly": """
|
|
|
|
AESTHETIC BIAS — Concept Art / Painterly:
|
|
Write the prompt as a brief for a senior concept artist. Visible brushwork, confident value structure, atmospheric perspective, painterly edges, controlled palette. References to studio concept artists (Craig Mullins, Sparth, Maciej Kuciara). Composition reads at thumbnail scale. Mood and shape language are foregrounded over rendered detail.""",
|
|
|
|
"Editorial Magazine": """
|
|
|
|
AESTHETIC BIAS — Editorial Magazine:
|
|
Write the prompt like an art director's brief for a Vogue, Wallpaper, or National Geographic spread. Clean composition, considered styling, decisive crop. Subject styled deliberately — wardrobe, posture, gaze. Lighting is intentional and named (window light, three-point, beauty dish). Color palette is curated. No fantastical elements unless the editorial itself is conceptual.""",
|
|
|
|
"Polaroid / Analog Film": """
|
|
|
|
AESTHETIC BIAS — Polaroid / Analog Film:
|
|
Write the prompt as if shot on Polaroid SX-70, Kodak Portra 400, Cinestill 800T, or 35mm film. Soft halation, gentle grain, slightly off color science (warm shadows, magenta highlights for tungsten film, blue cast for daylight stocks), light leaks if appropriate. Snapshot intimacy, not studio polish. Subject often candid, slightly off-center, imperfect framing.""",
|
|
|
|
"Watercolor / Ink Illustration": """
|
|
|
|
AESTHETIC BIAS — Watercolor / Ink Illustration:
|
|
Write the prompt for a watercolor or ink illustration. Bleeding pigments, paper texture visible, controlled white space, ink line variation. Soft edges where wet, crisp lines where dry. Loose painterly suggestion of detail rather than tight rendering. Often a single dominant hue with accent washes.""",
|
|
|
|
"Comic / Graphic Novel": """
|
|
|
|
AESTHETIC BIAS — Comic / Graphic Novel:
|
|
Write the prompt as a panel from a comic or graphic novel. Bold ink lines, dramatic inking with pools of black, halftone or flat color blocking, dynamic angle, strong silhouette. Reference style hints: Mike Mignola (chiaroscuro, heavy black), Jean Giraud/Moebius (clean line, dreamlike), Frank Miller (high contrast noir), Fiona Staples (modern soft color). Composition prioritizes readability and impact.""",
|
|
|
|
"3D Render / Octane": """
|
|
|
|
AESTHETIC BIAS — 3D Render / Octane:
|
|
Write the prompt as a brief for a high-end 3D render (Octane, Redshift, Unreal Engine 5). Subsurface scattering on skin, physically based materials, ray-traced global illumination, volumetric lighting, depth of field. Hyperreal materials but slightly idealized geometry. Specific HDR or studio lighting setup.""",
|
|
|
|
"Storybook / Children's Illustration": """
|
|
|
|
AESTHETIC BIAS — Storybook / Children's Illustration:
|
|
Write the prompt as a page from a beloved children's book. Soft warm palette, charming character design, gentle composition, hand-drawn or hand-painted feel. References: Quentin Blake, Beatrix Potter, Maurice Sendak, Jon Klassen, Oliver Jeffers, Studio Ghibli storyboards. Whimsy and warmth over realism.""",
|
|
}
|
|
|
|
|
|
_REFERENCE_IMAGE_SUBJECT_PREFACE = """\
|
|
|
|
You are also given a REFERENCE IMAGE. Treat it as STYLE/AESTHETIC reference. Study it for: lighting quality and direction, color palette and tonal relationships, compositional language, texture and grain, level of detail and abstraction, mood, era/medium signals. Then create a new scene that captures the same visual language. You may keep the subject from the reference image if it serves the constraints, but feel free to evolve it. Do not merely describe the reference — translate its DNA into something fresh.
|
|
"""
|
|
|
|
|
|
def _build_muse_user_message(
|
|
genre: str | None,
|
|
mood: str | None,
|
|
shot_type: str | None,
|
|
lighting: str | None,
|
|
color_palette: str | None,
|
|
artist: str | None,
|
|
subject: str | None,
|
|
era: str | None,
|
|
free_hint: str,
|
|
output_format: str,
|
|
) -> str:
|
|
parts = []
|
|
|
|
if genre:
|
|
parts.append(f"Genre/Style: {genre}")
|
|
if mood:
|
|
parts.append(f"Mood: {mood}")
|
|
if shot_type:
|
|
parts.append(f"Shot type: {shot_type}")
|
|
if lighting:
|
|
parts.append(f"Lighting: {lighting}")
|
|
if color_palette:
|
|
parts.append(f"Color palette: {color_palette}")
|
|
if artist:
|
|
parts.append(f"Artist reference: {artist}")
|
|
if subject:
|
|
parts.append(f"Subject: {subject}")
|
|
if era:
|
|
parts.append(f"Era: {era}")
|
|
if free_hint and free_hint.strip():
|
|
parts.append(f"Additional guidance: {free_hint.strip()}")
|
|
|
|
constraint_block = "\n".join(parts) if parts else "No constraints — dream freely."
|
|
|
|
if output_format == "Prompt Only":
|
|
return f"Create an image generation prompt using these creative constraints:\n\n{constraint_block}"
|
|
else:
|
|
return f"Create a full muse concept using these creative constraints:\n\n{constraint_block}"
|
|
|
|
|
|
def _extract_prompt_from_json(text: str) -> str:
|
|
"""Pull the 'prompt' field out of JSON output for the STRING output pin."""
|
|
try:
|
|
data = json.loads(text)
|
|
return str(data.get("prompt") or text)
|
|
except Exception:
|
|
# Try regex extraction if JSON parse fails
|
|
m = re.search(r'"prompt"\s*:\s*"(.*?)"(?:\s*[,}])', text, re.DOTALL)
|
|
if m:
|
|
return m.group(1).replace("\\n", "\n").replace('\\"', '"')
|
|
return text
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Muse engine (reuses VLMEngine in text-only mode)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_muse_engine = VLMEngine()
|
|
|
|
|
|
def _run_muse(
|
|
model_name: str,
|
|
genre_raw: str,
|
|
mood_raw: str,
|
|
shot_type_raw: str,
|
|
lighting_raw: str,
|
|
color_palette_raw: str,
|
|
artist_raw: str,
|
|
subject_raw: str,
|
|
era_raw: str,
|
|
free_hint: str,
|
|
output_format: str,
|
|
max_tokens: int,
|
|
temperature: float,
|
|
top_p: float,
|
|
repetition_penalty: float,
|
|
seed: int,
|
|
keep_model_loaded: bool,
|
|
device: str = "auto",
|
|
ctx: int | None = None,
|
|
n_batch: int | None = None,
|
|
gpu_layers: int | None = None,
|
|
top_k: int | None = None,
|
|
pool_size: int | None = None,
|
|
image=None,
|
|
video=None,
|
|
frame_count: int = 4,
|
|
reference_mode: str = "Style Only",
|
|
image_max_tokens: int | None = None,
|
|
aesthetic_bias: str = "None",
|
|
) -> tuple[str, str]:
|
|
|
|
torch.manual_seed(int(seed))
|
|
rng = random.Random(int(seed))
|
|
|
|
# Resolve Random selections
|
|
genre = _resolve(genre_raw, _get_list("genres"), rng)
|
|
mood = _resolve(mood_raw, _get_list("moods"), rng)
|
|
shot = _resolve(shot_type_raw, _get_list("shot_types"), rng)
|
|
light = _resolve(lighting_raw, _get_list("lighting"), rng)
|
|
palette = _resolve(color_palette_raw, _get_list("color_palettes"), rng)
|
|
artist = _resolve(artist_raw, _get_list("artists"), rng)
|
|
subject = _resolve(subject_raw, _get_list("subjects"), rng)
|
|
era = _resolve(era_raw, _get_list("eras"), rng)
|
|
|
|
# Log what was resolved
|
|
resolved = {k: v for k, v in {
|
|
"genre": genre, "mood": mood, "shot": shot, "lighting": light,
|
|
"palette": palette, "artist": artist, "subject": subject, "era": era,
|
|
}.items() if v}
|
|
if resolved:
|
|
print(f"[VLMPrompt/Muse] Resolved: {resolved}")
|
|
|
|
# Encode reference image / video frames (optional)
|
|
images_b64: list[str] = []
|
|
if image is not None:
|
|
b64 = _tensor_to_base64_png(image)
|
|
if b64:
|
|
images_b64.append(b64)
|
|
if video is not None:
|
|
for frame in _sample_video_frames(video, int(frame_count)):
|
|
b64 = _tensor_to_base64_png(frame)
|
|
if b64:
|
|
images_b64.append(b64)
|
|
|
|
has_reference = bool(images_b64)
|
|
|
|
base_system = _MUSE_SYSTEM_PROMPT_ONLY if output_format == "Prompt Only" else _MUSE_SYSTEM
|
|
if has_reference:
|
|
if reference_mode == "Style + Subject":
|
|
system_msg = base_system + _REFERENCE_IMAGE_SUBJECT_PREFACE
|
|
else: # "Style Only"
|
|
system_msg = base_system + _REFERENCE_IMAGE_PREFACE
|
|
print(f"[VLMPrompt/Muse] Reference image active (mode={reference_mode}, frames={len(images_b64)})")
|
|
else:
|
|
system_msg = base_system
|
|
|
|
# Append aesthetic bias if set
|
|
if aesthetic_bias and aesthetic_bias != "None":
|
|
bias_text = _AESTHETIC_BIASES.get(aesthetic_bias, "")
|
|
if bias_text:
|
|
system_msg = system_msg + bias_text
|
|
print(f"[VLMPrompt/Muse] Aesthetic bias: {aesthetic_bias}")
|
|
|
|
user_msg = _build_muse_user_message(
|
|
genre, mood, shot, light, palette, artist, subject, era, free_hint, output_format
|
|
)
|
|
|
|
all_models = MODELS_CATALOG.get("models") or {}
|
|
entry = all_models.get(model_name) or {}
|
|
stop_tokens = entry.get("stop_tokens")
|
|
|
|
try:
|
|
_muse_engine._load_model(
|
|
model_name=model_name,
|
|
device=device,
|
|
ctx=ctx,
|
|
n_batch=n_batch,
|
|
gpu_layers=gpu_layers,
|
|
image_max_tokens=image_max_tokens,
|
|
top_k=top_k,
|
|
pool_size=pool_size,
|
|
vision_mode=has_reference,
|
|
)
|
|
|
|
if has_reference and _muse_engine.chat_handler is None:
|
|
print("[VLMPrompt/Muse] Warning: reference image given but model has no mmproj — image ignored")
|
|
|
|
raw = _muse_engine._invoke(
|
|
system_prompt=system_msg,
|
|
user_prompt=user_msg,
|
|
images_b64=images_b64 if _muse_engine.chat_handler is not None else [],
|
|
max_tokens=int(max_tokens),
|
|
temperature=float(temperature),
|
|
top_p=float(top_p),
|
|
repetition_penalty=float(repetition_penalty),
|
|
seed=int(seed),
|
|
stop_tokens=stop_tokens,
|
|
)
|
|
|
|
if output_format == "Prompt Only":
|
|
return (raw, "{}")
|
|
elif output_format == "JSON Only":
|
|
prompt_str = _extract_prompt_from_json(raw)
|
|
return (prompt_str, raw)
|
|
else: # JSON + Prompt
|
|
prompt_str = _extract_prompt_from_json(raw)
|
|
return (prompt_str, raw)
|
|
|
|
finally:
|
|
if not keep_model_loaded:
|
|
_muse_engine.clear()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ComfyUI Nodes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class MuseNode:
|
|
"""Dream up vivid image prompts from genre/mood/artist constraints."""
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
all_models = MODELS_CATALOG.get("models") or {}
|
|
model_keys = sorted(all_models.keys()) or ["(edit models.json)"]
|
|
|
|
genres = _selectable("genres")
|
|
moods = _selectable("moods")
|
|
shot_types = _selectable("shot_types")
|
|
artists = _selectable("artists")
|
|
subjects = _selectable("subjects")
|
|
|
|
return {
|
|
"required": {
|
|
"model_name": (model_keys, {"default": model_keys[0]}),
|
|
"genre": (genres, {"default": "Random"}),
|
|
"mood": (moods, {"default": "Random"}),
|
|
"shot_type": (shot_types, {"default": "Random"}),
|
|
"artist": (artists, {"default": "Random"}),
|
|
"subject": (subjects, {"default": "Random"}),
|
|
"free_hint": ("STRING", {
|
|
"default": "",
|
|
"multiline": True,
|
|
"tooltip": "Optional freeform guidance — a word, phrase, character description, or constraint. Leave empty to dream freely.",
|
|
}),
|
|
"output_format": (["JSON + Prompt", "Prompt Only", "JSON Only"], {"default": "JSON + Prompt"}),
|
|
"aesthetic_bias": (_selectable("aesthetic_biases"), {
|
|
"default": "Midjourney v6/v7 — Painterly Realism",
|
|
"tooltip": "Bias the generated prompt toward a specific output aesthetic. 'Midjourney v6/v7' = cinematic painterly realism (recommended for the MidjourneyV1Z LoRA).",
|
|
}),
|
|
"reference_mode": (["Style Only", "Style + Subject"], {
|
|
"default": "Style Only",
|
|
"tooltip": "When a reference image is connected: 'Style Only' invents a totally new scene matching the visual language. 'Style + Subject' may keep the subject too.",
|
|
}),
|
|
"max_tokens": ("INT", {"default": 900, "min": 256, "max": 4096}),
|
|
"keep_model_loaded": ("BOOLEAN", {"default": False}),
|
|
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1}),
|
|
},
|
|
"optional": {
|
|
"image": ("IMAGE", {"tooltip": "Optional style reference image — Muse dreams a NEW scene with the same visual DNA"}),
|
|
"video": ("IMAGE", {"tooltip": "Optional video frames as style reference (sampled)"}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("STRING", "STRING")
|
|
RETURN_NAMES = ("prompt", "json")
|
|
FUNCTION = "muse"
|
|
CATEGORY = "VLMPrompt"
|
|
|
|
def muse(
|
|
self, model_name, genre, mood, shot_type, artist, subject,
|
|
free_hint, output_format, aesthetic_bias, reference_mode,
|
|
max_tokens, keep_model_loaded, seed,
|
|
image=None, video=None,
|
|
):
|
|
return _run_muse(
|
|
model_name=model_name,
|
|
genre_raw=genre,
|
|
mood_raw=mood,
|
|
shot_type_raw=shot_type,
|
|
lighting_raw="Random",
|
|
color_palette_raw="Random",
|
|
artist_raw=artist,
|
|
subject_raw=subject,
|
|
era_raw="Random",
|
|
free_hint=free_hint,
|
|
output_format=output_format,
|
|
max_tokens=max_tokens,
|
|
temperature=0.85,
|
|
top_p=0.92,
|
|
repetition_penalty=1.1,
|
|
seed=seed,
|
|
keep_model_loaded=keep_model_loaded,
|
|
image=image,
|
|
video=video,
|
|
reference_mode=reference_mode,
|
|
aesthetic_bias=aesthetic_bias,
|
|
)
|
|
|
|
|
|
class MuseAdvancedNode:
|
|
"""Full-control Muse node — every dimension explicit, including lighting, era, palette."""
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
all_models = MODELS_CATALOG.get("models") or {}
|
|
model_keys = sorted(all_models.keys()) or ["(edit models.json)"]
|
|
|
|
genres = _selectable("genres")
|
|
moods = _selectable("moods")
|
|
shot_types = _selectable("shot_types")
|
|
lightings = _selectable("lighting")
|
|
color_palettes = _selectable("color_palettes")
|
|
artists = _selectable("artists")
|
|
subjects = _selectable("subjects")
|
|
eras = _selectable("eras")
|
|
|
|
num_gpus = torch.cuda.device_count()
|
|
gpu_list = [f"cuda:{i}" for i in range(num_gpus)]
|
|
device_options = ["auto", "cpu"] + gpu_list
|
|
|
|
return {
|
|
"required": {
|
|
"model_name": (model_keys, {"default": model_keys[0]}),
|
|
"device": (device_options, {"default": "auto"}),
|
|
"genre": (genres, {"default": "Random"}),
|
|
"mood": (moods, {"default": "Random"}),
|
|
"shot_type": (shot_types, {"default": "Random"}),
|
|
"lighting": (lightings, {"default": "Random"}),
|
|
"color_palette": (color_palettes, {"default": "Random"}),
|
|
"artist": (artists, {"default": "Random"}),
|
|
"subject": (subjects, {"default": "Random"}),
|
|
"era": (eras, {"default": "Random"}),
|
|
"free_hint": ("STRING", {
|
|
"default": "",
|
|
"multiline": True,
|
|
"tooltip": "Optional freeform guidance. Leave empty to dream freely.",
|
|
}),
|
|
"output_format": (["JSON + Prompt", "Prompt Only", "JSON Only"], {"default": "JSON + Prompt"}),
|
|
"aesthetic_bias": (_selectable("aesthetic_biases"), {"default": "None"}),
|
|
"reference_mode": (["Style Only", "Style + Subject"], {"default": "Style Only"}),
|
|
"frame_count": ("INT", {"default": 4, "min": 1, "max": 32}),
|
|
"image_max_tokens": ("INT", {"default": 4096, "min": 256, "max": 1024000, "step": 256}),
|
|
"max_tokens": ("INT", {"default": 1024, "min": 256, "max": 4096}),
|
|
"temperature": ("FLOAT", {"default": 0.85, "min": 0.0, "max": 2.0, "step": 0.05}),
|
|
"top_p": ("FLOAT", {"default": 0.92, "min": 0.0, "max": 1.0, "step": 0.05}),
|
|
"repetition_penalty": ("FLOAT", {"default": 1.1, "min": 0.5, "max": 2.0, "step": 0.05}),
|
|
"ctx": ("INT", {"default": 8192, "min": 1024, "max": 131072, "step": 512}),
|
|
"n_batch": ("INT", {"default": 512, "min": 64, "max": 32768, "step": 64}),
|
|
"gpu_layers": ("INT", {"default": -1, "min": -1, "max": 200}),
|
|
"top_k": ("INT", {"default": 0, "min": 0, "max": 32768}),
|
|
"pool_size": ("INT", {"default": 4194304, "min": 1048576, "max": 10485760, "step": 524288}),
|
|
"keep_model_loaded": ("BOOLEAN", {"default": False}),
|
|
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1}),
|
|
},
|
|
"optional": {
|
|
"image": ("IMAGE", {"tooltip": "Optional style reference image"}),
|
|
"video": ("IMAGE", {"tooltip": "Optional video frames as style reference"}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("STRING", "STRING")
|
|
RETURN_NAMES = ("prompt", "json")
|
|
FUNCTION = "muse"
|
|
CATEGORY = "VLMPrompt"
|
|
|
|
def muse(
|
|
self,
|
|
model_name, device, genre, mood, shot_type, lighting, color_palette,
|
|
artist, subject, era, free_hint, output_format, aesthetic_bias, reference_mode,
|
|
frame_count, image_max_tokens,
|
|
max_tokens, temperature, top_p, repetition_penalty,
|
|
ctx, n_batch, gpu_layers, top_k, pool_size,
|
|
keep_model_loaded, seed,
|
|
image=None, video=None,
|
|
):
|
|
return _run_muse(
|
|
model_name=model_name,
|
|
genre_raw=genre,
|
|
mood_raw=mood,
|
|
shot_type_raw=shot_type,
|
|
lighting_raw=lighting,
|
|
color_palette_raw=color_palette,
|
|
artist_raw=artist,
|
|
subject_raw=subject,
|
|
era_raw=era,
|
|
free_hint=free_hint,
|
|
output_format=output_format,
|
|
max_tokens=max_tokens,
|
|
temperature=temperature,
|
|
top_p=top_p,
|
|
repetition_penalty=repetition_penalty,
|
|
seed=seed,
|
|
keep_model_loaded=keep_model_loaded,
|
|
device=device,
|
|
ctx=ctx,
|
|
n_batch=n_batch,
|
|
gpu_layers=gpu_layers,
|
|
top_k=top_k,
|
|
pool_size=pool_size,
|
|
image=image,
|
|
video=video,
|
|
frame_count=frame_count,
|
|
reference_mode=reference_mode,
|
|
image_max_tokens=image_max_tokens,
|
|
aesthetic_bias=aesthetic_bias,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Mappings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
NODE_CLASS_MAPPINGS = {
|
|
"VLMMuse": MuseNode,
|
|
"VLMMuseAdvanced": MuseAdvancedNode,
|
|
}
|
|
|
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
|
"VLMMuse": "Muse (GGUF)",
|
|
"VLMMuseAdvanced": "Muse Advanced (GGUF)",
|
|
}
|