From e1d965f3e9f97e0f2b60df96162efa8aa59bfeb2 Mon Sep 17 00:00:00 2001 From: Dag Thomas Olsen Date: Sat, 6 Dec 2025 20:53:18 +0100 Subject: [PATCH] next scene --- README.md | 49 ++ __init__.py | 14 + .../custom_prompts/qwen_next_scene_simple.txt | 18 + data/custom_prompts/qwen_next_scene_video.txt | 30 ++ nodes/qwenvl/__init__.py | 4 +- nodes/qwenvl/frame_prep.py | 136 +++++ nodes/qwenvl/next_scene.py | 482 ++++++++++++++++++ 7 files changed, 732 insertions(+), 1 deletion(-) create mode 100644 data/custom_prompts/qwen_next_scene_simple.txt create mode 100644 data/custom_prompts/qwen_next_scene_video.txt create mode 100644 nodes/qwenvl/frame_prep.py create mode 100644 nodes/qwenvl/next_scene.py diff --git a/README.md b/README.md index b6e81a6..e1942df 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,55 @@ Analyze video content frame-by-frame. --- +#### QwenVL Next Scene +**Display Name:** `APNext QwenVL Next Scene` + +Generate cinematic scene transitions locally using QwenVL models. Takes a previous scene description and **1-5 frame images**, then creates natural camera movements, framing evolution, and atmospheric shifts. Multiple frames help the model understand motion/progression. + +| Input | Description | +|-------|-------------| +| `images` | 1-5 frame images (batch) | +| `original_prompt` | Previous scene description | +| `qwen_model` | QwenVL model to use | +| `prompt_file` | Custom prompt template file | +| `custom_prompt` | Override with inline prompt (optional) | +| `max_frames` | Max frames to use from batch (1-5) | +| `focus_on` | Camera Movement, Framing Evolution, Environmental Reveals, Atmospheric Shifts | +| `transition_intensity` | Subtle, Moderate, or Dramatic | +| `keep_model_loaded` | Cache model in memory | + +**Returns:** `(next_scene_prompt, short_description)` + +**Custom Prompts:** Create your own prompt templates in `data/custom_prompts/`. Use `##ORIGINAL_PROMPT##` as placeholder for the previous scene description. Included templates: +- `next_scene.txt` - Default detailed cinematography prompt +- `qwen_next_scene_simple.txt` - Simplified version +- `qwen_next_scene_video.txt` - Optimized for AI video generation + +--- + +#### QwenVL Frame Prep +**Display Name:** `APNext QwenVL Frame Prep` + +Utility node to prepare multiple images for QwenVL Next Scene. Accepts up to 5 individual images or a batch, scales them to max dimensions, and outputs a batched tensor. + +| Input | Description | +|-------|-------------| +| `max_width` | Maximum width (default 1024) | +| `max_height` | Maximum height (default 1024) | +| `image_1` - `image_5` | Individual image inputs | +| `image_batch` | Pre-batched images (optional) | + +**Returns:** `(images, frame_count)` + +--- + +#### QwenVL Z-Image Vision +**Display Name:** `APNext QwenVL Z-Image Vision` + +Analyzes images and outputs in Z-Image TurnBuilder chat format with `<|im_start|>/<|im_end|>` tokens. + +--- + ### ๐Ÿฆ™ Ollama Nodes (Local LLM) #### Ollama Node diff --git a/__init__.py b/__init__.py index 78952cd..4e90e07 100644 --- a/__init__.py +++ b/__init__.py @@ -310,6 +310,20 @@ try: except Exception: pass +try: + from .nodes.qwenvl.next_scene import QwenVLNextScene + NEW_MAPPINGS["QwenVLNextScene"] = QwenVLNextScene + NEW_DISPLAY_MAPPINGS["QwenVLNextScene"] = "APNext QwenVL Next Scene" +except Exception: + pass + +try: + from .nodes.qwenvl.frame_prep import QwenVLFramePrep + NEW_MAPPINGS["QwenVLFramePrep"] = QwenVLFramePrep + NEW_DISPLAY_MAPPINGS["QwenVLFramePrep"] = "APNext QwenVL Frame Prep" +except Exception: + pass + # APNext Advanced FX Nodes try: from .nodes.image_fx.color_grading import APNextColorGrading diff --git a/data/custom_prompts/qwen_next_scene_simple.txt b/data/custom_prompts/qwen_next_scene_simple.txt new file mode 100644 index 0000000..da2eb25 --- /dev/null +++ b/data/custom_prompts/qwen_next_scene_simple.txt @@ -0,0 +1,18 @@ +You are a cinematographer assistant. Analyze the provided image(s) and generate the next scene in a visual narrative. + +PREVIOUS SCENE: +##ORIGINAL_PROMPT## + +Your task is to: +1. Understand what the previous prompt described +2. Analyze what the current image shows +3. Create a natural, cinematic transition to the next scene + +OUTPUT FORMAT: +Start with "Next Scene:" and provide a 2-3 sentence description including: +- Camera movement or framing change +- New elements appearing or exiting frame +- Atmospheric/environmental evolution + +CRITICAL: Output ONLY the "Next Scene:" prompt. No explanations or preamble. + diff --git a/data/custom_prompts/qwen_next_scene_video.txt b/data/custom_prompts/qwen_next_scene_video.txt new file mode 100644 index 0000000..d35a809 --- /dev/null +++ b/data/custom_prompts/qwen_next_scene_video.txt @@ -0,0 +1,30 @@ +You are a video director assistant specialized in creating smooth, continuous scene transitions for AI video generation. + +PREVIOUS SCENE DESCRIPTION: +##ORIGINAL_PROMPT## + +Your task is to: +1. Analyze the current frame(s) carefully +2. Understand the established motion, composition, and atmosphere +3. Generate the NEXT scene that flows naturally from the current one + +Focus on VIDEO-FRIENDLY transitions: +- Smooth camera movements (no jarring cuts) +- Gradual lighting/atmosphere changes +- Continuous subject motion +- Natural environmental progression + +AVOID: +- Abrupt scene changes +- Teleporting subjects +- Instant lighting changes +- Breaking spatial continuity + +OUTPUT: +Start with "Next Scene:" and describe in 2-4 sentences: +- The camera movement continuation +- Subject/environment progression +- Atmospheric evolution + +Keep descriptions concise for AI video generation. No explanations, just the prompt. + diff --git a/nodes/qwenvl/__init__.py b/nodes/qwenvl/__init__.py index 210498e..d9ddb44 100644 --- a/nodes/qwenvl/__init__.py +++ b/nodes/qwenvl/__init__.py @@ -8,5 +8,7 @@ from .vision_node import QwenVLVisionNode from .vision_cloner import QwenVLVisionCloner from .video_node import QwenVLVideoNode from .zimage_vision import QwenVLZImageVision +from .next_scene import QwenVLNextScene +from .frame_prep import QwenVLFramePrep -__all__ = ['QwenVLVisionNode', 'QwenVLVisionCloner', 'QwenVLVideoNode', 'QwenVLZImageVision'] +__all__ = ['QwenVLVisionNode', 'QwenVLVisionCloner', 'QwenVLVideoNode', 'QwenVLZImageVision', 'QwenVLNextScene', 'QwenVLFramePrep'] diff --git a/nodes/qwenvl/frame_prep.py b/nodes/qwenvl/frame_prep.py new file mode 100644 index 0000000..5897390 --- /dev/null +++ b/nodes/qwenvl/frame_prep.py @@ -0,0 +1,136 @@ +# QwenVL Frame Prep Node +# Prepares multiple images for QwenVL Next Scene node + +import numpy as np +import torch +from PIL import Image + +from ...utils.constants import CUSTOM_CATEGORY + + +class QwenVLFramePrep: + """ + Prepares multiple images for the QwenVL Next Scene node. + - Accepts multiple image inputs + - Scales images to max width (default 1024) maintaining aspect ratio + - Batches them into a single IMAGE output for QwenVL Next Scene + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "max_width": ("INT", {"default": 1024, "min": 256, "max": 2048, "step": 64}), + "max_height": ("INT", {"default": 1024, "min": 256, "max": 2048, "step": 64}), + }, + "optional": { + "image_1": ("IMAGE",), + "image_2": ("IMAGE",), + "image_3": ("IMAGE",), + "image_4": ("IMAGE",), + "image_5": ("IMAGE",), + "image_batch": ("IMAGE",), # Can also accept a pre-batched input + } + } + + RETURN_TYPES = ("IMAGE", "INT") + RETURN_NAMES = ("images", "frame_count") + FUNCTION = "prepare_frames" + CATEGORY = f"{CUSTOM_CATEGORY}/LLM" + + @staticmethod + def tensor2pil(image): + """Convert tensor to PIL image""" + return Image.fromarray( + np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8) + ) + + @staticmethod + def pil2tensor(image): + """Convert PIL image to tensor""" + return torch.from_numpy(np.array(image).astype(np.float32) / 255.0) + + def scale_image(self, pil_image, max_width, max_height): + """Scale image to fit within max dimensions while maintaining aspect ratio""" + width, height = pil_image.size + + # Calculate scale factors + scale_w = max_width / width if width > max_width else 1.0 + scale_h = max_height / height if height > max_height else 1.0 + scale = min(scale_w, scale_h) + + if scale < 1.0: + new_width = int(width * scale) + new_height = int(height * scale) + pil_image = pil_image.resize((new_width, new_height), Image.Resampling.LANCZOS) + + return pil_image + + def prepare_frames( + self, + max_width=1024, + max_height=1024, + image_1=None, + image_2=None, + image_3=None, + image_4=None, + image_5=None, + image_batch=None, + ): + frames = [] + + # Collect individual images + individual_images = [image_1, image_2, image_3, image_4, image_5] + + for img in individual_images: + if img is not None: + # Handle batched inputs - take first frame from each + if len(img.shape) == 4: + frames.append(img[0]) + else: + frames.append(img) + + # Add frames from batch input + if image_batch is not None: + if len(image_batch.shape) == 4: + for i in range(image_batch.shape[0]): + frames.append(image_batch[i]) + else: + frames.append(image_batch) + + if not frames: + raise ValueError("No images provided! Connect at least one image input.") + + # Process and scale frames + processed_frames = [] + target_size = None + + for i, frame_tensor in enumerate(frames): + # Convert to PIL + pil_image = self.tensor2pil(frame_tensor) + + # Scale to max dimensions + pil_image = self.scale_image(pil_image, max_width, max_height) + + # Use first frame's size as target for all frames + if target_size is None: + target_size = pil_image.size + else: + # Resize to match first frame if different + if pil_image.size != target_size: + pil_image = pil_image.resize(target_size, Image.Resampling.LANCZOS) + + # Convert back to tensor + tensor = self.pil2tensor(pil_image) + processed_frames.append(tensor) + + print(f"๐Ÿ–ผ๏ธ Frame {i+1}: {pil_image.size[0]}x{pil_image.size[1]}") + + # Stack into batch + output = torch.stack(processed_frames, dim=0) + frame_count = len(processed_frames) + + print(f"โœ… Prepared {frame_count} frames for QwenVL Next Scene") + print(f" Output shape: {output.shape}") + + return (output, frame_count) diff --git a/nodes/qwenvl/next_scene.py b/nodes/qwenvl/next_scene.py new file mode 100644 index 0000000..e5a0311 --- /dev/null +++ b/nodes/qwenvl/next_scene.py @@ -0,0 +1,482 @@ +# QwenVL Next Scene Node +# Generates cinematic scene transitions using local QwenVL models + +import os +import re +import random +import json +import numpy as np +import torch +import gc +from pathlib import Path +from PIL import Image +from transformers import AutoModelForVision2Seq, AutoProcessor, AutoTokenizer + +from ...utils.constants import CUSTOM_CATEGORY, qwenvl_models, prompt_dir +import folder_paths + + +class QwenVLNextScene: + """ + Generates the next scene in a visual narrative using local QwenVL models. + Takes an input prompt (previous scene description) and multiple frame images, then creates + a cinematic transition FROM the previous scene TO the next scene with camera movements, + framing evolution, and atmospheric shifts. + + Supports 1-5 frames as input for better motion/progression understanding. + Supports custom prompt files from data/custom_prompts/ directory. + """ + + # Class-level cache for model + _cached_model = None + _cached_processor = None + _cached_tokenizer = None + _cached_model_name = None + + # Default prompt file + DEFAULT_PROMPT_FILE = "next_scene.txt" + + def __init__(self): + pass + + @staticmethod + def get_prompt_files(): + """Get list of available prompt files from custom_prompts directory""" + try: + files = [f for f in os.listdir(prompt_dir) if f.endswith(".txt")] + return ["(none)"] + sorted(files) + except Exception: + return ["(none)"] + + def load_prompt_from_file(self, prompt_file): + """Load prompt content from a file in the custom_prompts directory""" + if prompt_file == "(none)" or not prompt_file: + return None + + file_path = os.path.join(prompt_dir, prompt_file) + if not os.path.exists(file_path): + print(f"โš ๏ธ Prompt file not found: {file_path}") + return None + + try: + with open(file_path, "r", encoding="utf-8") as f: + return f.read().strip() + except Exception as e: + print(f"โš ๏ธ Error loading prompt file: {e}") + return None + + @classmethod + def INPUT_TYPES(cls): + prompt_files = cls.get_prompt_files() + default_prompt = cls.DEFAULT_PROMPT_FILE if cls.DEFAULT_PROMPT_FILE in prompt_files else "(none)" + + return { + "required": { + "images": ("IMAGE",), + "original_prompt": ("STRING", {"multiline": True, "default": ""}), + "qwen_model": (qwenvl_models, {"default": qwenvl_models[0] if qwenvl_models else "Qwen3-VL-4B-Instruct"}), + "prompt_file": (prompt_files, {"default": default_prompt}), + "max_frames": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}), + "seed": ("INT", {"default": -1, "min": -1, "max": 0xffffffffffffffff}), + "randomize_each_run": ("BOOLEAN", {"default": True}), + "add_scene_prefix": ("BOOLEAN", {"default": True}), + "keep_model_loaded": ("BOOLEAN", {"default": True}), + }, + "optional": { + "custom_prompt": ("STRING", {"multiline": True, "default": ""}), + "scene_prefix_text": ("STRING", {"default": "NEW SCENE:"}), + "focus_on": ( + ["Automatic", "Camera Movement", "Framing Evolution", "Environmental Reveals", "Atmospheric Shifts"], + {"default": "Automatic"} + ), + "transition_intensity": ( + ["Subtle", "Moderate", "Dramatic"], + {"default": "Moderate"} + ), + "max_tokens": ("INT", {"default": 1024, "min": 256, "max": 4096}), + "temperature": ("FLOAT", {"default": 0.7, "min": 0.1, "max": 1.5}), + } + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("next_scene_prompt", "short_description") + FUNCTION = "generate_next_scene" + CATEGORY = f"{CUSTOM_CATEGORY}/LLM" + + @staticmethod + def tensor2pil(image): + """Convert tensor to PIL image""" + return Image.fromarray( + np.clip(255.0 * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8) + ) + + @classmethod + def ensure_model(cls, model_name): + """Get model path and cache directory""" + models_dir = Path(folder_paths.models_dir) / "LLM" / "Qwen-VL" + models_dir.mkdir(parents=True, exist_ok=True) + + # Handle full repo paths (e.g., "huihui-ai/Model-Name") vs simple names (e.g., "Qwen3-VL-4B") + if "/" in model_name: + repo_id = model_name # Use as-is + folder_name = model_name.replace("/", "--") # Safe folder name + else: + repo_id = f"Qwen/{model_name}" # Default Qwen namespace + folder_name = model_name + + target = models_dir / folder_name + + # Check if model files exist locally (manually placed) + if target.exists(): + safetensors = list(target.glob("*.safetensors")) + bins = list(target.glob("*.bin")) + config = target / "config.json" + if (safetensors or bins) and config.exists(): + print(f"๐Ÿ“ Using local model: {target}") + return str(target), None # No cache_dir needed for local + + # Return repo_id and cache_dir for downloading to local folder + print(f"๐Ÿ“ฅ Will download {repo_id} to {models_dir}") + return repo_id, str(models_dir) + + @classmethod + def load_model(cls, model_name, keep_loaded=True): + """Load model with caching support""" + if keep_loaded and cls._cached_model is not None and cls._cached_model_name == model_name: + print(f"โ™ป๏ธ Using cached {model_name}") + return cls._cached_model, cls._cached_processor, cls._cached_tokenizer + + if cls._cached_model_name != model_name: + cls.clear_model() + + model_path, cache_dir = cls.ensure_model(model_name) + print(f"๐Ÿ”„ Loading {model_name}...") + + device = "cuda" if torch.cuda.is_available() else "cpu" + dtype = torch.float16 if torch.cuda.is_available() else torch.float32 + + # Build kwargs - add cache_dir if downloading + load_kwargs = { + "device_map": {"": 0} if device == "cuda" else device, + "dtype": dtype, + "attn_implementation": "sdpa", + "use_safetensors": True, + "trust_remote_code": True, + } + if cache_dir: + load_kwargs["cache_dir"] = cache_dir + + model = AutoModelForVision2Seq.from_pretrained(model_path, **load_kwargs).eval() + + processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True, cache_dir=cache_dir) + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, cache_dir=cache_dir) + + if keep_loaded: + cls._cached_model = model + cls._cached_processor = processor + cls._cached_tokenizer = tokenizer + cls._cached_model_name = model_name + + print(f"โœ… Loaded {model_name}") + return model, processor, tokenizer + + @classmethod + def clear_model(cls): + """Clear cached model""" + if cls._cached_model is not None: + print(f"๐Ÿงน Clearing cached model") + cls._cached_model = None + cls._cached_processor = None + cls._cached_tokenizer = None + cls._cached_model_name = None + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + def build_prompt(self, original_prompt, focus_on, transition_intensity, num_frames=1, prompt_file="next_scene.txt", custom_prompt=""): + """Build the full prompt for scene generation""" + + # Priority: custom_prompt > prompt_file > default fallback + system_prompt = None + + if custom_prompt and custom_prompt.strip(): + system_prompt = custom_prompt.strip() + print(f"๐Ÿ“„ Using custom prompt (inline)") + else: + system_prompt = self.load_prompt_from_file(prompt_file) + if system_prompt: + print(f"๐Ÿ“„ Using prompt file: {prompt_file}") + + # Add multi-frame context if applicable + frame_context = "" + if num_frames > 1: + frame_context = f""" +NOTE: You are provided with {num_frames} sequential frames showing the progression of the current scene. +Analyze the MOTION and CHANGES across these frames to understand the direction/flow of the scene. +Frame 1 is the earliest, Frame {num_frames} is the most recent. +Use this temporal information to predict a natural continuation. + +""" + + if original_prompt and original_prompt.strip() and system_prompt: + # Use the loaded system prompt with original prompt context for proper transitions + full_prompt = system_prompt.replace("##ORIGINAL_PROMPT##", original_prompt) + if frame_context: + # Insert frame context after the original prompt section + full_prompt = full_prompt.replace("Your task is to:", f"{frame_context}Your task is to:") + else: + # Create a simplified prompt that just analyzes the image without previous context + full_prompt = f"""You are an expert cinematographer and visual storytelling assistant. + +Analyze the provided {"images" if num_frames > 1 else "image"} and generate a detailed description of the "Next Scene" in a cinematic narrative. +{frame_context} +NOTE: No previous scene prompt was provided, so focus on analyzing the current {"frames" if num_frames > 1 else "image"} and suggesting a natural cinematic evolution from what you see. + +First, understand what's happening in the current {"sequence" if num_frames > 1 else "image"}, then create a natural, cinematic transition to the next scene. + +Consider these elements: + +CAMERA MOVEMENTS: +- Dolly shots (push-in, pull-back) +- Tracking moves (lateral, forward, backward) +- Panning (left, right) +- Tilting (up, down) +- Crane shots (rising, descending) +- Zooming (in, out) + +FRAMING EVOLUTION: +- Wide shot to close-up transitions +- Close-up to wide shot reveals +- Angle shifts (high angle, low angle, dutch angle) +- Reframing of subjects + +ENVIRONMENTAL REVEALS: +- New characters entering frame +- Expanded scenery and locations +- Background elements becoming foreground +- Hidden details revealed + +ATMOSPHERIC SHIFTS: +- Lighting changes (dramatic to soft, bright to dark) +- Weather evolution (clear to storm, fog rolling in) +- Time-of-day transitions (day to night, golden hour) +- Color palette shifts +- Mood and tone evolution + +OUTPUT FORMAT: +Start with "Next Scene:" and provide a detailed, cinematic description (2-4 sentences) that: +1. {"Continues the motion/direction observed in the frames" if num_frames > 1 else "Describes a specific camera movement or framing change"} +2. Mentions what new elements appear or what exits frame +3. Captures the atmospheric or environmental evolution +4. Uses precise, professional cinematography language + +CRITICAL: OUTPUT ONLY THE FINISHED "NEXT SCENE:" PROMPT - NO explanations or preambles.""" + + # Add focus guidance if specified + if focus_on != "Automatic": + full_prompt += f"\n\nFOCUS: Emphasize {focus_on.lower()} in your next scene description." + + # Add intensity guidance + intensity_map = { + "Subtle": "The transition should be subtle and gentle, maintaining most of the original composition.", + "Moderate": "The transition should be noticeable but natural, introducing clear changes while maintaining continuity.", + "Dramatic": "The transition should be bold and impactful, creating significant visual change while respecting the narrative." + } + full_prompt += f"\n\nINTENSITY: {intensity_map[transition_intensity]}" + + return full_prompt + + def extract_frames(self, images, max_frames): + """Extract evenly spaced frames from an image batch""" + pil_images = [] + + # Handle different tensor shapes + if len(images.shape) == 3: + # Single image (H, W, C) + pil_images.append(self.tensor2pil(images)) + elif len(images.shape) == 4: + # Batch of images (B, H, W, C) + batch_size = images.shape[0] + + if batch_size <= max_frames: + # Use all available frames + for i in range(batch_size): + pil_images.append(self.tensor2pil(images[i])) + else: + # Sample evenly spaced frames + indices = np.linspace(0, batch_size - 1, max_frames, dtype=int) + for idx in indices: + pil_images.append(self.tensor2pil(images[idx])) + + return pil_images + + def generate_next_scene( + self, + images, + original_prompt="", + qwen_model="Qwen3-VL-4B-Instruct", + prompt_file="next_scene.txt", + max_frames=3, + seed=-1, + randomize_each_run=True, + add_scene_prefix=True, + keep_model_loaded=True, + custom_prompt="", + scene_prefix_text="NEW SCENE:", + focus_on="Automatic", + transition_intensity="Moderate", + max_tokens=1024, + temperature=0.7 + ): + try: + # Handle seed for randomization + if randomize_each_run and seed == -1: + current_seed = random.randint(0, 0xffffffffffffffff) + elif seed == -1: + current_seed = 12345 + else: + current_seed = seed + + random.seed(current_seed) + + print("\n" + "="*80) + print("๐ŸŽฌ QWENVL NEXT SCENE - Processing") + print("="*80) + if original_prompt and original_prompt.strip(): + print(f"๐Ÿ“ Previous Scene (transitioning FROM): {original_prompt[:100]}{'...' if len(original_prompt) > 100 else ''}") + print("โœจ Using full transition mode - creating next scene based on previous prompt") + else: + print("๐Ÿ“ Previous Scene: (None - analyzing images only)") + print("โ„น๏ธ Using image-only mode - no previous prompt for transition") + print(f"๐Ÿค– Model: {qwen_model}") + print(f"๐ŸŽฏ Focus: {focus_on}") + print(f"โšก Intensity: {transition_intensity}") + print(f"๐ŸŽฒ Using seed: {current_seed}") + print("-"*80) + + # Load model + model, processor, tokenizer = self.load_model(qwen_model, keep_model_loaded) + + # Extract frames from the batch + pil_images = self.extract_frames(images, max_frames) + num_frames = len(pil_images) + + print(f"๐Ÿ–ผ๏ธ Using {num_frames} frame{'s' if num_frames > 1 else ''}") + for i, img in enumerate(pil_images): + print(f" Frame {i+1}: {img.size}") + print("๐Ÿ”„ Generating next scene...") + + # Build the prompt with frame count context + full_prompt = self.build_prompt(original_prompt, focus_on, transition_intensity, num_frames, prompt_file, custom_prompt) + + # Prepare conversation with multiple images + conversation = [{"role": "user", "content": []}] + + # Add frame labels for multiple frames + if num_frames > 1: + for i, pil_image in enumerate(pil_images): + conversation[0]["content"].append({"type": "text", "text": f"Frame {i+1}:"}) + conversation[0]["content"].append({"type": "image", "image": pil_image}) + conversation[0]["content"].append({"type": "text", "text": f"\n{full_prompt}"}) + else: + # Single image - no frame label needed + conversation[0]["content"].append({"type": "image", "image": pil_images[0]}) + conversation[0]["content"].append({"type": "text", "text": full_prompt}) + + # Apply chat template + chat = processor.apply_chat_template( + conversation, tokenize=False, add_generation_prompt=True + ) + + # Process inputs with all images + processed = processor(text=chat, images=pil_images, return_tensors="pt") + + # Move to device + model_device = next(model.parameters()).device + model_inputs = { + key: value.to(model_device) if torch.is_tensor(value) else value + for key, value in processed.items() + } + + # Generate + stop_tokens = [tokenizer.eos_token_id] + if hasattr(tokenizer, "eot_id") and tokenizer.eot_id is not None: + stop_tokens.append(tokenizer.eot_id) + + with torch.no_grad(): + outputs = model.generate( + **model_inputs, + max_new_tokens=max_tokens, + temperature=temperature, + do_sample=True, + top_p=0.9, + eos_token_id=stop_tokens, + pad_token_id=tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id, + ) + + if torch.cuda.is_available(): + torch.cuda.synchronize() + + # Decode + input_len = model_inputs["input_ids"].shape[-1] + result = tokenizer.decode(outputs[0, input_len:], skip_special_tokens=True).strip() + + print(f"โœ… Response received ({len(result)} chars)") + + if not result: + print("โš ๏ธ Model returned empty response") + fallback_text = "The camera pulls back to reveal more of the surrounding environment, as lighting shifts to create a different mood and atmosphere." + result = f"{scene_prefix_text} {fallback_text}" if add_scene_prefix else fallback_text + short_description = result + else: + # Extract a shorter version (first sentence or first 150 chars) + sentences = re.split(r'(?<=[.!?])\s+', result) + short_description = sentences[0] if sentences else result[:150] + + print("-"*80) + print("๐ŸŽฌ NEXT SCENE PROMPT (Full):") + print("-"*80) + print(result) + print("-"*80) + print("๐Ÿ“‹ Short Description:") + print(short_description) + print("="*80 + "\n") + + # Handle the scene prefix based on user preference + if not add_scene_prefix: + # Remove any existing "Next Scene:" or "NEW SCENE:" or custom prefix + result = re.sub(r'^(Next Scene:|NEW SCENE:)\s*', '', result, flags=re.IGNORECASE).strip() + short_description = re.sub(r'^(Next Scene:|NEW SCENE:)\s*', '', short_description, flags=re.IGNORECASE).strip() + else: + # Ensure the custom prefix is used + # First, remove any existing standard prefixes + result = re.sub(r'^(Next Scene:|NEW SCENE:)\s*', '', result, flags=re.IGNORECASE).strip() + short_description = re.sub(r'^(Next Scene:|NEW SCENE:)\s*', '', short_description, flags=re.IGNORECASE).strip() + + # Then add the custom prefix (ensure it has proper spacing) + prefix = scene_prefix_text.strip() + if prefix and not prefix.endswith(' ') and not prefix.endswith(':'): + prefix += ' ' + elif prefix and prefix.endswith(':'): + prefix += ' ' + + result = prefix + result + short_description = prefix + short_description + + # Clear model if not keeping loaded + if not keep_model_loaded: + self.clear_model() + + return (result, short_description) + + except Exception as e: + import traceback + print(f"\nโŒ An error occurred in QwenVLNextScene: {e}") + print(traceback.format_exc()) + print("="*80 + "\n") + + if not keep_model_loaded: + self.clear_model() + + error_message = f"Error generating next scene: {str(e)}" + return (error_message, error_message)