diff --git a/README.md b/README.md index 493ceac..c6e9440 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # FramePack for ComfyUI -20250421 Update: Added support for first/last frame image-to-video generation from TTPlanetPig + +**20250506 Update:** Added support for `FramePack_F1`. +- **Download F1 Workflow (中文)**: [https://www.runninghub.cn/post/1919141028262252546](https://www.runninghub.cn/post/1919141028262252546) +- **Download F1 Workflow (English)**: [https://www.runninghub.ai/post/1919141028262252546](https://www.runninghub.ai/post/1919141028262252546) + +**20250421 Update:** Added support for first/last frame image-to-video generation from TTPlanetPig [TTPlanetPig](https://github.com/TTPlanetPig) https://github.com/lllyasviel/FramePack/pull/167 ## Online Access @@ -53,25 +58,37 @@ This is a simple implementation of https://github.com/lllyasviel/FramePack. If t local_dir_use_symlinks=False ) + # Download FramePackF1_HY model + snapshot_download( + repo_id="lllyasviel/FramePack_F1_I2V_HY_20250503", + local_dir="FramePackF1_HY", + ignore_patterns=["transformer/*", "*.git*", "*.log*", "*.md"], + local_dir_use_symlinks=False + ) + 3. **Manual Download** - HunyuanVideo: [HuggingFace Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo/tree/main) - Flux Redux BFL: [HuggingFace Link](https://huggingface.co/lllyasviel/flux_redux_bfl/tree/main) - FramePackI2V: [HuggingFace Link](https://huggingface.co/lllyasviel/FramePackI2V_HY/tree/main) + - FramePackF1_HY: [HuggingFace Link](https://huggingface.co/lllyasviel/FramePack_F1_I2V_HY_20250503/tree/main) 4. **File Structure After Download** ``` comfyui/models/ flux_redux_bfl ├── feature_extractor - │   └── preprocessor_config.json + │ └── preprocessor_config.json ├── image_embedder - │   ├── config.json - │   └── diffusion_pytorch_model.safetensors + │ ├── config.json + │ └── diffusion_pytorch_model.safetensors ├── image_encoder - │   ├── config.json - │   └── model.safetensors + │ ├── config.json + │ └── model.safetensors ├── model_index.json └── README.md + FramePackF1_HY + ├── config.json # Example structure, actual files might differ + └── diffusion_pytorch_model.safetensors # Example structure FramePackI2V_HY ├── config.json ├── diffusion_pytorch_model-00001-of-00003.safetensors @@ -84,35 +101,80 @@ comfyui/models/ ├── model_index.json ├── README.md ├── scheduler - │   └── scheduler_config.json + │ └── scheduler_config.json ├── text_encoder - │   ├── config.json - │   ├── model-00001-of-00004.safetensors - │   ├── model-00002-of-00004.safetensors - │   ├── model-00003-of-00004.safetensors - │   ├── model-00004-of-00004.safetensors - │   └── model.safetensors.index.json + │ ├── config.json + │ ├── model-00001-of-00004.safetensors + │ ├── model-00002-of-00004.safetensors + │ ├── model-00003-of-00004.safetensors + │ ├── model-00004-of-00004.safetensors + │ └── model.safetensors.index.json ├── text_encoder_2 - │   ├── config.json - │   └── model.safetensors + │ ├── config.json + │ └── model.safetensors ├── tokenizer - │   ├── special_tokens_map.json - │   ├── tokenizer_config.json - │   └── tokenizer.json + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── tokenizer.json ├── tokenizer_2 - │   ├── merges.txt - │   ├── special_tokens_map.json - │   ├── tokenizer_config.json - │   └── vocab.json + │ ├── merges.txt + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── vocab.json └── vae ├── config.json └── diffusion_pytorch_model.safetensors ``` + ## Example: - - -https://github.com/user-attachments/assets/4378bb8c-a8f4-4f16-a835-cde976c6144e - - -![image](https://github.com/user-attachments/assets/ea936caf-c0ca-48f4-af20-64090771d382) - +``` +comfyui/models/ + flux_redux_bfl + ├── feature_extractor + │ └── preprocessor_config.json + ├── image_embedder + │ ├── config.json + │ └── diffusion_pytorch_model.safetensors + ├── image_encoder + │ ├── config.json + │ └── model.safetensors + ├── model_index.json + └── README.md + FramePackF1_HY + ├── config.json # Example structure, actual files might differ + └── diffusion_pytorch_model.safetensors # Example structure + FramePackI2V_HY + ├── config.json + ├── diffusion_pytorch_model-00001-of-00003.safetensors + ├── diffusion_pytorch_model-00002-of-00003.safetensors + ├── diffusion_pytorch_model-00003-of-00003.safetensors + ├── diffusion_pytorch_model.safetensors.index.json + └── README.md + HunyuanVideo + ├── config.json + ├── model_index.json + ├── README.md + ├── scheduler + │ └── scheduler_config.json + ├── text_encoder + │ ├── config.json + │ ├── model-00001-of-00004.safetensors + │ ├── model-00002-of-00004.safetensors + │ ├── model-00003-of-00004.safetensors + │ ├── model-00004-of-00004.safetensors + │ └── model.safetensors.index.json + ├── text_encoder_2 + │ ├── config.json + │ └── model.safetensors + ├── tokenizer + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── tokenizer.json + ├── tokenizer_2 + │ ├── merges.txt + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── vocab.json + └── vae + ├── config.json + └── diffusion_pytorch_model.safetensors \ No newline at end of file diff --git a/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-312.pyc b/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-312.pyc index f9113b9..060fa15 100644 Binary files a/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-312.pyc and b/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-312.pyc differ diff --git a/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-312.pyc b/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-312.pyc index 13ebfc2..d935936 100644 Binary files a/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-312.pyc and b/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-312.pyc differ diff --git a/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-312.pyc b/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-312.pyc index 5d80c94..1c3591d 100644 Binary files a/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-312.pyc and b/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-312.pyc differ diff --git a/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-312.pyc b/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-312.pyc index f965972..421d648 100644 Binary files a/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-312.pyc and b/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-312.pyc differ diff --git a/nodes.py b/nodes.py index b81fe5a..dca52ef 100644 --- a/nodes.py +++ b/nodes.py @@ -10,6 +10,7 @@ import safetensors.torch as sf import numpy as np import argparse import math +import time from PIL import Image from diffusers import AutoencoderKLHunyuanVideo @@ -401,6 +402,431 @@ class Kiki_FramePack: _, _, info = torchvision.io.read_video(video_path, pts_unit='sec') return info['video_fps'] +# --- Start of Kiki_FramePack_F1 Class --- +class Kiki_FramePack_F1: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "ref_image": ("IMAGE", ), + "prompt": ("STRING", {"multiline": True}), + "total_second_length": ("INT", {"default": 5, "min": 1, "max": 120, "step": 1}), + "fps": ("INT", {"default": 30, "min": 1, "max": 60, "step": 1}), + "seed": ("INT", {"default": 3407}), + "steps": ("INT", {"default": 25, "min": 1, "max": 100, "step": 1}), + "gs": ("FLOAT", {"default": 10.0, "min": 1.0, "max": 32.0, "step": 0.1, "round": 0.01, "label": "Distilled CFG Scale"}), + "use_teacache": ("BOOLEAN", {"default": True}), + "upscale": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 2.0, "step": 0.1, "description": "Resolution scaling factor."}), + }, + "optional": { + "n_prompt": ("STRING", {"multiline": True, "default": ""}), + } + } + + RETURN_TYPES = ("IMAGE", "FLOAT") + RETURN_NAMES = ("frames", "fps") + CATEGORY = "Runninghub/FramePack" + FUNCTION = "run_f1" + + TITLE = 'RunningHub FramePack F1' + OUTPUT_NODE = True + + def __init__(self): + self.high_vram = False + self.frames = None + self.fps = None + + hunyuan_root = os.path.join(folder_paths.models_dir, 'HunyuanVideo') + flux_redux_bfl_root = os.path.join(folder_paths.models_dir, 'flux_redux_bfl') + framePackF1_root = os.path.join(folder_paths.models_dir, 'FramePackF1_HY') + + if not os.path.isdir(framePackF1_root): + print(f"Warning: FramePack F1 model directory not found at {framePackF1_root}") + + self.text_encoder = LlamaModel.from_pretrained(hunyuan_root, subfolder='text_encoder', torch_dtype=torch.float16).cpu() + self.text_encoder_2 = CLIPTextModel.from_pretrained(hunyuan_root, subfolder='text_encoder_2', torch_dtype=torch.float16).cpu() + self.tokenizer = LlamaTokenizerFast.from_pretrained(hunyuan_root, subfolder='tokenizer') + self.tokenizer_2 = CLIPTokenizer.from_pretrained(hunyuan_root, subfolder='tokenizer_2') + self.vae = AutoencoderKLHunyuanVideo.from_pretrained(hunyuan_root, subfolder='vae', torch_dtype=torch.float16).cpu() + + self.feature_extractor = SiglipImageProcessor.from_pretrained(flux_redux_bfl_root, subfolder='feature_extractor') + self.image_encoder = SiglipVisionModel.from_pretrained(flux_redux_bfl_root, subfolder='image_encoder', torch_dtype=torch.float16).cpu() + + try: + self.transformer_f1 = HunyuanVideoTransformer3DModelPacked.from_pretrained(framePackF1_root, torch_dtype=torch.bfloat16).cpu() + except Exception as e: + print(f"Error loading FramePack F1 transformer model from {framePackF1_root}: {e}") + print("Please ensure the F1 model weights (e.g., transformer.safetensors) are correctly placed in the directory.") + self.transformer_f1 = None + + self.vae.eval() + self.text_encoder.eval() + self.text_encoder_2.eval() + self.image_encoder.eval() + if self.transformer_f1: + self.transformer_f1.eval() + + if not self.high_vram: + self.vae.enable_slicing() + self.vae.enable_tiling() + + if self.transformer_f1: + self.transformer_f1.high_quality_fp32_output_for_inference = True + print('F1 transformer.high_quality_fp32_output_for_inference = True') + + self.transformer_f1.to(dtype=torch.bfloat16) + + self.transformer_f1.requires_grad_(False) + + if not self.high_vram: + DynamicSwapInstaller.install_model(self.transformer_f1, device=gpu) + + self.vae.to(dtype=torch.float16) + self.image_encoder.to(dtype=torch.float16) + self.text_encoder.to(dtype=torch.float16) + self.text_encoder_2.to(dtype=torch.float16) + self.vae.requires_grad_(False) + self.text_encoder.requires_grad_(False) + self.text_encoder_2.requires_grad_(False) + self.image_encoder.requires_grad_(False) + + if not self.high_vram: + DynamicSwapInstaller.install_model(self.text_encoder, device=gpu) + + def strict_align(self, h, w, scale): + raw_h = h * scale + raw_w = w * scale + aligned_h = int(round(raw_h / 64)) * 64 + aligned_w = int(round(raw_w / 64)) * 64 + assert (aligned_h % 64 == 0) and (aligned_w % 64 == 0), "尺寸必须是64的倍数" + assert (aligned_h//8) % 8 == 0 and (aligned_w//8) % 8 == 0, "潜在空间需要8的倍数" + return aligned_h, aligned_w + + def preprocess_image(self, image): + if image is None: return None + if image.dim() == 4 and image.shape[0] == 1: + img_tensor = image[0] + else: + img_tensor = image + print(f"Warning: Unexpected input image tensor shape: {image.shape}. Assuming HWC.") + + image_np = 255. * img_tensor.cpu().numpy() + image = Image.fromarray(np.clip(image_np, 0, 255).astype(np.uint8)).convert("RGB") + input_image = np.array(image) + return input_image + + def run_f1(self, **kwargs): + if not self.transformer_f1: + print("Error: Kiki_FramePack_F1 cannot run because the transformer model failed to load.") + return (torch.empty((0, 1, 1, 3), dtype=torch.float32), 0.0) + + try: + image = kwargs['ref_image'] + image_np = self.preprocess_image(image) + prompt = kwargs['prompt'] + n_prompt = kwargs.get('n_prompt', "") + seed = kwargs['seed'] + total_second_length = kwargs['total_second_length'] + fps = kwargs['fps'] + steps = kwargs['steps'] + gs = kwargs['gs'] + use_teacache = kwargs['use_teacache'] + upscale = kwargs['upscale'] + cfg = 1.0 + rs = 0.0 + latent_window_size = 9 + + random_str = ''.join(random.choices(string.ascii_letters + string.digits, k=16)) + video_path = os.path.join(folder_paths.get_output_directory(), f'{random_str}_f1.mp4') + + # --- Initialize Progress Bar (Aligned with demo's section calc) --- + # Use demo's calculation for total_latent_sections, assuming 30fps basis for consistency + total_latent_sections = int(max(round((total_second_length * 30) / (latent_window_size * 4)), 1)) + total_progress_steps = total_latent_sections * steps + self.pbar = comfy.utils.ProgressBar(total_progress_steps) + + # Call exec_f1, passing latent_window_size as well + self.exec_f1(input_image=image_np, prompt=prompt, n_prompt=n_prompt, seed=seed, + total_second_length=total_second_length, video_path=video_path, fps=fps, + steps=steps, gs=gs, cfg=cfg, rs=rs, latent_window_size=latent_window_size, # Pass latent_window_size + use_teacache=use_teacache, scale=upscale, + gpu_memory_preservation=6) + + if os.path.exists(video_path): + self.fps = float(fps) + self.frames = self.extract_frames_to_tensor(video_path) + print(f'F1 Video saved: {video_path} | FPS: {self.fps} | Frames: {self.frames.shape[0] if self.frames is not None else 0}') + else: + self.frames = torch.empty((0, 1, 1, 3), dtype=torch.float32) + self.fps = 0.0 + print(f'F1 Video generation failed or file not found: {video_path}') + + except Exception as e: + print(f"Error in run_f1: {str(e)}") + traceback.print_exc() + self.frames = torch.empty((0, 1, 1, 3), dtype=torch.float32) + self.fps = 0.0 + + return (self.frames, self.fps) + + @torch.no_grad() + def exec_f1(self, input_image, video_path, + prompt, n_prompt, seed, total_second_length, fps, + steps, gs, cfg, rs, latent_window_size, # Receive latent_window_size + use_teacache, scale, + gpu_memory_preservation=6): + + print("--- Starting Kiki_FramePack_F1 exec_f1 (Aligned with Demo Logic) ---") + print(f"Params: seed={seed}, length={total_second_length}s@{fps}fps, steps={steps}, gs={gs}, cfg={cfg}, rs={rs}, lws={latent_window_size}") + + vae_time_stride = 4 + + # --- Use Demo's total_latent_sections calculation --- + total_latent_sections = int(max(round((total_second_length * 30) / (latent_window_size * 4)), 1)) + print(f"Total generation sections (Demo calc): {total_latent_sections}") + + # --- Calculate target frames needed (still useful for trimming) --- + target_pixel_frames = int(round(total_second_length * fps)) + + try: + # --- 1. Initialization & Setup --- + torch.manual_seed(seed) + rnd = torch.Generator("cpu").manual_seed(seed) + + # ... (Unload models if needed) ... + + # --- 2. Encoding Inputs --- + print('Encoding text prompts...') + if not self.high_vram: + fake_diffusers_current_device(self.text_encoder, gpu) + load_model_as_complete(self.text_encoder_2, target_device=gpu) + llama_vec, clip_l_pooler = encode_prompt_conds(prompt, self.text_encoder, self.text_encoder_2, self.tokenizer, self.tokenizer_2) + llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, self.text_encoder, self.text_encoder_2, self.tokenizer, self.tokenizer_2) + llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512) + llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512) + + print('Processing reference image...') + H, W, C = input_image.shape + if scale == 1.0: + height, width = find_nearest_bucket(H, W, resolution=640) + height, width = self.strict_align(height, width, 1.0) + else: + height, width = self.strict_align(H, W, scale) + print(f"Target dimensions: {width}x{height}") + input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height) + input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1 + input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None] + + print('VAE encoding reference image...') + if not self.high_vram: load_model_as_complete(self.vae, target_device=gpu) + start_latent = vae_encode(input_image_pt.to(self.vae.device, dtype=self.vae.dtype), self.vae) + print(f"Start latent shape: {start_latent.shape}") + + print('CLIP Vision encoding reference image...') + if not self.high_vram: load_model_as_complete(self.image_encoder, target_device=gpu) + image_encoder_output = hf_clip_vision_encode(input_image_np, self.feature_extractor, self.image_encoder.to(gpu)) + image_embeddings = image_encoder_output.last_hidden_state + + transformer_dtype = self.transformer_f1.dtype + start_latent = start_latent.to(transformer_dtype).cpu() + + # --- 3. Diffusion Loop (Aligned with Demo) --- + print(f'Starting diffusion loop for {total_latent_sections} sections...') + + latent_channels = start_latent.shape[1] + latent_height = start_latent.shape[-2] + latent_width = start_latent.shape[-1] + history_context_size = 16 + 2 + 1 + + # --- Initialize history_latents like demo --- + # Start with zeros matching context size + history_latents = torch.zeros(size=(1, latent_channels, history_context_size, latent_height, latent_width), dtype=torch.float32).cpu() # Use float32 like demo? + # Immediately add start_latent + history_latents = torch.cat([history_latents, start_latent.to(history_latents.dtype)], dim=2) + total_generated_latent_frames = 1 # Account for start_latent + history_pixels = None + + # ... (Progress bar callback setup) ... + current_section_step = 0 + total_progress_steps = total_latent_sections * steps + def callback_f1(d): + # ... (Update pbar logic remains the same) ... + nonlocal current_section_step + step_in_section = d['i'] + current_total_step = current_section_step * steps + step_in_section + 1 + if hasattr(self, 'pbar') and self.pbar: + self.pbar.update_absolute(current_total_step, total_progress_steps) + + # Calculate frames generated per step based on demo + frames_per_latent_window = latent_window_size * 4 - 3 + + for section_index in range(total_latent_sections): + section_start_time = time.time() + print(f'Generating section {section_index + 1} / {total_latent_sections}') + current_section_step = section_index + + # ... (Load transformer if needed) ... + + # --- Prepare context and indices (same as before, uses history_latents) --- + indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0) + clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1) + clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1) + + # Get history context from the *end* of the current history_latents + # No padding needed here because history starts with context + start_latent + history_context = history_latents[:, :, -history_context_size:, :, :] + clean_latents_4x, clean_latents_2x, clean_latents_1x = history_context.split([16, 2, 1], dim=2) + clean_latents = torch.cat([start_latent.cpu(), clean_latents_1x.cpu()], dim=2) + + # --- Prepare sample_kwargs (same as before) --- + sample_kwargs = dict( + transformer=self.transformer_f1, + sampler='unipc', + width=width, + height=height, + frames=frames_per_latent_window, # Use demo's frame count + real_guidance_scale=cfg, + distilled_guidance_scale=gs, + guidance_rescale=rs, + num_inference_steps=steps, + generator=rnd, + # --- Add missing positive prompt embeddings & ENSURE DTYPE --- + prompt_embeds=llama_vec.to(gpu, dtype=transformer_dtype), + prompt_embeds_mask=llama_attention_mask.to(gpu), # Mask dtype usually okay + # --- Existing embeddings/poolers & ENSURE DTYPE --- + prompt_poolers=clip_l_pooler.to(gpu, dtype=transformer_dtype), + negative_prompt_embeds=llama_vec_n.to(gpu, dtype=transformer_dtype), + negative_prompt_embeds_mask=llama_attention_mask_n.to(gpu), # Mask dtype usually okay + negative_prompt_poolers=clip_l_pooler_n.to(gpu, dtype=transformer_dtype), + device=gpu, # Device is already GPU + dtype=transformer_dtype, # Explicitly passing transformer's dtype + image_embeddings=image_embeddings.to(gpu, dtype=transformer_dtype), + latent_indices=latent_indices.to(gpu), # Indices dtype usually okay + clean_latents=clean_latents.to(gpu, dtype=transformer_dtype), # Ensure correct dtype + clean_latent_indices=clean_latent_indices.to(gpu), # Indices dtype usually okay + clean_latents_2x=clean_latents_2x.to(gpu, dtype=transformer_dtype), # Ensure correct dtype + clean_latent_2x_indices=clean_latent_2x_indices.to(gpu), # Indices dtype usually okay + clean_latents_4x=clean_latents_4x.to(gpu, dtype=transformer_dtype), # Ensure correct dtype + clean_latent_4x_indices=clean_latent_4x_indices.to(gpu), # Indices dtype usually okay + callback=callback_f1, + ) + + # ... (Initialize teacache) ... + if hasattr(self.transformer_f1, 'initialize_teacache'): + self.transformer_f1.initialize_teacache(enable_teacache=use_teacache, num_steps=steps) + + # --- Call sample_hunyuan --- + generated_latents = sample_hunyuan(**sample_kwargs) + + generated_latents = generated_latents.to(cpu, dtype=torch.float32) + print(f" Sampled latent section shape: {generated_latents.shape}") + + # --- Update history_latents (Aligned with Demo: Always append) --- + total_generated_latent_frames += int(generated_latents.shape[2]) + history_latents = torch.cat([history_latents, generated_latents.to(history_latents.dtype)], dim=2) + + # --- Decode and append pixels (Aligned with Demo) --- + if not self.high_vram: + offload_model_from_device_for_memory_preservation(self.transformer_f1, target_device=gpu, preserved_memory_gb=8) + load_model_as_complete(self.vae, target_device=gpu) + else: + if self.vae.device != gpu: self.vae.to(gpu) + + # Calculate the slice of history to decode based on total generated frames + real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :] # Use actual generated frames + + if history_pixels is None: + # First time: decode the current relevant history + history_pixels = vae_decode(real_history_latents.to(gpu, dtype=self.vae.dtype), self.vae).cpu() + print(f" Decoded initial pixels. Shape: {history_pixels.shape}") + else: + # Subsequent times: decode only the part needed for smooth append + section_latent_frames = latent_window_size * 2 + overlapped_frames = latent_window_size * 4 - 3 # Use demo's overlap calculation + + # Decode the relevant tail end of the history latents + current_latents_to_decode = real_history_latents[:, :, -section_latent_frames:, :, :] + current_pixels = vae_decode(current_latents_to_decode.to(gpu, dtype=self.vae.dtype), self.vae).cpu() + + # Append smoothly using demo's overlap value + history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames) + print(f" Appended pixels. New history shape: {history_pixels.shape}") + + # ... (Unload VAE if needed) ... + if not self.high_vram: + unload_complete_models(self.vae) + + section_end_time = time.time() + print(f" Section {section_index + 1} took {section_end_time - section_start_time:.2f} seconds.") + + # --- 4. Final Saving (Aligned with Demo, keeping variable fps) --- + print('Saving final video...') + if history_pixels is None or history_pixels.shape[2] == 0: + raise ValueError("No pixel frames were generated or decoded.") + + if history_pixels.shape[2] > target_pixel_frames: + print(f"Trimming final video from {history_pixels.shape[2]} to {target_pixel_frames} frames.") + history_pixels = history_pixels[:,:,:target_pixel_frames,:,:] + + save_bcthw_as_mp4( + history_pixels, + video_path, + fps=fps, # Keep user FPS for now + # crf=18 # Omit crf until utils.py is confirmed synced + ) + print(f"Final video saved to: {video_path}") + + except Exception as e: + print(f"Error during Kiki_FramePack_F1 execution: {str(e)}") + traceback.print_exc() + if os.path.exists(video_path): + try: os.remove(video_path) + except OSError: pass + if hasattr(self, 'pbar') and self.pbar: self.pbar.update_absolute(total_progress_steps, total_progress_steps) + raise + + finally: + print('Cleaning up models...') + unload_complete_models( + self.text_encoder, self.text_encoder_2, self.image_encoder, self.vae, self.transformer_f1 + ) + torch.cuda.empty_cache() + print("--- Finished Kiki_FramePack_F1 exec_f1 (Aligned with Demo Logic) ---") + + def extract_frames_to_tensor(self, video_path): + try: + video_tensor, _, metadata = torchvision.io.read_video(video_path, pts_unit='sec', output_format='TCHW') + + video_tensor = video_tensor.permute(0, 2, 3, 1) + + video_tensor = video_tensor.float() / 255.0 + + print(f"Extracted video tensor shape: {video_tensor.shape}") + return video_tensor + + except Exception as e: + print(f"Error extracting frames using torchvision.io.read_video: {e}") + traceback.print_exc() + return torch.empty((0, 1, 1, 3), dtype=torch.float32) + + def get_fps_with_torchvision(self, video_path): + try: + _, _, metadata = torchvision.io.read_video(video_path, pts_unit='sec') + fps = metadata.get('video_fps', 30.0) + return float(fps) + except Exception as e: + print(f"Error reading FPS using torchvision.io.read_video: {e}") + traceback.print_exc() + return 30.0 + +# NODE CLASS MAPPINGS NODE_CLASS_MAPPINGS = { "RunningHub_FramePack": Kiki_FramePack, + "RunningHub_FramePack_F1": Kiki_FramePack_F1 +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "RunningHub_FramePack": Kiki_FramePack.TITLE, + "RunningHub_FramePack_F1": Kiki_FramePack_F1.TITLE } diff --git a/requirements.txt b/requirements.txt index 60eb54c..52725ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,12 @@ -accelerate>=1.6.0 +torch +torchvision +numpy +Pillow diffusers>=0.33.1 transformers>=4.46.2 -scipy>=1.12.0 -torchsde>=0.2.6 einops safetensors +accelerate>=1.6.0 +scipy>=1.12.0 +torchsde>=0.2.6 +opencv-python