From 3f94acd14046ce655e94a38e6274354121a65252 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 May 2025 02:13:54 +0800 Subject: [PATCH] support FramePack_F1 --- README.md | 120 +++-- .../__pycache__/uni_pc_fm.cpython-312.pyc | Bin 5920 -> 5920 bytes .../__pycache__/wrapper.cpython-312.pyc | Bin 2742 -> 2742 bytes .../hunyuan_video_packed.cpython-312.pyc | Bin 51609 -> 51609 bytes .../k_diffusion_hunyuan.cpython-312.pyc | Bin 4675 -> 4675 bytes nodes.py | 426 ++++++++++++++++++ requirements.txt | 11 +- 7 files changed, 525 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 493ceac..c6e9440 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # FramePack for ComfyUI -20250421 Update: Added support for first/last frame image-to-video generation from TTPlanetPig + +**20250506 Update:** Added support for `FramePack_F1`. +- **Download F1 Workflow (中文)**: [https://www.runninghub.cn/post/1919141028262252546](https://www.runninghub.cn/post/1919141028262252546) +- **Download F1 Workflow (English)**: [https://www.runninghub.ai/post/1919141028262252546](https://www.runninghub.ai/post/1919141028262252546) + +**20250421 Update:** Added support for first/last frame image-to-video generation from TTPlanetPig [TTPlanetPig](https://github.com/TTPlanetPig) https://github.com/lllyasviel/FramePack/pull/167 ## Online Access @@ -53,25 +58,37 @@ This is a simple implementation of https://github.com/lllyasviel/FramePack. If t local_dir_use_symlinks=False ) + # Download FramePackF1_HY model + snapshot_download( + repo_id="lllyasviel/FramePack_F1_I2V_HY_20250503", + local_dir="FramePackF1_HY", + ignore_patterns=["transformer/*", "*.git*", "*.log*", "*.md"], + local_dir_use_symlinks=False + ) + 3. **Manual Download** - HunyuanVideo: [HuggingFace Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo/tree/main) - Flux Redux BFL: [HuggingFace Link](https://huggingface.co/lllyasviel/flux_redux_bfl/tree/main) - FramePackI2V: [HuggingFace Link](https://huggingface.co/lllyasviel/FramePackI2V_HY/tree/main) + - FramePackF1_HY: [HuggingFace Link](https://huggingface.co/lllyasviel/FramePack_F1_I2V_HY_20250503/tree/main) 4. **File Structure After Download** ``` comfyui/models/ flux_redux_bfl ├── feature_extractor - │   └── preprocessor_config.json + │ └── preprocessor_config.json ├── image_embedder - │   ├── config.json - │   └── diffusion_pytorch_model.safetensors + │ ├── config.json + │ └── diffusion_pytorch_model.safetensors ├── image_encoder - │   ├── config.json - │   └── model.safetensors + │ ├── config.json + │ └── model.safetensors ├── model_index.json └── README.md + FramePackF1_HY + ├── config.json # Example structure, actual files might differ + └── diffusion_pytorch_model.safetensors # Example structure FramePackI2V_HY ├── config.json ├── diffusion_pytorch_model-00001-of-00003.safetensors @@ -84,35 +101,80 @@ comfyui/models/ ├── model_index.json ├── README.md ├── scheduler - │   └── scheduler_config.json + │ └── scheduler_config.json ├── text_encoder - │   ├── config.json - │   ├── model-00001-of-00004.safetensors - │   ├── model-00002-of-00004.safetensors - │   ├── model-00003-of-00004.safetensors - │   ├── model-00004-of-00004.safetensors - │   └── model.safetensors.index.json + │ ├── config.json + │ ├── model-00001-of-00004.safetensors + │ ├── model-00002-of-00004.safetensors + │ ├── model-00003-of-00004.safetensors + │ ├── model-00004-of-00004.safetensors + │ └── model.safetensors.index.json ├── text_encoder_2 - │   ├── config.json - │   └── model.safetensors + │ ├── config.json + │ └── model.safetensors ├── tokenizer - │   ├── special_tokens_map.json - │   ├── tokenizer_config.json - │   └── tokenizer.json + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── tokenizer.json ├── tokenizer_2 - │   ├── merges.txt - │   ├── special_tokens_map.json - │   ├── tokenizer_config.json - │   └── vocab.json + │ ├── merges.txt + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── vocab.json └── vae ├── config.json └── diffusion_pytorch_model.safetensors ``` + ## Example: - - -https://github.com/user-attachments/assets/4378bb8c-a8f4-4f16-a835-cde976c6144e - - -![image](https://github.com/user-attachments/assets/ea936caf-c0ca-48f4-af20-64090771d382) - +``` +comfyui/models/ + flux_redux_bfl + ├── feature_extractor + │ └── preprocessor_config.json + ├── image_embedder + │ ├── config.json + │ └── diffusion_pytorch_model.safetensors + ├── image_encoder + │ ├── config.json + │ └── model.safetensors + ├── model_index.json + └── README.md + FramePackF1_HY + ├── config.json # Example structure, actual files might differ + └── diffusion_pytorch_model.safetensors # Example structure + FramePackI2V_HY + ├── config.json + ├── diffusion_pytorch_model-00001-of-00003.safetensors + ├── diffusion_pytorch_model-00002-of-00003.safetensors + ├── diffusion_pytorch_model-00003-of-00003.safetensors + ├── diffusion_pytorch_model.safetensors.index.json + └── README.md + HunyuanVideo + ├── config.json + ├── model_index.json + ├── README.md + ├── scheduler + │ └── scheduler_config.json + ├── text_encoder + │ ├── config.json + │ ├── model-00001-of-00004.safetensors + │ ├── model-00002-of-00004.safetensors + │ ├── model-00003-of-00004.safetensors + │ ├── model-00004-of-00004.safetensors + │ └── model.safetensors.index.json + ├── text_encoder_2 + │ ├── config.json + │ └── model.safetensors + ├── tokenizer + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── tokenizer.json + ├── tokenizer_2 + │ ├── merges.txt + │ ├── special_tokens_map.json + │ ├── tokenizer_config.json + │ └── vocab.json + └── vae + ├── config.json + └── diffusion_pytorch_model.safetensors \ No newline at end of file diff --git a/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-312.pyc b/diffusers_helper/k_diffusion/__pycache__/uni_pc_fm.cpython-312.pyc index f9113b967e75e9c8f299841bdd3607aff201b687..060fa15efad05133dc881ac7f6d1b8ab1a644be8 100644 GIT binary patch delta 20 acmZ3Ww?L2kG%qg~0}!Nmvu@-T6$bz`bp$g2 delta 20 acmZ3Ww?L2kG%qg~0}$9~GH&D+6$bz^eFNnH diff --git a/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-312.pyc b/diffusers_helper/k_diffusion/__pycache__/wrapper.cpython-312.pyc index 13ebfc28dafe63ea78bca601d27677fa75cfeac5..d935936110c63a1caff9eb7a9990f27693b1dc0d 100644 GIT binary patch delta 20 acmdlcx=ob(G%qg~0}!Nmvu@;G%>@88R0OsF delta 20 acmdlcx=ob(G%qg~0}$9~GH&Ev%>@86Tm)zU diff --git a/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-312.pyc b/diffusers_helper/models/__pycache__/hunyuan_video_packed.cpython-312.pyc index 5d80c94f4517c990d95dc51b8eec4190e2f36b61..1c3591da558ccf5e7a5b4f7f1b364c437d85d675 100644 GIT binary patch delta 22 ccmbO^nR(`9X71Cxyj%=GkmAj{k-P6C07p0mAOHXW delta 22 ccmbO^nR(`9X71Cxyj%=GV57;nk-P6C07XCr(f|Me diff --git a/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-312.pyc b/diffusers_helper/pipelines/__pycache__/k_diffusion_hunyuan.cpython-312.pyc index f9659721b7a5b74cfe370701348d315fe103a6c8..421d6489abf4fde2c96c961ced9aa5649cfcfc10 100644 GIT binary patch delta 20 acmX@Ca#)4?G%qg~0}!Nmvu@ target_pixel_frames: + print(f"Trimming final video from {history_pixels.shape[2]} to {target_pixel_frames} frames.") + history_pixels = history_pixels[:,:,:target_pixel_frames,:,:] + + save_bcthw_as_mp4( + history_pixels, + video_path, + fps=fps, # Keep user FPS for now + # crf=18 # Omit crf until utils.py is confirmed synced + ) + print(f"Final video saved to: {video_path}") + + except Exception as e: + print(f"Error during Kiki_FramePack_F1 execution: {str(e)}") + traceback.print_exc() + if os.path.exists(video_path): + try: os.remove(video_path) + except OSError: pass + if hasattr(self, 'pbar') and self.pbar: self.pbar.update_absolute(total_progress_steps, total_progress_steps) + raise + + finally: + print('Cleaning up models...') + unload_complete_models( + self.text_encoder, self.text_encoder_2, self.image_encoder, self.vae, self.transformer_f1 + ) + torch.cuda.empty_cache() + print("--- Finished Kiki_FramePack_F1 exec_f1 (Aligned with Demo Logic) ---") + + def extract_frames_to_tensor(self, video_path): + try: + video_tensor, _, metadata = torchvision.io.read_video(video_path, pts_unit='sec', output_format='TCHW') + + video_tensor = video_tensor.permute(0, 2, 3, 1) + + video_tensor = video_tensor.float() / 255.0 + + print(f"Extracted video tensor shape: {video_tensor.shape}") + return video_tensor + + except Exception as e: + print(f"Error extracting frames using torchvision.io.read_video: {e}") + traceback.print_exc() + return torch.empty((0, 1, 1, 3), dtype=torch.float32) + + def get_fps_with_torchvision(self, video_path): + try: + _, _, metadata = torchvision.io.read_video(video_path, pts_unit='sec') + fps = metadata.get('video_fps', 30.0) + return float(fps) + except Exception as e: + print(f"Error reading FPS using torchvision.io.read_video: {e}") + traceback.print_exc() + return 30.0 + +# NODE CLASS MAPPINGS NODE_CLASS_MAPPINGS = { "RunningHub_FramePack": Kiki_FramePack, + "RunningHub_FramePack_F1": Kiki_FramePack_F1 +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "RunningHub_FramePack": Kiki_FramePack.TITLE, + "RunningHub_FramePack_F1": Kiki_FramePack_F1.TITLE } diff --git a/requirements.txt b/requirements.txt index 60eb54c..52725ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,12 @@ -accelerate>=1.6.0 +torch +torchvision +numpy +Pillow diffusers>=0.33.1 transformers>=4.46.2 -scipy>=1.12.0 -torchsde>=0.2.6 einops safetensors +accelerate>=1.6.0 +scipy>=1.12.0 +torchsde>=0.2.6 +opencv-python