From 1a494b91889dbbb1fa24d6e59433e594cb6ec8f3 Mon Sep 17 00:00:00 2001 From: Bubbliiiing <47347516+bubbliiiing@users.noreply.github.com> Date: Tue, 5 Aug 2025 19:23:02 +0800 Subject: [PATCH] Comyfui Update && Fix bug in api && Dataload (#276) Update Wan2.2 Comfyui, fix bug in api, fix bug in dataloader, fix bug in readme, fix bug in multigpu --- README.md | 7 +- README_ja-JP.md | 7 +- README_zh-CN.md | 6 +- comfyui/README.md | 14 +- comfyui/comfyui_nodes.py | 12 + comfyui/wan2_2/nodes.py | 661 +++++++++++++++++++++ comfyui/wan2_2/v1/wan2.2_workflow_i2v.json | 473 +++++++++++++++ comfyui/wan2_2/v1/wan2.2_workflow_t2v.json | 383 ++++++++++++ requirements.txt | 1 + videox_fun/api/api.py | 4 +- videox_fun/api/api_multi_nodes.py | 4 +- videox_fun/data/dataset_image_video.py | 12 +- videox_fun/ui/cogvideox_fun_ui.py | 2 + videox_fun/ui/wan2_2_ui.py | 1 + videox_fun/ui/wan_fun_ui.py | 2 + videox_fun/ui/wan_ui.py | 2 + 16 files changed, 1570 insertions(+), 21 deletions(-) create mode 100755 comfyui/wan2_2/nodes.py create mode 100644 comfyui/wan2_2/v1/wan2.2_workflow_i2v.json create mode 100644 comfyui/wan2_2/v1/wan2.2_workflow_t2v.json diff --git a/README.md b/README.md index 5720ee5..3ae49f2 100755 --- a/README.md +++ b/README.md @@ -549,10 +549,9 @@ CogVideoX-Fun can be found in [Readme Train](scripts/cogvideox_fun/README_TRAIN. | Name | Hugging Face | Model Scope | Description | |--|--|--|--| -| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | Wan2.1-1.3B Text-to-Video Weights | -| Wan2.2-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | Wan2.1-14B Text-to-Video Weights | -| Wan2.2-I2V-14B-480P | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | Wan2.1-14B-480P Image-to-Video Weights | - +| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | Wan2.2-5B Text-to-Video Weights | +| Wan2.2-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | Wan2.2-14B Text-to-Video Weights | +| Wan2.2-I2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | Wan2.2-I2V-A14B Image-to-Video Weights | ## 2. Wan2.1-Fun diff --git a/README_ja-JP.md b/README_ja-JP.md index 2dca49a..faf5c95 100755 --- a/README_ja-JP.md +++ b/README_ja-JP.md @@ -548,10 +548,9 @@ CogVideoX-Funは[Readme Train](scripts/cogvideox_fun/README_TRAIN.md)と[Readme | モデル名 | Hugging Face | Model Scope | 説明 | |--|--|--|--| -| Wan2.2-TI2V-5B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | 万象2.1-1.3B テキストから動画生成重み | -| Wan2.2-T2V-14B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | 万象2.1-14B テキストから動画生成重み | -| Wan2.2-I2V-14B-480P | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | 万象2.1-14B-480P 画像から動画生成重み | - +| Wan2.2-TI2V-5B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | 万象2.2-5B テキストから動画生成重み | +| Wan2.2-T2V-A14B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | 万象2.2-14B テキストから動画生成重み | +| Wan2.2-I2V-A14B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | 万象2.2-14B 画像から動画生成重み | ## 2. Wan2.1-Fun diff --git a/README_zh-CN.md b/README_zh-CN.md index 7e59955..93e4992 100755 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -538,9 +538,9 @@ CogVideoX-Fun可以查看[Readme Train](scripts/cogvideox_fun/README_TRAIN.md) | 名称 | Hugging Face | Model Scope | 描述 | |--|--|--|--| -| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | 万象2.1-1.3B文生视频权重 | -| Wan2.2-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | 万象2.1-14B文生视频权重 | -| Wan2.2-I2V-14B-480P | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | 万象2.1-14B-480P图生视频权重 | +| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | 万象2.2-5B文生视频权重 | +| Wan2.2-T2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | 万象2.2-14B文生视频权重 | +| Wan2.2-I2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | 万象2.2-14B图生视频权重 | ## 2. Wan2.1-Fun diff --git a/comfyui/README.md b/comfyui/README.md index 08c754d..9b27fdb 100755 --- a/comfyui/README.md +++ b/comfyui/README.md @@ -40,7 +40,15 @@ remote_onnx_det = "https://huggingface.co/yzd-v/DWPose/resolve/main/yolox_l.onnx remote_onnx_pose = "https://huggingface.co/yzd-v/DWPose/resolve/main/dw-ll_ucoco_384.onnx" remote_zoe= "https://huggingface.co/lllyasviel/Annotators/resolve/main/ZoeD_M12_N.pt" ``` -#### i. Wan2.1-Fun +#### i. Wan2.2 + +| Name | Hugging Face | Model Scope | Description | +|--|--|--|--| +| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | Wan2.2-5B Text-to-Video Weights | +| Wan2.2-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | Wan2.2-14B Text-to-Video Weights | +| Wan2.2-I2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | Wan2.2-I2V-A14B Image-to-Video Weights | + +#### ii. Wan2.1-Fun V1.1: | Name | Storage Size | Hugging Face | Model Scope | Description | @@ -60,7 +68,7 @@ V1.0: | Wan2.1-Fun-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control) | Wan2.1-Fun-1.3B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | | Wan2.1-Fun-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control) | Wan2.1-Fun-14B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | -#### ii. Wan2.1 +#### iii. Wan2.1 | Name | Hugging Face | Model Scope | Description | |--|--|--|--| @@ -69,7 +77,7 @@ V1.0: | Wan2.1-I2V-14B-480P | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P) | Wanxiang 2.1-14B-480P image-to-video weights | | Wan2.1-I2V-14B-720P| [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | Wanxiang 2.1-14B-720P image-to-video weights | -#### iii. CogVideoX-Fun +#### iv. CogVideoX-Fun V1.5: diff --git a/comfyui/comfyui_nodes.py b/comfyui/comfyui_nodes.py index c91e0d6..0a772ae 100755 --- a/comfyui/comfyui_nodes.py +++ b/comfyui/comfyui_nodes.py @@ -16,6 +16,8 @@ from .wan2_1.nodes import (LoadWanLora, LoadWanModel, WanI2VSampler, from .wan2_1_fun.nodes import (LoadWanFunLora, LoadWanFunModel, WanFunInpaintSampler, WanFunT2VSampler, WanFunV2VSampler) +from .wan2_2.nodes import (LoadWan2_2Lora, LoadWan2_2Model, Wan2_2I2VSampler, + Wan2_2T2VSampler) class FunTextBox: @classmethod @@ -319,6 +321,11 @@ NODE_CLASS_MAPPINGS = { "WanFunInpaintSampler": WanFunInpaintSampler, "WanFunV2VSampler": WanFunV2VSampler, + "LoadWan2_2Model": LoadWan2_2Model, + "LoadWan2_2Lora": LoadWan2_2Lora, + "Wan2_2T2VSampler": Wan2_2T2VSampler, + "Wan2_2I2VSampler": Wan2_2I2VSampler, + "VideoToCanny": VideoToCanny, "VideoToDepth": VideoToDepth, "VideoToOpenpose": VideoToPose, @@ -354,6 +361,11 @@ NODE_DISPLAY_NAME_MAPPINGS = { "WanFunInpaintSampler": "Wan Fun Sampler for Image to Video", "WanFunV2VSampler": "Wan Fun Sampler for Video to Video", + "LoadWan2_2Model": "Load Wan 2.2 Model", + "LoadWan2_2Lora": "Load Wan 2.2 Lora", + "Wan2_2T2VSampler": "Wan 2.2 Sampler for Text to Video", + "Wan2_2I2VSampler": "Wan 2.2 Sampler for Image to Video", + "VideoToCanny": "Video To Canny", "VideoToDepth": "Video To Depth", "VideoToOpenpose": "Video To Pose", diff --git a/comfyui/wan2_2/nodes.py b/comfyui/wan2_2/nodes.py new file mode 100755 index 0000000..70b3817 --- /dev/null +++ b/comfyui/wan2_2/nodes.py @@ -0,0 +1,661 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import copy +import gc +import json +import os + +import comfy.model_management as mm +import cv2 +import folder_paths +import numpy as np +import torch +from comfy.utils import ProgressBar, load_torch_file +from diffusers import FlowMatchEulerDiscreteScheduler +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from ...videox_fun.pipeline import Wan2_2I2VPipeline, Wan2_2Pipeline +from ...videox_fun.ui.controller import all_cheduler_dict +from ...videox_fun.utils.fp8_optimization import ( + convert_model_weight_to_float8, convert_weight_dtype_wrapper, replace_parameters_by_name) +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import (get_image_to_video_latent, filter_kwargs, + get_video_to_video_latent, + save_videos_grid) +from ...videox_fun.models.cache_utils import get_teacache_coefficients +from ..comfyui_utils import eas_cache_dir, script_directory, to_pil + +# Used in lora cache +transformer_cpu_cache = {} +transformer_high_cpu_cache = {} +# lora path before +lora_path_before = "" +lora_high_path_before = "" + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +class LoadWan2_2Model: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'Wan2.2-T2V-A14B', + 'Wan2.2-I2V-A14B', + ], + { + "default": 'Wan2.2-T2V-A14B', + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "config": ( + [ + "wan2.2/wan_civitai_t2v.yaml", + "wan2.2/wan_civitai_i2v.yaml", + ], + { + "default": "wan2.2/wan_civitai_t2v.yaml", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'fp16' + } + ), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model, precision, config): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + # Init processbar + pbar = ProgressBar(5) + + # Load config + config_path = f"{script_directory}/config/{config}" + config = OmegaConf.load(config_path) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = None + + # Check if the model exists in any of the possible folders within folder_paths.models_dir + for folder in possible_folders: + candidate_path = os.path.join(folder_paths.models_dir, folder, model) + if os.path.exists(candidate_path): + model_name = candidate_path + break + + # If model_name is still None, check eas_cache_dir for each possible folder + if model_name is None and os.path.exists(eas_cache_dir): + for folder in possible_folders: + candidate_path = os.path.join(eas_cache_dir, folder, model) + if os.path.exists(candidate_path): + model_name = candidate_path + break + + # If model_name is still None, prompt the user to download the model + if model_name is None: + print(f"Please download cogvideoxfun model to one of the following directories:") + for folder in possible_folders: + print(f"- {os.path.join(folder_paths.models_dir, folder)}") + if os.path.exists(eas_cache_dir): + print(f"- {os.path.join(eas_cache_dir, folder)}") + raise ValueError("Please download Fun model") + + vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + pbar.update(1) + + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + pbar.update(1) + + # Get pipeline + model_type = "Inpaint" + if model_type == "Inpaint": + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = Wan2_2I2VPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + ) + else: + raise ValueError(f"Model type {model_type} not supported") + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device=device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [], + 'config': config, + } + return (funmodels,) + +class LoadWan2_2Lora: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ("FunModels",), + "lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "lora_high_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "lora_cache":([False, True], {"default": False,}), + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "load_lora" + CATEGORY = "CogVideoXFUNWrapper" + + def load_lora(self, funmodels, lora_name, lora_high_name, strength_model, lora_cache): + if lora_name is not None: + funmodels['lora_cache'] = lora_cache + funmodels['loras'] = funmodels.get("loras", []) + [folder_paths.get_full_path("loras", lora_name)] + funmodels['loras_high'] = funmodels.get("loras_high", []) + [folder_paths.get_full_path("loras", lora_high_name)] + funmodels['strength_model'] = funmodels.get("strength_model", []) + [strength_model] + return (funmodels,) + else: + return (funmodels,) + +class Wan2_2T2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "width": ( + "INT", {"default": 832, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 480, "min": 64, "max": 2048, "step": 16} + ), + "is_image":( + [ + False, + True + ], + { + "default": False, + } + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + [ + "Flow", + ], + { + "default": 'Flow' + } + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional":{ + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, width, height, is_image, seed, steps, cfg, scheduler, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + config = funmodels['config'] + weight_dtype = funmodels['dtype'] + + # Get boundary for wan + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + + # Load Sampler + pipeline.scheduler = all_cheduler_dict[scheduler](**filter_kwargs(all_cheduler_dict[scheduler], OmegaConf.to_container(config['scheduler_kwargs']))) + + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + + generator= torch.Generator(device).manual_seed(seed) + + video_length = 1 if is_image else video_length + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype, sub_transformer_name="transformer_2") + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) + + +class Wan2_2I2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + [ + "Flow", + ], + { + "default": 'Flow' + } + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional":{ + "start_img": ("IMAGE",), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, scheduler, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, start_img=None, end_img=None, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + start_img = [to_pil(_start_img) for _start_img in start_img] if start_img is not None else None + end_img = [to_pil(_end_img) for _end_img in end_img] if end_img is not None else None + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + original_width, original_height = start_img[0].size if type(start_img) is list else Image.open(start_img).size + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + config = funmodels['config'] + weight_dtype = funmodels['dtype'] + + # Get boundary for wan + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + + # Load Sampler + pipeline.scheduler = all_cheduler_dict[scheduler](**filter_kwargs(all_cheduler_dict[scheduler], OmegaConf.to_container(config['scheduler_kwargs']))) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_img, end_img, video_length=video_length, sample_size=(height, width)) + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype, sub_transformer_name="transformer_2") + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) + diff --git a/comfyui/wan2_2/v1/wan2.2_workflow_i2v.json b/comfyui/wan2_2/v1/wan2.2_workflow_i2v.json new file mode 100644 index 0000000..daa231f --- /dev/null +++ b/comfyui/wan2_2/v1/wan2.2_workflow_i2v.json @@ -0,0 +1,473 @@ +{ + "id": "ca87b2cd-bd4a-4f31-82ec-e5e028f8848c", + "revision": 0, + "last_node_id": 101, + "last_link_id": 70, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 68 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 306, + 495 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 66 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "6.png", + "image" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 34, + 550 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 69 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -75, + -297 + ], + "size": [ + 350.7127990722656, + 125.54820251464844 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1277, + -70 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 70 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 101, + "type": "Wan2_2I2VSampler", + "pos": [ + 819.5313110351562, + -63.450443267822266 + ], + "size": [ + 325.5747985839844, + 402 + ], + "flags": { + "pinned": true + }, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 67 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 68 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 69 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 66 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 70 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2I2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 43, + 6, + "Flow", + 0.15000000000000002, + true, + 5, + true, + 0 + ] + }, + { + "id": 99, + "type": "LoadWan2_2Model", + "pos": [ + 347.27996826171875, + -299.0150146484375 + ], + "size": [ + 276.705078125, + 130 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2Model" + }, + "widgets_values": [ + "Wan2.2-I2V-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + } + ], + "links": [ + [ + 66, + 87, + 0, + 101, + 3, + "IMAGE" + ], + [ + 67, + 99, + 0, + 101, + 0, + "FunModels" + ], + [ + 68, + 75, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 69, + 73, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 70, + 101, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Group", + "bounding": [ + 220, + 409, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 272.66873113564736, + 429.81230900451357 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "ffe64f2b8c101da85e0707c59b355f21a83f5464", + "comfy-core": "0.3.44", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/wan2_2/v1/wan2.2_workflow_t2v.json b/comfyui/wan2_2/v1/wan2.2_workflow_t2v.json new file mode 100644 index 0000000..8a19bdf --- /dev/null +++ b/comfyui/wan2_2/v1/wan2.2_workflow_t2v.json @@ -0,0 +1,383 @@ +{ + "id": "f52daf1e-5f66-4c8b-9f68-890324faf01e", + "revision": 0, + "last_node_id": 87, + "last_link_id": 57, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 55 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1126, + -73 + ], + "size": [ + 390.9534912109375, + 538.0116577148438 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 54 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 56 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 87, + "type": "LoadWan2_2Model", + "pos": [ + 305.3617858886719, + -300.4237060546875 + ], + "size": [ + 276.705078125, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 57 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2Model" + }, + "widgets_values": [ + "Wan2.2-T2V-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_t2v.yaml", + "fp16" + ] + }, + { + "id": 86, + "type": "Wan2_2T2VSampler", + "pos": [ + 746.95556640625, + -69.9979248046875 + ], + "size": [ + 312.3404235839844, + 430 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 57 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 55 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 56 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 54 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2T2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 43, + "fixed", + 50, + 6, + "Flow", + 0.15000000000000002, + true, + 5, + true, + 0 + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -92, + -294 + ], + "size": [ + 351.1499938964844, + 130.12660217285156 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 54, + 86, + 0, + 17, + 0, + "IMAGE" + ], + [ + 55, + 75, + 0, + 86, + 1, + "STRING_PROMPT" + ], + [ + 56, + 73, + 0, + 86, + 2, + "STRING_PROMPT" + ], + [ + 57, + 87, + 0, + 86, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9090909090909091, + "offset": [ + 149.60461329280074, + 495.60652716079886 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "ffe64f2b8c101da85e0707c59b355f21a83f5464", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1b2c287..d795621 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ tensorboard beautifulsoup4 ftfy func_timeout +onnxruntime accelerate>=0.25.0 gradio>=3.41.2 diffusers>=0.30.1 diff --git a/videox_fun/api/api.py b/videox_fun/api/api.py index 253a1d8..a3c0238 100755 --- a/videox_fun/api/api.py +++ b/videox_fun/api/api.py @@ -207,8 +207,8 @@ def infer_forward_api(_: gr.Blocks, app: FastAPI, controller): cfg_skip_ratio = cfg_skip_ratio, enable_riflex = enable_riflex, riflex_k = riflex_k, - base_model_2_path = base_model_2_path, - lora_model_2_path = lora_model_2_path, + base_model_2_dropdown = base_model_2_path, + lora_model_2_dropdown = lora_model_2_path, fps = fps, is_api = True, ) diff --git a/videox_fun/api/api_multi_nodes.py b/videox_fun/api/api_multi_nodes.py index aaf9d15..ebd98d9 100755 --- a/videox_fun/api/api_multi_nodes.py +++ b/videox_fun/api/api_multi_nodes.py @@ -213,8 +213,8 @@ if ray is not None: cfg_skip_ratio = cfg_skip_ratio, enable_riflex = enable_riflex, riflex_k = riflex_k, - base_model_2_path = base_model_2_path, - lora_model_2_path = lora_model_2_path, + base_model_2_dropdown = base_model_2_path, + lora_model_2_dropdown = lora_model_2_path, fps = fps, is_api = True, ) diff --git a/videox_fun/data/dataset_image_video.py b/videox_fun/data/dataset_image_video.py index dfaded0..8ebc242 100755 --- a/videox_fun/data/dataset_image_video.py +++ b/videox_fun/data/dataset_image_video.py @@ -785,13 +785,15 @@ class ImageVideoControlDataset(Dataset): class ImageVideoSafetensorsDataset(Dataset): def __init__( self, - ann_path + ann_path, + data_root=None, ): # Loading annotations from files print(f"loading annotations from {ann_path} ...") if ann_path.endswith('.json'): dataset = json.load(open(ann_path)) - + + self.data_root = data_root self.dataset = dataset self.length = len(self.dataset) print(f"data scale: {self.length}") @@ -800,5 +802,9 @@ class ImageVideoSafetensorsDataset(Dataset): return self.length def __getitem__(self, idx): - state_dict = load_file(self.dataset[idx]["file_path"]) + if self.data_root is None: + path = self.dataset[idx]["file_path"] + else: + path = os.path.join(self.data_root, self.dataset[idx]["file_path"]) + state_dict = load_file(path) return state_dict \ No newline at end of file diff --git a/videox_fun/ui/cogvideox_fun_ui.py b/videox_fun/ui/cogvideox_fun_ui.py index e73e60e..748ca96 100755 --- a/videox_fun/ui/cogvideox_fun_ui.py +++ b/videox_fun/ui/cogvideox_fun_ui.py @@ -158,6 +158,8 @@ class CogVideoXFunController(Fun_Controller): cfg_skip_ratio = None, enable_riflex = None, riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, fps = None, is_api = False, ): diff --git a/videox_fun/ui/wan2_2_ui.py b/videox_fun/ui/wan2_2_ui.py index b7214bc..84195fc 100644 --- a/videox_fun/ui/wan2_2_ui.py +++ b/videox_fun/ui/wan2_2_ui.py @@ -110,6 +110,7 @@ class Wan2_2_Controller(Fun_Controller): if self.ulysses_degree > 1 or self.ring_degree > 1: from functools import partial self.transformer.enable_multi_gpus_inference() + self.transformer_2.enable_multi_gpus_inference() if self.fsdp_dit: shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) self.pipeline.transformer = shard_fn(self.pipeline.transformer) diff --git a/videox_fun/ui/wan_fun_ui.py b/videox_fun/ui/wan_fun_ui.py index a89536c..19b4043 100755 --- a/videox_fun/ui/wan_fun_ui.py +++ b/videox_fun/ui/wan_fun_ui.py @@ -188,6 +188,8 @@ class Wan_Fun_Controller(Fun_Controller): cfg_skip_ratio = None, enable_riflex = None, riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, fps = None, is_api = False, ): diff --git a/videox_fun/ui/wan_ui.py b/videox_fun/ui/wan_ui.py index 3613817..858c5d3 100755 --- a/videox_fun/ui/wan_ui.py +++ b/videox_fun/ui/wan_ui.py @@ -180,6 +180,8 @@ class Wan_Controller(Fun_Controller): cfg_skip_ratio = None, enable_riflex = None, riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, fps = None, is_api = False, ):