diff --git a/README.md b/README.md index a99f780..985611c 100644 --- a/README.md +++ b/README.md @@ -1 +1,239 @@ -# ComfyUI-LightVAE \ No newline at end of file +# ComfyUI-LightVAE + +
+ +![LightX2V](https://cdn-uploads.huggingface.co/production/uploads/680de13385293771bc57400b/tTnp8-ARpj3wGxfo5P55c.png) + +**High-Performance VAE Custom Nodes** + +[![HuggingFace](https://img.shields.io/badge/🤗-HuggingFace-yellow)](https://huggingface.co/lightx2v/Autoencoders) +[![GitHub](https://img.shields.io/badge/GitHub-LightX2V-blue?logo=github)](https://github.com/ModelTC/LightX2V) +[![License](https://img.shields.io/badge/License-Apache%202.0-green.svg)](LICENSE) + +English | [简体中文](README_CN.md) + +
+ + +## 📖 Introduction + +ComfyUI-LightVAE is a collection of LightX2V VAE custom nodes designed for ComfyUI, supporting high-performance video VAE models including **LightVAE** and **LightTAE**. + +The LightX2V team has deeply optimized VAE, creating two major series: **LightVAE** and **LightTAE**, which significantly reduce memory usage and improve inference speed while maintaining high quality. + + +## ✨ Key Features + + + + + + +
+ +### 🎯 LightVAE Series +**Feature: Best Balance ⚖️** + +- ✅ Uses **Causal 3D Conv** (same as official) +- ✅ **Near-official quality** ⭐⭐⭐⭐ +- ✅ **~50% less memory** (~4-5 GB) +- ✅ **2-3x faster** +- ✅ Balances quality, speed, and memory 🏆 + + + +### ⚡ LightTAE Series +**Feature: Ultra-fast + High Quality 🏆** + +- ✅ Minimal memory usage (~0.4 GB) +- ✅ Lightning-fast inference +- ✅ **Near-official quality** ⭐⭐⭐⭐ +- ✅ **Surpasses open-source TAE** + +
+ + +## 🚀 Performance Comparison + +**Test Environment**: H100 GPU, BF16, 81-frame video (480P) + +| Model | Encode Time | Decode Time | Encode Memory | Decode Memory | Quality | +|:------|:------------|:------------|:--------------|:--------------|:--------| +| **lightvaew2_1** | 1.5s | 2.1s | 4.8GB | 5.6GB | ⭐⭐⭐⭐⭐ | +| **lighttaew2_1** | 0.4s | 0.25s | 0.009GB | 0.4GB | ⭐⭐⭐⭐ | +| [Wan2.1_VAE](https://github.com/Wan-Video/Wan2.1) | 4.2s | 5.5s | 8.5GB | 10.1GB | ⭐⭐⭐⭐ | +| [taew2_1](https://github.com/madebyollin/taesd) | 0.4s | 0.25s | 0.009GB | 0.4GB | ⭐⭐⭐ | + +**Performance Improvements:** +- 🚀 LightVAE is **2-3x faster** than official VAE, **50% less memory** +- ⚡ LightTAE is **10+ times faster** than official VAE, **95%+ less memory** +- 🎨 Near-official [VAE](https://github.com/Wan-Video/Wan2.1) quality, surpasses open-source [TAE](https://github.com/madebyollin/taesd) + + +## 📦 Installation + +### 1. Install LightX2V Dependencies + +```bash +# Clone LightX2V repository +git clone https://github.com/ModelTC/LightX2V +cd LightX2V + +python setup_vae.py install +``` + +### 2. Install ComfyUI-WanVideoWrapper + +LightVAE nodes depend on WanVideoWrapper for main model support: + +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/kijai/ComfyUI-WanVideoWrapper +``` + +### 3. Install ComfyUI-LightVAE + +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/YOUR_USERNAME/ComfyUI-LightVAE +``` + +### 4. Restart ComfyUI + + +## 📥 Download Models + +### Main Models (Diffusion Models) + +**Option 1: Distilled Models (Recommended, 4-step)** +- 🔗 [Wan2.1 Distilled Models](https://huggingface.co/lightx2v/Wan2.1-Distill-Models) and [Wan2.2 Distilled Models](https://huggingface.co/lightx2v/Wan2.2-Distill-Models) +- ✅ Supports BF16 format +- ✅ Supports FP8 format (requires models with `_comfyui.safetensors` suffix) + +**Option 2: Original Models (20-step)** +- 🔗 [Wan2.1 Official Models](https://huggingface.co/lightx2v/Wan2.1-Official-Models) and [Wan2.2 Official Models](https://huggingface.co/lightx2v/Wan2.2-Official-Models) +- ✅ Supports BF16 format +- ✅ Supports FP8 format (requires models with `_comfyui.safetensors` suffix) + +```bash +# Download to ComfyUI/models/diffusion_models/ +huggingface-cli download lightx2v/Wan2.1/2-Distill-Models \ + --local-dir ./ComfyUI/models/diffusion_models/ +``` + +### VAE Models + +**All VAE Models** (Required): +- 🔗 [LightX2V Autoencoders](https://huggingface.co/lightx2v/Autoencoders) + +```bash +# Download all VAE models +huggingface-cli download lightx2v/Autoencoders \ + --local-dir ./ComfyUI/models/vae/ + +# Or download only what you need (Recommended) +huggingface-cli download lightx2v/Autoencoders lightvaew2_1.pth \ + --local-dir ./ComfyUI/models/vae/ +``` + +**Supported VAE Models**: +- `Wan2.1_VAE.pth` / `.safetensors` - Official VAE 2.1 +- `Wan2.2_VAE.pth` / `.safetensors` - Official VAE 2.2 +- `lightvaew2_1.pth` / `.safetensors` - Optimized VAE 2.1 ⭐ **Recommended** +- `taew2_1.pth` / `.safetensors` - Open-source TAE 2.1 +- `taew2_2.pth` / `.safetensors` - Open-source TAE 2.2 +- `lighttaew2_1.pth` / `.safetensors` - Optimized TAE 2.1 ⚡ **Fastest** +- `lighttaew2_2.pth` / `.safetensors` - Optimized TAE 2.2 + + +## 🎯 Node Documentation + +### 1. LightX2V VAE Decoder Loader +![VAE Loader](example/imgs/vae_loader.png) +Loads VAE decoder models. + +**Input Parameters:** +- `vae_filename` - VAE model filename (automatically lists from `./models/vae/`) +- `dtype` - Data type (bfloat16 / float16 / float32) +- `device` - Compute device (cuda / cpu) + +**Output:** +- `vae` - VAE model object + +**Features:** +- ✅ Automatically identifies VAE type from filename +- ✅ Supports all LightX2V VAE models + + + +### 2. LightX2V VAE Decode +![VAE Decode](example/imgs/vae_decoder.png) +Decodes latent to video frames. + +**Input Parameters:** +- `vae` - VAE object from Loader +- `latent` - Latent representation + +**Output:** +- `IMAGE` - Decoded video frames + +**Supports:** +- ✅ All VAE series (WanVAE, LightVAE) +- ✅ All TAE series (TAE, LightTAE) + + + +## 🖼️ Example Workflows + +### Wan2.1 I2V 4-step FP8 + LightVAE + +High-performance configuration using 4-step distilled model + LightVAE optimized decoder. + +![Wan2.1 LightVAE](example/imgs/wan21_lightvae.png) + +**Workflow File**: [`example/workflows/wan2.1_I2V_4step_fp8_lightvae.json`](example/workflows/wan2.1_I2V_4step_fp8_lightvae.json) + +### Wan2.2 TI2V + LightVAE + +Wan2.2 Text-Image-to-Video + LightVAE decoding. + +**Workflow File**: [`example/workflows/wan2.2_TI2V_lightvae.json`](example/workflows/wan2.2_TI2V_lightvae.json) + + + +## ⚠️ Important Notes + +### Model Compatibility +- ⚠️ **Wan2.1 VAE** can only be used with **Wan2.1/Wan2.2-A1B** backbone models +- ⚠️ **Wan2.2 VAE** can only be used with **Wan2.2 TI2V** backbone models +- ❌ Do not mix different versions of VAE and backbone models + + +## 📚 Related Resources + +- **Project Homepage**: https://github.com/ModelTC/LightX2V +- **VAE Models**: https://huggingface.co/lightx2v/Autoencoders +- **Video Generation Models**: https://huggingface.co/lightx2v/ +- **ComfyUI-WanVideoWrapper**: https://github.com/kijai/ComfyUI-WanVideoWrapper +- **TAE Series Models**: https://github.com/madebyollin/taesd +- **Wan-AI**: https://huggingface.co/Wan-AI + + +## 🙏 Acknowledgements + +If this project helps you, please give a ⭐ to [LightX2V](https://github.com/ModelTC/LightX2V) and this repository! + + + +## 📞 Support + +- **GitHub Issues**: Issues page of this repository +- **LightX2V Issues**: https://github.com/ModelTC/LightX2V/issues +- **HuggingFace**: https://huggingface.co/lightx2v + +
+ +**Enjoy using LightX2V VAE!** 🚀 + +
+ diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..43240ed --- /dev/null +++ b/README_CN.md @@ -0,0 +1,239 @@ +# ComfyUI-LightVAE + +
+ +![LightX2V](https://cdn-uploads.huggingface.co/production/uploads/680de13385293771bc57400b/tTnp8-ARpj3wGxfo5P55c.png) + +**高性能 VAE 自定义节点** + +[![HuggingFace](https://img.shields.io/badge/🤗-HuggingFace-yellow)](https://huggingface.co/lightx2v/Autoencoders) +[![GitHub](https://img.shields.io/badge/GitHub-LightX2V-blue?logo=github)](https://github.com/ModelTC/LightX2V) +[![License](https://img.shields.io/badge/License-Apache%202.0-green.svg)](LICENSE) + +[English](README.md) | 简体中文 + +
+ + +## 📖 简介 + +ComfyUI-LightVAE 是为 ComfyUI 设计的 LightX2V VAE 自定义节点集合,支持包括 **LightVAE** 和 **LightTAE** 在内的高性能视频 VAE 模型。 + +LightX2V 团队对 VAE 进行了深度优化,衍生出两大系列:**LightVAE** 和 **LightTAE**,在保持高质量的同时显著降低内存占用并提升推理速度。 + + +## ✨ 核心特性 + + + + + + +
+ +### 🎯 LightVAE 系列 +**特点:最佳平衡 ⚖️** + +- ✅ 使用 **Causal 3D Conv**(与官方相同) +- ✅ **质量接近官方** ⭐⭐⭐⭐ +- ✅ 内存减少 **~50%**(~4-5 GB) +- ✅ 速度提升 **2-3x** +- ✅ 平衡质量、速度和内存 🏆 + + + +### ⚡ LightTAE 系列 +**特点:极速 + 高质量 🏆** + +- ✅ 极低内存占用(~0.4 GB) +- ✅ 极速推理 +- ✅ **质量接近官方** ⭐⭐⭐⭐ +- ✅ **超越开源 TAE** + +
+ + +## 🚀 性能对比 + +**测试环境**: H100 GPU, BF16, 81 帧视频 (480P) + +| 模型 | 编码时间 | 解码时间 | 编码内存 | 解码内存 | 质量 | +|:-----|:---------|:---------|:---------|:---------|:-----| +| **lightvaew2_1** | 1.5s | 2.1s | 4.8GB | 5.6GB | ⭐⭐⭐⭐⭐ | +| **lighttaew2_1** | 0.4s | 0.25s | 0.009GB | 0.4GB | ⭐⭐⭐⭐ | +| [Wan2.1_VAE](https://github.com/Wan-Video/Wan2.1) | 4.2s | 5.5s | 8.5GB | 10.1GB | ⭐⭐⭐⭐ | +| [taew2_1](https://github.com/madebyollin/taesd) | 0.4s | 0.25s | 0.009GB | 0.4GB | ⭐⭐⭐ | + +**性能提升:** +- 🚀 LightVAE 比官方 VAE **快 2-3 倍**,内存**节省 50%** +- ⚡ LightTAE 比官方 VAE **快 10+ 倍**,内存**节省 95%+** +- 🎨 质量接近官方 [VAE](https://github.com/Wan-Video/Wan2.1),超过开源 [TAE](https://github.com/madebyollin/taesd) + + +## 📦 安装 + +### 1. 安装 LightX2V 依赖 + +```bash +# 克隆 LightX2V 仓库 +git clone https://github.com/ModelTC/LightX2V +cd LightX2V + +python setup_vae.py install +``` + +### 2. 安装 ComfyUI-WanVideoWrapper + +LightVAE 节点依赖 WanVideoWrapper 提供的主模型支持: + +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/kijai/ComfyUI-WanVideoWrapper +``` + +### 3. 安装 ComfyUI-LightVAE + +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/YOUR_USERNAME/ComfyUI-LightVAE +``` + +### 4. 重启 ComfyUI + + +## 📥 下载模型 + +### 主模型(Diffusion Models) + +**选项 1: 蒸馏模型(推荐,4-step)** +- 🔗 [Wan2.1蒸馏模型](https://huggingface.co/lightx2v/Wan2.1-Distill-Models) 和 [Wan2.2蒸馏模型](https://huggingface.co/lightx2v/Wan2.2-Distill-Models) +- ✅ 支持 BF16 格式 +- ✅ 支持 FP8 格式(需使用 `_comfyui.safetensors` 后缀的模型) + +**选项 2: 原始模型(20-step)** +- 🔗 [Wan2.1官方模型](https://huggingface.co/lightx2v/Wan2.1-Official-Models) 和 [Wan2.2官方模型](https://huggingface.co/lightx2v/Wan2.2-Official-Models) +- ✅ 支持 BF16 格式 +- ✅ 支持 FP8 格式(需使用 `_comfyui.safetensors` 后缀的模型) + +```bash +# 下载到 ComfyUI/models/diffusion_models/ +huggingface-cli download lightx2v/Wan2.1/2-Distill-Models \ + --local-dir ./ComfyUI/models/diffusion_models/ +``` + +### VAE 模型 + +**所有 VAE 模型**(必需): +- 🔗 [LightX2V Autoencoders](https://huggingface.co/lightx2v/Autoencoders) + +```bash +# 下载所有 VAE 模型 +huggingface-cli download lightx2v/Autoencoders \ + --local-dir ./ComfyUI/models/vae/ + +# 或只下载需要的(推荐) +huggingface-cli download lightx2v/Autoencoders lightvaew2_1.pth \ + --local-dir ./ComfyUI/models/vae/ +``` + +**支持的 VAE 模型**: +- `Wan2.1_VAE.pth` / `.safetensors` - 官方 VAE 2.1 +- `Wan2.2_VAE.pth` / `.safetensors` - 官方 VAE 2.2 +- `lightvaew2_1.pth` / `.safetensors` - 优化 VAE 2.1 ⭐ **推荐** +- `taew2_1.pth` / `.safetensors` - 开源 TAE 2.1 +- `taew2_2.pth` / `.safetensors` - 开源 TAE 2.2 +- `lighttaew2_1.pth` / `.safetensors` - 优化 TAE 2.1 ⚡ **最快** +- `lighttaew2_2.pth` / `.safetensors` - 优化 TAE 2.2 + + +## 🎯 节点说明 + +### 1. LightX2V VAE Decoder Loader +![VAE Loader](example/imgs/vae_loader.png) +加载 VAE 解码器模型。 + +**输入参数:** +- `vae_filename` - VAE 模型文件名(从 `./models/vae/` 目录自动列出) +- `dtype` - 数据类型(bfloat16 / float16 / float32) +- `device` - 计算设备(cuda / cpu) + +**输出:** +- `vae` - VAE 模型对象 + +**特点:** +- ✅ 自动从文件名识别 VAE 类型 +- ✅ 支持所有 LightX2V VAE 模型 + + + +### 2. LightX2V VAE Decode +![VAE Decode](example/imgs/vae_decoder.png) +解码 latent 到视频帧。 + +**输入参数:** +- `vae` - 来自 Loader 的 VAE 对象 +- `latent` - 潜在表示 + +**输出:** +- `IMAGE` - 解码后的视频帧 + +**支持:** +- ✅ 所有 VAE 系列(WanVAE, LightVAE) +- ✅ 所有 TAE 系列(TAE, LightTAE) + + + +## 🖼️ 示例工作流 + +### Wan2.1 I2V 4-step FP8 + LightVAE + +使用 4-step 蒸馏模型 + LightVAE 优化解码器的高性能配置。 + +![Wan2.1 LightVAE](example/imgs/wan21_lightvae.png) + +**工作流文件**: [`example/workflows/wan2.1_I2V_4step_fp8_lightvae.json`](example/workflows/wan2.1_I2V_4step_fp8_lightvae.json) + +### Wan2.2 TI2V + LightVAE + +Wan2.2 文本图像到视频 + LightVAE 解码。 + +**工作流文件**: [`example/workflows/wan2.2_TI2V_lightvae.json`](example/workflows/wan2.2_TI2V_lightvae.json) + + + +## ⚠️ 注意事项 + +### 模型兼容性 +- ⚠️ **Wan2.1 VAE** 只能用于 **Wan2.1/Wan2.2-A1B** 主干模型 +- ⚠️ **Wan2.2 VAE** 只能用于 **Wan2.2 TI2V** 主干模型 +- ❌ 不要混用不同版本的 VAE 和主干模型 + + +## 📚 相关资源 + +- **项目主页**: https://github.com/ModelTC/LightX2V +- **VAE 模型**: https://huggingface.co/lightx2v/Autoencoders +- **视频生成模型**: https://huggingface.co/lightx2v/ +- **ComfyUI-WanVideoWrapper**: https://github.com/kijai/ComfyUI-WanVideoWrapper +- **tae系列模型**: https://github.com/madebyollin/taesd +- **Wan-AI**: https://huggingface.co/Wan-AI + + +## 🙏 致谢 + +如果这个项目对你有帮助,请给 ⭐ [LightX2V](https://github.com/ModelTC/LightX2V) 和本仓库! + + + +## 📞 支持 + +- **GitHub Issues**: 本仓库的 Issues 页面 +- **LightX2V Issues**: https://github.com/ModelTC/LightX2V/issues +- **HuggingFace**: https://huggingface.co/lightx2v + +
+ +**享受使用 LightX2V VAE!** 🚀 + +
+ diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..1b3b42b --- /dev/null +++ b/__init__.py @@ -0,0 +1,16 @@ +from .nodes import ( + VAEDecoderLoader, + LightVAEDecode +) + +NODE_CLASS_MAPPINGS = { + "Lightx2v-VAEDecoderLoader": VAEDecoderLoader, + "Lightx2v-VAEDecode": LightVAEDecode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "Lightx2v-VAEDecoderLoader": "LightX2V VAE Decoder Loader", + "Lightx2v-VAEDecode": "LightX2V VAE Decode", +} + +__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] diff --git a/example/imgs/example.jpg b/example/imgs/example.jpg new file mode 100644 index 0000000..6185149 Binary files /dev/null and b/example/imgs/example.jpg differ diff --git a/example/imgs/vae_decode.png b/example/imgs/vae_decode.png new file mode 100644 index 0000000..0bc1f7e Binary files /dev/null and b/example/imgs/vae_decode.png differ diff --git a/example/imgs/vae_loader.png b/example/imgs/vae_loader.png new file mode 100644 index 0000000..9e2b323 Binary files /dev/null and b/example/imgs/vae_loader.png differ diff --git a/example/workflows/wan2.1_I2V_4step_fp8_lightvae.json b/example/workflows/wan2.1_I2V_4step_fp8_lightvae.json new file mode 100644 index 0000000..95434d6 --- /dev/null +++ b/example/workflows/wan2.1_I2V_4step_fp8_lightvae.json @@ -0,0 +1,1487 @@ +{ + "id": "c6e410bc-5e2c-460b-ae81-c91b6094fbb1", + "revision": 0, + "last_node_id": 75, + "last_link_id": 87, + "nodes": [ + { + "id": 45, + "type": "WanVideoVRAMManagement", + "pos": [ + -140, + 10 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vram_management_args", + "type": "VRAM_MANAGEMENTARGS", + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanVideoVRAMManagement" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 42, + "type": "Note", + "pos": [ + -490, + 10 + ], + "size": [ + 314.96246337890625, + 152.77333068847656 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Adjust the blocks to swap based on your VRAM, this is a tradeoff between speed and memory usage.\n\nAlternatively there's option to use VRAM management introduced in DiffSynt-Studios. This is usually slower, but saves even more VRAM compared to BlockSwap" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 44, + "type": "Note", + "pos": [ + -550, + -250 + ], + "size": [ + 303.0501403808594, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "If you have Triton installed, connect this for ~30% speed increase" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 35, + "type": "WanVideoTorchCompileSettings", + "pos": [ + -220, + -250 + ], + "size": [ + 390.5999755859375, + 202 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ] + }, + { + "id": 56, + "type": "Note", + "pos": [ + 240, + 90 + ], + "size": [ + 359.0753479003906, + 88 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "https://huggingface.co/Kijai/WanVideo_comfy/blob/main/umt5-xxl-enc-fp8_e4m3fn.safetensors" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 63, + "type": "Note", + "pos": [ + 240, + 270 + ], + "size": [ + 359.0753479003906, + 88 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "https://huggingface.co/Kijai/WanVideo_comfy/blob/main/umt5-xxl-enc-fp8_e4m3fn.safetensors" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 39, + "type": "WanVideoBlockSwap", + "pos": [ + 280, + -310 + ], + "size": [ + 315, + 202 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "slot_index": 0, + "links": [ + 50 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 20, + false, + false, + true, + 0, + 0, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 33, + "type": "Note", + "pos": [ + 240, + -50 + ], + "size": [ + 359.0753479003906, + 88 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan2_1_VAE_bf16.safetensors" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 36, + "type": "Note", + "pos": [ + 640, + -580 + ], + "size": [ + 340, + 170 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "fp8_fast seems to cause huge quality degradation\n\nfp_16_fast enables \"Full FP16 Accmumulation in FP16 GEMMs\" feature available in the very latest pytorch nightly, this is around 20% speed boost. \n\nSageattn if you have it installed can be used for almost double inference speed\n\nhttps://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan2_1-I2V-14B-720P_fp8_e4m3fn.safetensors" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 52, + "type": "WanVideoTeaCache", + "pos": [ + 1840, + 890 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "cache_args", + "type": "CACHEARGS", + "links": [ + 56 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTeaCache" + }, + "widgets_values": [ + 0.25, + 1, + -1, + "offload_device", + "true", + "e" + ] + }, + { + "id": 54, + "type": "Note", + "pos": [ + 2200, + 720 + ], + "size": [ + 327.61932373046875, + 88 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Enhance-a-video can increase the fidelity of the results, too high values lead to noisy results." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 53, + "type": "Note", + "pos": [ + 2200, + 900 + ], + "size": [ + 320, + 160 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "TeaCache could be considered to be sort of an automated step skipper \n\nThe relative l1 threshold -value determines how aggressive this is, higher values are faster but quality suffers more. Very first steps should NEVER be skipped with this model or it kills the motion. When using the pre-calculated coefficients, the treshold value should be much higher than with the default coefficients." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 65, + "type": "CLIPVisionLoader", + "pos": [ + 650, + 280 + ], + "size": [ + 340, + 60 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 68 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader", + "models": [ + { + "name": "clip_vision_h.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/clip_vision/clip_vision_h.safetensors?download=true", + "directory": "clip_vision" + } + ] + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 59, + "type": "WanVideoClipVisionEncode", + "pos": [ + 907.02099609375, + 457.4317932128906 + ], + "size": [ + 327.5999755859375, + 262 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 68 + }, + { + "name": "image_1", + "type": "IMAGE", + "link": 63 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "negative_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_CLIPEMBEDS", + "links": [ + 58 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoClipVisionEncode", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + 1, + 1, + "center", + "average", + true, + 0, + 0.20000000000000004 + ] + }, + { + "id": 11, + "type": "LoadWanVideoT5TextEncoder", + "pos": [ + 640, + -50 + ], + "size": [ + 350, + 130 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wan_t5_model", + "type": "WANTEXTENCODER", + "slot_index": 0, + "links": [ + 15 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVideoT5TextEncoder", + "models": [ + { + "name": "umt5-xxl-enc-bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/umt5-xxl-enc-bf16.safetensors?download=true", + "directory": "text_encoders" + } + ] + }, + "widgets_values": [ + "umt5_xxl_fp16.safetensors", + "bf16", + "offload_device", + "disabled" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 55, + "type": "WanVideoEnhanceAVideo", + "pos": [ + 1850, + 711.1280517578125 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "feta_args", + "type": "FETAARGS", + "links": [ + 57 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoEnhanceAVideo" + }, + "widgets_values": [ + 2, + 0, + 1 + ] + }, + { + "id": 38, + "type": "WanVideoVAELoader", + "pos": [ + 640, + 140 + ], + "size": [ + 350, + 82 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 43, + 66 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoVAELoader", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan2_1_VAE_bf16.safetensors?download=true", + "directory": "vae" + } + ] + }, + "widgets_values": [ + "Wan2.1_VAE.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 30, + "type": "VHS_VideoCombine", + "pos": [ + 2180.822998046875, + -579.9119873046875 + ], + "size": [ + 630, + 310 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 87 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "WanVideo2_1_I2V", + "format": "video/h265-mp4", + "pix_fmt": "yuv420p10le", + "crf": 22, + "save_metadata": false, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_1_I2V_00024.mp4", + "subfolder": "", + "type": "output", + "format": "video/h265-mp4", + "frame_rate": 16, + "workflow": "WanVideo2_1_I2V_00024.png", + "fullpath": "/data/nvme0/gaopeng1/workspace/github/ComfyUI/output/WanVideo2_1_I2V_00024.mp4" + } + } + } + }, + { + "id": 58, + "type": "WanVideoImageToVideoEncode", + "pos": [ + 1341.2952880859375, + 413.4355163574219 + ], + "size": [ + 352.79998779296875, + 390 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "vae", + "shape": 7, + "type": "WANVAE", + "link": 66 + }, + { + "name": "clip_embeds", + "shape": 7, + "type": "WANVIDIMAGE_CLIPEMBEDS", + "link": 58 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 59 + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_embeds", + "shape": 7, + "type": "WANVIDIMAGE_EMBEDS", + "link": null + }, + { + "name": "temporal_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "extra_latents", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "add_cond_latents", + "shape": 7, + "type": "ADD_COND_LATENTS", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 60 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 61 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 65 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoImageToVideoEncode", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + 832, + 480, + 81, + 0.030000000000000006, + 1, + 1, + true, + false, + false + ] + }, + { + "id": 60, + "type": "ImageResizeKJ", + "pos": [ + 936.13232421875, + 853.13818359375 + ], + "size": [ + 315, + 266 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 64 + }, + { + "name": "get_image_size", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 59, + 63 + ] + }, + { + "name": "width", + "type": "INT", + "links": [ + 60 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 61 + ] + } + ], + "properties": { + "Node name for S&R": "ImageResizeKJ", + "cnr_id": "comfyui-kjnodes", + "ver": "52c2e31a903fec2dd654fb614ea82ba2757d5028" + }, + "widgets_values": [ + 720, + 728, + "lanczos", + false, + 16, + 0 + ] + }, + { + "id": 61, + "type": "LoadImage", + "pos": [ + 70.51425170898438, + 547.34619140625 + ], + "size": [ + 413.10479736328125, + 498.3180847167969 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 64 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "cnr_id": "comfy-core", + "ver": "0.3.26" + }, + "widgets_values": [ + "example.jpg", + "image" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 16, + "type": "WanVideoTextEncode", + "pos": [ + 1222.5087890625, + -241.90980529785156 + ], + "size": [ + 459.6058654785156, + 570.80029296875 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "t5", + "shape": 7, + "type": "WANTEXTENCODER", + "link": 15 + }, + { + "name": "model_to_offload", + "shape": 7, + "type": "WANVIDEOMODEL", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "slot_index": 0, + "links": [ + 30 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTextEncode" + }, + "widgets_values": [ + "The white dragon warrior stands still, eyes full of determination and strength. The camera slowly moves closer or circles around the warrior, highlighting the powerful presence and heroic spirit of the character.", + "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走", + true, + false, + "gpu" + ] + }, + { + "id": 22, + "type": "WanVideoModelLoader", + "pos": [ + 640, + -330 + ], + "size": [ + 340, + 274 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 50 + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "slot_index": 0, + "links": [ + 29 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoModelLoader", + "models": [ + { + "name": "Wan2_1-I2V-14B-720P_fp8_e4m3fn.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1-I2V-14B-720P_fp8_e4m3fn.safetensors?download=true", + "directory": "diffusion_models" + } + ] + }, + "widgets_values": [ + "wan2.1_i2v_720p_scaled_fp8_e4m3_lightx2v_4step_comfyui.safetensors", + "fp16", + "fp8_e4m3fn_scaled", + "offload_device", + "sageattn" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 27, + "type": "WanVideoSampler", + "pos": [ + 2582.6533203125, + 60.40684127807617 + ], + "size": [ + 315, + 710 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 29 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 65 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 30 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": 57 + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": null + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": null + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + }, + { + "name": "teacache_args", + "shape": 7, + "type": "TEACACHEARGS", + "link": 56 + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 33, + 86 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 4, + 1, + 5, + 1057359483639287, + "fixed", + true, + "euler", + 0, + 1, + false, + "comfy", + 0, + -1, + false + ] + }, + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 1458.4403076171875, + -786.5699462890625 + ], + "size": [ + 310, + 300 + ], + "flags": {}, + "order": 24, + "mode": 4, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 43 + }, + { + "name": "samples", + "type": "LATENT", + "link": 33 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 512, + 480, + 144, + 792, + "default" + ] + }, + { + "id": 73, + "type": "Lightx2v-VAEDecoderLoader", + "pos": [ + 1305.482666015625, + -413.7549133300781 + ], + "size": [ + 291.8736267089844, + 106 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAE", + "links": [ + 85 + ] + } + ], + "properties": { + "Node name for S&R": "Lightx2v-VAEDecoderLoader" + }, + "widgets_values": [ + "lighttaew2_1.safetensors", + "bfloat16", + "cuda" + ] + }, + { + "id": 75, + "type": "Lightx2v-VAEDecode", + "pos": [ + 1689.6263427734375, + -406.6926574707031 + ], + "size": [ + 327.094970703125, + 104.76178741455078 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "VAE", + "link": 85 + }, + { + "name": "latent", + "type": "LATENT", + "link": 86 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "Lightx2v-VAEDecode" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 15, + 11, + 0, + 16, + 0, + "WANTEXTENCODER" + ], + [ + 29, + 22, + 0, + 27, + 0, + "WANVIDEOMODEL" + ], + [ + 30, + 16, + 0, + 27, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 33, + 27, + 0, + 28, + 1, + "LATENT" + ], + [ + 43, + 38, + 0, + 28, + 0, + "VAE" + ], + [ + 50, + 39, + 0, + 22, + 1, + "BLOCKSWAPARGS" + ], + [ + 56, + 52, + 0, + 27, + 17, + "TEACACHEARGS" + ], + [ + 57, + 55, + 0, + 27, + 4, + "FETAARGS" + ], + [ + 58, + 59, + 0, + 58, + 1, + "WANVIDIMAGE_CLIPEMBEDS" + ], + [ + 59, + 60, + 0, + 58, + 2, + "IMAGE" + ], + [ + 60, + 60, + 1, + 58, + 8, + "INT" + ], + [ + 61, + 60, + 2, + 58, + 9, + "INT" + ], + [ + 63, + 60, + 0, + 59, + 1, + "IMAGE" + ], + [ + 64, + 61, + 0, + 60, + 0, + "IMAGE" + ], + [ + 65, + 58, + 0, + 27, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 66, + 38, + 0, + 58, + 0, + "WANVAE" + ], + [ + 68, + 65, + 0, + 59, + 0, + "CLIP_VISION" + ], + [ + 85, + 73, + 0, + 75, + 0, + "VAE" + ], + [ + 86, + 27, + 0, + 75, + 1, + "LATENT" + ], + [ + 87, + 75, + 0, + 30, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 2, + "title": "Load models", + "bounding": [ + 630, + -400, + 380, + 760 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Optional", + "bounding": [ + -560, + -320, + 745, + 496.3733215332031 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Image Encode", + "bounding": [ + 630, + 390, + 1180, + 680 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8140274938684746, + "offset": [ + -382.97792893759515, + 459.42837653740787 + ] + }, + "node_versions": { + "ComfyUI-WanVideoWrapper": "56a4221a380a572ee976c90dc886a132f5fa279e", + "comfy-core": "0.3.27", + "comfyui-kjnodes": "26aea1889d02d0bf072ac9aae0e90e5815ffa175", + "comfyui-videohelpersuite": "df55f01d1df2f7bf5cc772294bc2e6d8bab22d66" + }, + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "frontendVersion": "1.25.11" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example/workflows/wan2.2_TI2V_lightvae.json b/example/workflows/wan2.2_TI2V_lightvae.json new file mode 100644 index 0000000..a4498b6 --- /dev/null +++ b/example/workflows/wan2.2_TI2V_lightvae.json @@ -0,0 +1,1149 @@ +{ + "id": "206247b6-9fec-4ed2-8927-e4f388c674d4", + "revision": 0, + "last_node_id": 99, + "last_link_id": 148, + "nodes": [ + { + "id": 46, + "type": "WanVideoTextEmbedBridge", + "pos": [ + 768.0983276367188, + 914.539794921875 + ], + "size": [ + 315, + 46 + ], + "flags": {}, + "order": 13, + "mode": 2, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 54 + }, + { + "name": "negative", + "shape": 7, + "type": "CONDITIONING", + "link": 55 + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + } + ], + "properties": { + "Node name for S&R": "WanVideoTextEmbedBridge", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [] + }, + { + "id": 50, + "type": "CLIPTextEncode", + "pos": [ + 318.0990905761719, + 1174.5400390625 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 11, + "mode": 2, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 53 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 55 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.26" + }, + "widgets_values": [ + "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" + ] + }, + { + "id": 48, + "type": "CLIPLoader", + "pos": [ + -41.90084457397461, + 924.539794921875 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 0, + "mode": 2, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 52, + 53 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.3.26" + }, + "widgets_values": [ + "umt5_xxl_fp16.safetensors", + "wan", + "default" + ] + }, + { + "id": 49, + "type": "CLIPTextEncode", + "pos": [ + 318.0990905761719, + 924.539794921875 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 10, + "mode": 2, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 52 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 54 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.26" + }, + "widgets_values": [ + "high quality nature video featuring a red panda balancing on a bamboo stem while a bird lands on it's head, on the background there is a waterfall" + ] + }, + { + "id": 51, + "type": "Note", + "pos": [ + -11.900712966918945, + 754.539306640625 + ], + "size": [ + 253.16725158691406, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "You can also use native ComfyUI text encoding with these nodes instead of the original, the models are node specific and can't otherwise be mixed." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 35, + "type": "WanVideoTorchCompileSettings", + "pos": [ + -303.0111999511719, + -842.4323120117188 + ], + "size": [ + 390.5999755859375, + 202 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "slot_index": 0, + "links": [ + 121 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTorchCompileSettings", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 90, + "type": "WanVideoExperimentalArgs", + "pos": [ + 918.6209106445312, + -165.79469299316406 + ], + "size": [ + 308.767578125, + 274 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "exp_args", + "type": "EXPERIMENTALARGS", + "links": [ + 139 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoExperimentalArgs", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3f4e8433bdc7adeba7166b40e10ffcb36670f15d" + }, + "widgets_values": [ + "", + true, + false, + 0, + false, + 1, + 1.25, + 20, + true, + 0 + ] + }, + { + "id": 94, + "type": "WanVideoEasyCache", + "pos": [ + 966.5114135742188, + -510.91009521484375 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "cache_args", + "type": "CACHEARGS", + "links": [ + 141 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoEasyCache", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3f4e8433bdc7adeba7166b40e10ffcb36670f15d" + }, + "widgets_values": [ + 0.015, + 10, + -1, + "offload_device" + ] + }, + { + "id": 91, + "type": "WanVideoSLG", + "pos": [ + 973.4921875, + 195.37266540527344 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "slg_args", + "type": "SLGARGS", + "links": [ + 140 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoSLG", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3f4e8433bdc7adeba7166b40e10ffcb36670f15d" + }, + "widgets_values": [ + "7,8,9", + 0.1, + 0.7 + ] + }, + { + "id": 78, + "type": "WanVideoEmptyEmbeds", + "pos": [ + 579.958984375, + -131.9554443359375 + ], + "size": [ + 272.431640625, + 126 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "control_embeds", + "shape": 7, + "type": "WANVIDIMAGE_EMBEDS", + "link": null + }, + { + "name": "extra_latents", + "shape": 7, + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 102 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoEmptyEmbeds", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "6bc53b771d5d2af316801cb69e2ee10dbf7d18b1" + }, + "widgets_values": [ + 832, + 480, + 121 + ] + }, + { + "id": 16, + "type": "WanVideoTextEncode", + "pos": [ + 412.7743225097656, + 135.32998657226562 + ], + "size": [ + 453.0067138671875, + 328.0239562988281 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "t5", + "shape": 7, + "type": "WANTEXTENCODER", + "link": 15 + }, + { + "name": "model_to_offload", + "shape": 7, + "type": "WANVIDEOMODEL", + "link": 79 + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "slot_index": 0, + "links": [ + 30 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTextEncode", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + "日出,侧光,边缘光,中近景,中心构图,柔光,暖色调,低饱和度,晴天光,镜头前是一位满脸皱纹的外国老人,身穿深色棉质衬衫,双手交叠放在膝盖上,身体微微前倾,眼神坚定望向远方。晨光从画面左侧斜射而来,在他面部轮廓勾勒出柔和的高光,突显出岁月留下的深刻纹路。背景为模糊的田野与渐变橙黄的天空,营造出宁静而庄重的氛围。", + "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards\"", + true, + false, + "gpu" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 11, + "type": "LoadWanVideoT5TextEncoder", + "pos": [ + -223.19448852539062, + -498.4925842285156 + ], + "size": [ + 377.1661376953125, + 130 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wan_t5_model", + "type": "WANTEXTENCODER", + "slot_index": 0, + "links": [ + 15 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVideoT5TextEncoder", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + "umt5_xxl_fp16.safetensors", + "bf16", + "offload_device", + "disabled" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 22, + "type": "WanVideoModelLoader", + "pos": [ + 157.20700073242188, + -839.4575805664062 + ], + "size": [ + 477.4410095214844, + 274 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": 121 + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": null + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "slot_index": 0, + "links": [ + 29, + 79 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoModelLoader", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + "wan2.2_ti2v_lightx2v.safetensors", + "bf16", + "disabled", + "offload_device", + "sageattn" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 27, + "type": "WanVideoSampler", + "pos": [ + 1315.2401123046875, + -401.48028564453125 + ], + "size": [ + 315, + 975 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 29 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 102 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 30 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": 141 + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": 140 + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": 139 + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": null + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 117, + 147 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "WanVideoSampler", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + 30, + 5, + 8, + 47, + "fixed", + true, + "flowmatch_pusa", + 0, + 1, + "", + "comfy", + 0, + -1, + "" + ] + }, + { + "id": 99, + "type": "Lightx2v-VAEDecode", + "pos": [ + 1248.7696533203125, + -713.8431396484375 + ], + "size": [ + 179.72909545898438, + 46 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "VAE", + "link": 146 + }, + { + "name": "latent", + "type": "LATENT", + "link": 147 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 148 + ] + } + ], + "properties": { + "Node name for S&R": "Lightx2v-VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 1412.816650390625, + -1122.835693359375 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 16, + "mode": 4, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 43 + }, + { + "name": "samples", + "type": "LATENT", + "link": 117 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanVideoDecode", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 38, + "type": "WanVideoVAELoader", + "pos": [ + 722.0611572265625, + -1094.1468505859375 + ], + "size": [ + 372.7727966308594, + 82 + ], + "flags": {}, + "order": 8, + "mode": 4, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 43 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoVAELoader", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d9b1f4d1a5aea91d101ae97a54714a5861af3f50" + }, + "widgets_values": [ + "Wan2.2_VAE.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 92, + "type": "VHS_VideoCombine", + "pos": [ + 1917.0638427734375, + -844.4730224609375 + ], + "size": [ + 934.6708984375, + 875.6947631835938 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 148 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine", + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8" + }, + "widgets_values": { + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "WanVideoWrapper_5BI2V", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideoWrapper_5BI2V_00005.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 24, + "workflow": "WanVideoWrapper_5BI2V_00005.png", + "fullpath": "/data/nvme0/gaopeng1/workspace/github/ComfyUI/temp/WanVideoWrapper_5BI2V_00005.mp4" + } + } + } + }, + { + "id": 98, + "type": "Lightx2v-VAEDecoderLoader", + "pos": [ + 911.6253662109375, + -899.477294921875 + ], + "size": [ + 291.8736267089844, + 106 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAE", + "links": [ + 146 + ] + } + ], + "properties": { + "Node name for S&R": "Lightx2v-VAEDecoderLoader" + }, + "widgets_values": [ + "lighttaew2_2.safetensors", + "bfloat16", + "cuda" + ] + } + ], + "links": [ + [ + 15, + 11, + 0, + 16, + 0, + "WANTEXTENCODER" + ], + [ + 29, + 22, + 0, + 27, + 0, + "WANVIDEOMODEL" + ], + [ + 30, + 16, + 0, + 27, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 43, + 38, + 0, + 28, + 0, + "VAE" + ], + [ + 52, + 48, + 0, + 49, + 0, + "CLIP" + ], + [ + 53, + 48, + 0, + 50, + 0, + "CLIP" + ], + [ + 54, + 49, + 0, + 46, + 0, + "CONDITIONING" + ], + [ + 55, + 50, + 0, + 46, + 1, + "CONDITIONING" + ], + [ + 79, + 22, + 0, + 16, + 1, + "WANVIDEOMODEL" + ], + [ + 102, + 78, + 0, + 27, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 117, + 27, + 0, + 28, + 1, + "LATENT" + ], + [ + 121, + 35, + 0, + 22, + 0, + "WANCOMPILEARGS" + ], + [ + 139, + 90, + 0, + 27, + 10, + "EXPERIMENTALARGS" + ], + [ + 140, + 91, + 0, + 27, + 8, + "SLGARGS" + ], + [ + 141, + 94, + 0, + 27, + 6, + "CACHEARGS" + ], + [ + 146, + 98, + 0, + 99, + 0, + "VAE" + ], + [ + 147, + 27, + 0, + 99, + 1, + "LATENT" + ], + [ + 148, + 99, + 0, + 92, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "ComfyUI text encoding alternative", + "bounding": [ + -104.71692657470703, + 610.6055908203125, + 1210.621337890625, + 805.9080810546875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8954302432552969, + "offset": [ + -340.7324416173109, + 889.6400044839784 + ] + }, + "frontendVersion": "1.25.11", + "node_versions": { + "ComfyUI-WanVideoWrapper": "5a2383621a05825d0d0437781afcb8552d9590fd", + "comfy-core": "0.3.26", + "ComfyUI-KJNodes": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", + "ComfyUI-VideoHelperSuite": "0a75c7958fe320efcb052f1d9f8451fd20c730a8" + }, + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes.py b/nodes.py new file mode 100644 index 0000000..b14b206 --- /dev/null +++ b/nodes.py @@ -0,0 +1,143 @@ +""" +LightX2V VAE Nodes for ComfyUI +VAE Loader + 各类 VAE 的 Decode 节点 +""" + +import os +import torch +import folder_paths +from lightx2v.models.video_encoders.hf.wan.vae import WanVAE +from lightx2v.models.video_encoders.hf.wan.vae_2_2 import Wan2_2_VAE +from lightx2v.models.video_encoders.hf.wan.vae_tiny import WanVAE_tiny, Wan2_2_VAE_tiny + +class VAEDecoderLoader: + @classmethod + def INPUT_TYPES(cls): + vae_default_dir = "./models/vae/" + return { + "required": { + "vae_filename": (os.listdir(vae_default_dir) ,{"default": "Wan2.1_VAE.safetensors"}), + "dtype": (["bfloat16", "float16", "float32"], {"default": "bfloat16"}), + "device": (["cuda", "cpu"], {"default": "cuda"}), + } + } + + RETURN_TYPES = ("VAE",) + RETURN_NAMES = ("vae",) + FUNCTION = "load_vae" + CATEGORY = "LightX2V" + + def load_vae(self, vae_filename: str, dtype: str, device: str): + dtype_map = { + "bfloat16": torch.bfloat16, + "float16": torch.float16, + "float32": torch.float32 + } + vae_name = os.path.splitext(vae_filename)[0] + vae_default_dir = "./models/vae/" + torch_dtype = dtype_map[dtype] + vae_path = os.path.join(vae_default_dir, vae_filename) + + if not os.path.exists(vae_path): + raise FileNotFoundError(f"VAE model not found: {vae_path}") + + if vae_name == "Wan2.1_VAE": + model = WanVAE( + vae_path=vae_path, + dtype=torch_dtype, + device=device, + use_lightvae=False + ) + + elif vae_name == "Wan2.2_VAE": + model = Wan2_2_VAE( + vae_path=vae_path, + dtype=torch_dtype, + device=device + ) + + elif vae_name == "lightvaew2_1": + model = WanVAE( + vae_path=vae_path, + dtype=torch_dtype, + device=device, + use_lightvae=True + ) + + elif vae_name == "taew2_1": + model = WanVAE_tiny( + vae_path=vae_path, + dtype=torch_dtype, + device=device, + need_scaled=False + ).to(device) + + elif vae_name == "taew2_2": + model = Wan2_2_VAE_tiny( + vae_path=vae_path, + dtype=torch_dtype, + device=device, + need_scaled=False + ).to(device) + + elif vae_name == "lighttaew2_1": + model = WanVAE_tiny( + vae_path=vae_path, + dtype=torch_dtype, + device=device, + need_scaled=True + ).to(device) + + elif vae_name == "lighttaew2_2": + model = Wan2_2_VAE_tiny( + vae_path=vae_path, + dtype=torch_dtype, + device=device, + need_scaled=True + ).to(device) + + else: + raise ValueError(f"Unknown vae_name: {vae_name}") + + return ({ + "model": model, + "vae_name": vae_name, + "dtype": torch_dtype, + "device": device, + },) + + + +class LightVAEDecode: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "vae": ("VAE",), + "latent": ("LATENT",), + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "decode" + CATEGORY = "LightX2V" + + @torch.no_grad() + def decode(self, vae: dict, latent: dict): + model = vae["model"] + dtype = vae["dtype"] + device = vae["device"] + + latent_samples = latent["samples"].to(device, dtype) + with torch.no_grad(): + video = model.decode(latent_samples.squeeze(0)).half().permute(0, 2, 3, 4, 1) + + if len(video.shape) == 5: #Combine batches + video = video.reshape(-1, video.shape[-3], video.shape[-2], video.shape[-1]) + return (video.add_(1.0).div_(2.0),) + + +__all__ = [ + "VAEDecoderLoader", + "LightVAEDecode" +] \ No newline at end of file