diff --git a/.gitignore b/.gitignore index 0037a7803..b05436798 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,5 @@ docs/source/inference/examples/ # Static images !docs/source/_static/images/**/*.png +!comfyui/assets/**/*.png +!comfyui/assets/**/*.gif diff --git a/comfyui/README.md b/comfyui/README.md new file mode 100644 index 000000000..aa641593e --- /dev/null +++ b/comfyui/README.md @@ -0,0 +1,138 @@ +# ComfyUI-FastVideo + +A custom node suite for ComfyUI that provides accelerated video generation using [FastVideo](https://github.com/hao-ai-labs/FastVideo). See the [blog post](https://hao-ai-lab.github.io/blogs/fastvideo/) about FastVideo V1 to learn more. + +## Multi-GPU Parallel Inference + +One of the key features ComfyUI-FastVideo brings to ComfyUI is its ability to distribute the generation workload across multiple GPUs, resulting in significantly faster inference times. + +![Wan2.1-I2V-14B-480P-Diffusers](./assets/wani2v.gif) + +Example of Wan2.1-I2V-14B-480P-Diffusers model running on 4 GPUs. +## Features + +- Generate high-quality videos from text prompts and images +- Configurable video parameters (prompt, resolution, frame count, FPS) +- Support for multiple GPUs with tensor and sequence parallelism +- Advanced configuration options for VAE, Text Encoder, and DIT components +- Interruption/cancellation support for long-running generations + +## Installation + +### Requirements + +- [ComfyUI](https://github.com/comfyanonymous/ComfyUI) +- CUDA-capable GPU(s) with sufficient VRAM + +### Install using ComfyUI Manager + +Coming soon! + +### Manual Installation + +#### Copy the FastVideo `comfyui` directory into your ComfyUI custom_nodes directory: + +```bash +cp -r /path/to/FastVideo/comfyui /path/to/ComfyUI/custom_nodes/FastVideo +``` + +#### Install dependencies: + +Currently, the only dependency is `fastvideo`, which can be installed using pip. + +```bash +pip install fastvideo +``` + +#### Install missing custom nodes: + +`ComfyUI-VideoHelperSuite`: + +```bash +cd /path/to/ComfyUI/custom_nodes +git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git +``` + +If you're seeing `ImportError: libGL.so.1: cannot open shared object file: No such file or directory`, +you may need to install ffmpeg + +```bash +apt-get update && apt-get install ffmpeg +``` + +## Usage + +After installation, the following nodes will be available in the ComfyUI interface under the "fastvideo" category: + +- **Video Generator**: The main node for generating videos from prompts +- **Inference Args**: Configure video generation parameters +- **VAE Config** +- **Text Encoder Config** +- **DIT Config** +- **Load Image Path**: Load images for potential conditioning + +You may have noticed many arguments on the nodes have 'auto' as the default value. This is because FastVideo will automatically detect the best values for these parameters based on the model and the hardware. However, you can also manually configure these parameters to get the best performance for your specific use case. We plan on releasing more optimized workflow files for different models and hardware configurations in the future. + +You can see what some of the default configurations are by looking at the FastVideo repo: +- [Wan2.1-I2V-14B-480P-Diffusers](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/v1/configs/wan_14B_i2v_480p_pipeline.json) +- [FastHunyuan-diffusers](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/v1/configs/fasthunyuan_t2v.json) + +### Node Configuration + +#### Video Generator + +- **prompt**: Text description of the video to generate +- **output_path**: Directory where generated videos will be saved +- **num_gpus**: Number of GPUs to use for generation +- **model_path**: Path to the FastVideo model +- **embedded_cfg_scale**: Classifier-free guidance scale +- **sp_size**: Sequence parallelism size (usually should match num_gpus) +- **tp_size**: Tensor parallelism size (usually should match num_gpus) +- **precision**: Model precision (fp16 or bf16) + +`model_path takes either a model id from huggingface or a local path to a model. Models by default will be downloaded to ~/.cache/huggingface/hub/ and cached for subsequent runs.` + +#### Inference Args + +- **height/width**: Resolution of the output video +- **num_frames**: Number of frames to generate +- **num_inference_steps**: Number of diffusion steps per frame +- **guidance_scale**: Classifier-free guidance scale +- **flow_shift**: Frame flow shift parameter +- **seed**: Random seed for reproducible generation +- **fps**: Frames per second of the output video +- **image_path**: Optional path to input image for conditioning (for i2v models) + +## Memory Management + +Models will remain loaded in GPU memory between runs when you only change inference arguments (such as prompt, resolution, frame count, FPS, guidance scale, etc.) or the prompt text. This allows for faster subsequent generations since the model doesn't need to be reloaded. + +However, if you need to change the following parameters, you will need to restart the ComfyUI server: +- **Number of GPUs** (`num_gpus`) +- **Model path** (`model_path`) +- **Tensor parallelism size** (`tp_size`) +- **Sequence parallelism size** (`sp_size`) + +These parameters affect the model's distribution across GPUs and require a complete reinitialization of the model pipeline. + +## Example workflows + +### Text to Video + +FastVideo-FastHunyuan-diffusers + +![FastVideo-FastHunyuan-diffusers](./assets/fasthunyuan.png) + +- [FastHunyuan-diffusers.json](./examples/FastHunyuan-diffusers.json) + +### Image to Video + +Wan2.1-I2V-14B-480P-Diffusers + +![Wan2.1-I2V-14B-480P-Diffusers](./assets/wani2v.png) + +- [Wan2.1-I2V-14B-480P-Diffusers.json](./examples/Wan2.1-I2V-14B-480P-Diffusers.json) + +## License + +This project is licensed under Apache 2.0. diff --git a/comfyui/__init__.py b/comfyui/__init__.py new file mode 100644 index 000000000..258ca1fb1 --- /dev/null +++ b/comfyui/__init__.py @@ -0,0 +1,5 @@ +from .video_generator.nodes import (NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS) + +WEB_DIRECTORY = "./web" +__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY'] \ No newline at end of file diff --git a/comfyui/assets/fasthunyuan.png b/comfyui/assets/fasthunyuan.png new file mode 100644 index 000000000..feddc85f6 Binary files /dev/null and b/comfyui/assets/fasthunyuan.png differ diff --git a/comfyui/assets/wani2v.gif b/comfyui/assets/wani2v.gif new file mode 100644 index 000000000..d8ffd3d9c Binary files /dev/null and b/comfyui/assets/wani2v.gif differ diff --git a/comfyui/assets/wani2v.png b/comfyui/assets/wani2v.png new file mode 100644 index 000000000..e308d3072 Binary files /dev/null and b/comfyui/assets/wani2v.png differ diff --git a/comfyui/examples/FastHunyuan-diffusers.json b/comfyui/examples/FastHunyuan-diffusers.json new file mode 100644 index 000000000..b5731ee1e --- /dev/null +++ b/comfyui/examples/FastHunyuan-diffusers.json @@ -0,0 +1,645 @@ +{ + "id": "23a1f065-bbba-4a8f-b144-944e1318fcbf", + "revision": 0, + "last_node_id": 8, + "last_link_id": 7, + "nodes": [ + { + "id": 4, + "type": "VAEConfig", + "pos": [ + 374.2159423828125, + 554.85888671875 + ], + "size": [ + 334.080078125, + 322 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae_config", + "type": "VAE_CONFIG", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "VAEConfig" + }, + "widgets_values": [ + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999 + ], + "auto_widget_states": { + "load_encoder": { + "isAuto": true, + "value": -99999, + "cachedValue": true + }, + "load_decoder": { + "isAuto": true, + "value": -99999, + "cachedValue": true + }, + "tile_sample_min_height": { + "isAuto": true, + "value": -99999, + "cachedValue": 256 + }, + "tile_sample_min_width": { + "isAuto": true, + "value": -99999, + "cachedValue": 256 + }, + "tile_sample_min_num_frames": { + "isAuto": true, + "value": -99999, + "cachedValue": 16 + }, + "tile_sample_stride_height": { + "isAuto": true, + "value": -99999, + "cachedValue": 192 + }, + "tile_sample_stride_width": { + "isAuto": true, + "value": -99999, + "cachedValue": 192 + }, + "tile_sample_stride_num_frames": { + "isAuto": true, + "value": -99999, + "cachedValue": 12 + }, + "blend_num_frames": { + "isAuto": true, + "value": -99999, + "cachedValue": 0 + }, + "use_tiling": { + "isAuto": true, + "value": -99999, + "cachedValue": true + }, + "use_temporal_tiling": { + "isAuto": true, + "value": -99999, + "cachedValue": true + }, + "use_parallel_tiling": { + "isAuto": true, + "value": -99999, + "cachedValue": true + } + } + }, + { + "id": 5, + "type": "TextEncoderConfig", + "pos": [ + 416.4937744140625, + 953.6171875 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder_config", + "type": "TEXT_ENCODER_CONFIG", + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "TextEncoderConfig" + }, + "widgets_values": [ + -99999, + -99999, + -99999 + ], + "auto_widget_states": { + "prefix": { + "isAuto": true, + "value": -99999, + "cachedValue": "" + }, + "quant_config": { + "isAuto": true, + "value": -99999, + "cachedValue": "" + }, + "lora_config": { + "isAuto": true, + "value": -99999, + "cachedValue": "" + } + } + }, + { + "id": 6, + "type": "DITConfig", + "pos": [ + 415.1928405761719, + 1154.1573486328125 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "dit_config", + "type": "DIT_CONFIG", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "DITConfig" + }, + "widgets_values": [ + -99999, + -99999 + ], + "auto_widget_states": { + "prefix": { + "isAuto": true, + "value": -99999, + "cachedValue": "" + }, + "quant_config": { + "isAuto": true, + "value": -99999, + "cachedValue": "" + } + } + }, + { + "id": 1, + "type": "VideoGenerator", + "pos": [ + 818.804931640625, + 348.9299621582031 + ], + "size": [ + 400, + 436 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "inference_args", + "shape": 7, + "type": "INFERENCE_ARGS", + "link": 3 + }, + { + "name": "vae_config", + "shape": 7, + "type": "VAE_CONFIG", + "link": 2 + }, + { + "name": "text_encoder_config", + "shape": 7, + "type": "TEXT_ENCODER_CONFIG", + "link": 7 + }, + { + "name": "dit_config", + "shape": 7, + "type": "DIT_CONFIG", + "link": 6 + } + ], + "outputs": [ + { + "name": "video_path", + "type": "STRING", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VideoGenerator" + }, + "widgets_values": [ + "A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes wide with interest. The playful yet serene atmosphere is complemented by soft natural light filtering through the petals. Mid-shot, warm and cheerful tones.", + "/workspace/ComfyUI/outputs_video/", + 2, + "FastVideo/FastHunyuan-diffusers", + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999, + -99999 + ], + "auto_widget_states": { + "embedded_cfg_scale": { + "isAuto": true, + "value": -99999, + "cachedValue": 6 + }, + "sp_size": { + "isAuto": true, + "value": -99999, + "cachedValue": 2 + }, + "tp_size": { + "isAuto": true, + "value": -99999, + "cachedValue": 2 + }, + "vae_precision": { + "isAuto": true, + "value": -99999, + "cachedValue": "fp16" + }, + "vae_tiling": { + "isAuto": true, + "value": -99999, + "cachedValue": true + }, + "vae_sp": { + "isAuto": true, + "value": -99999, + "cachedValue": true + }, + "text_encoder_precision": { + "isAuto": true, + "value": -99999, + "cachedValue": "fp16" + }, + "precision": { + "isAuto": true, + "value": -99999, + "cachedValue": "fp16" + }, + "use_cpu_offload": { + "isAuto": true, + "value": -99999, + "cachedValue": true + } + } + }, + { + "id": 3, + "type": "VHS_LoadVideoPath", + "pos": [ + 1350.136962890625, + 331.20361328125 + ], + "size": [ + 231.8896484375, + 286 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "video", + "type": "STRING", + "widget": { + "name": "video" + }, + "link": 4 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "Wan", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "", + "type": "path", + "format": "video/", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 2, + "type": "InferenceArgs", + "pos": [ + 411.46307373046875, + 178.18182373046875 + ], + "size": [ + 278.73828125, + 298 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "inference_args", + "type": "INFERENCE_ARGS", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "InferenceArgs" + }, + "widgets_values": [ + 720, + 1280, + 45, + 6, + -99999, + -99999, + 1025, + "fixed", + 24, + -99999, + -99999 + ], + "auto_widget_states": { + "height": { + "isAuto": false, + "value": 720, + "cachedValue": 720 + }, + "width": { + "isAuto": false, + "value": 1280, + "cachedValue": 1280 + }, + "num_frames": { + "isAuto": false, + "value": 45, + "cachedValue": 45 + }, + "num_inference_steps": { + "isAuto": false, + "value": 6, + "cachedValue": 6 + }, + "guidance_scale": { + "isAuto": true, + "value": -99999, + "cachedValue": 1 + }, + "flow_shift": { + "isAuto": true, + "value": -99999, + "cachedValue": 17 + }, + "seed": { + "isAuto": false, + "value": 1025, + "cachedValue": 1024 + }, + "fps": { + "isAuto": false, + "value": 24, + "cachedValue": 24 + }, + "image_path": { + "isAuto": true, + "value": -99999, + "cachedValue": "X://insert/path/here.mp4" + }, + "enable_teacache": { + "isAuto": true, + "value": -99999, + "cachedValue": true + } + } + }, + { + "id": 8, + "type": "VHS_VideoCombine", + "pos": [ + 1668.3499755859375, + 328.22625732421875 + ], + "size": [ + 507.507080078125, + 622.2227172851562 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 5 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "._00003.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 24, + "workflow": "._00003.png", + "fullpath": "/workspace/ComfyUI/temp/._00003.mp4" + } + } + } + } + ], + "links": [ + [ + 2, + 4, + 0, + 1, + 1, + "VAE_CONFIG" + ], + [ + 3, + 2, + 0, + 1, + 0, + "INFERENCE_ARGS" + ], + [ + 4, + 1, + 0, + 3, + 2, + "STRING" + ], + [ + 5, + 3, + 0, + 8, + 0, + "IMAGE" + ], + [ + 6, + 6, + 0, + 1, + 3, + "DIT_CONFIG" + ], + [ + 7, + 5, + 0, + 1, + 2, + "TEXT_ENCODER_CONFIG" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.9090909090909091, + "offset": [ + 112.86678372727341, + -71.45635903989245 + ] + }, + "frontendVersion": "1.20.4", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/examples/Wan2.1-I2V-14B-480P-Diffusers.json b/comfyui/examples/Wan2.1-I2V-14B-480P-Diffusers.json new file mode 100644 index 000000000..e5a7087ee --- /dev/null +++ b/comfyui/examples/Wan2.1-I2V-14B-480P-Diffusers.json @@ -0,0 +1,697 @@ +{ + "id": "23a1f065-bbba-4a8f-b144-944e1318fcbf", + "revision": 0, + "last_node_id": 8, + "last_link_id": 7, + "nodes": [ + { + "id": 7, + "type": "LoadImagePath", + "pos": [ + 33.15385437011719, + 191.2037353515625 + ], + "size": [ + 270, + 334 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "image_path", + "type": "STRING", + "links": [ + 1 + ] + }, + { + "name": "IMAGE", + "type": "IMAGE", + "links": null + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImagePath" + }, + "widgets_values": [ + "woman.jpg", + "image" + ] + }, + { + "id": 3, + "type": "VHS_LoadVideoPath", + "pos": [ + 1350.136962890625, + 331.20361328125 + ], + "size": [ + 231.8896484375, + 286 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "video", + "type": "STRING", + "widget": { + "name": "video" + }, + "link": 4 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideoPath" + }, + "widgets_values": { + "video": "", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "Wan", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "", + "type": "path", + "format": "video/", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 8, + "type": "VHS_VideoCombine", + "pos": [ + 1668.3499755859375, + 328.22625732421875 + ], + "size": [ + 214.7587890625, + 334 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 5 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": {} + } + } + }, + { + "id": 4, + "type": "VAEConfig", + "pos": [ + 374.2159423828125, + 554.85888671875 + ], + "size": [ + 334.080078125, + 322 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae_config", + "type": "VAE_CONFIG", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "VAEConfig" + }, + "widgets_values": [ + true, + true, + 256, + 256, + 16, + 192, + 192, + 12, + 0, + true, + true, + true + ], + "auto_widget_states": { + "load_encoder": { + "isAuto": true, + "value": true, + "cachedValue": true + }, + "load_decoder": { + "isAuto": true, + "value": true, + "cachedValue": true + }, + "tile_sample_min_height": { + "isAuto": true, + "value": 256, + "cachedValue": 256 + }, + "tile_sample_min_width": { + "isAuto": true, + "value": 256, + "cachedValue": 256 + }, + "tile_sample_min_num_frames": { + "isAuto": true, + "value": 16, + "cachedValue": 16 + }, + "tile_sample_stride_height": { + "isAuto": true, + "value": 192, + "cachedValue": 192 + }, + "tile_sample_stride_width": { + "isAuto": true, + "value": 192, + "cachedValue": 192 + }, + "tile_sample_stride_num_frames": { + "isAuto": true, + "value": 12, + "cachedValue": 12 + }, + "blend_num_frames": { + "isAuto": true, + "value": 0, + "cachedValue": 0 + }, + "use_tiling": { + "isAuto": true, + "value": true, + "cachedValue": true + }, + "use_temporal_tiling": { + "isAuto": true, + "value": true, + "cachedValue": true + }, + "use_parallel_tiling": { + "isAuto": true, + "value": true, + "cachedValue": true + } + } + }, + { + "id": 2, + "type": "InferenceArgs", + "pos": [ + 411.46307373046875, + 178.18182373046875 + ], + "size": [ + 278.73828125, + 298 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "image_path", + "shape": 7, + "type": "STRING", + "widget": { + "name": "image_path" + }, + "link": 1 + } + ], + "outputs": [ + { + "name": "inference_args", + "type": "INFERENCE_ARGS", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "InferenceArgs" + }, + "widgets_values": [ + 832, + 480, + 45, + 20, + 1, + 17, + 1024, + "fixed", + 24, + "X://insert/path/here.mp4", + true + ], + "auto_widget_states": { + "height": { + "isAuto": false, + "value": 832, + "cachedValue": 720 + }, + "width": { + "isAuto": false, + "value": 480, + "cachedValue": 1280 + }, + "num_frames": { + "isAuto": false, + "value": 45, + "cachedValue": 45 + }, + "num_inference_steps": { + "isAuto": false, + "value": 20, + "cachedValue": 6 + }, + "guidance_scale": { + "isAuto": true, + "value": 1, + "cachedValue": 1 + }, + "flow_shift": { + "isAuto": true, + "value": 17, + "cachedValue": 17 + }, + "seed": { + "isAuto": false, + "value": 1024, + "cachedValue": 1024 + }, + "fps": { + "isAuto": false, + "value": 24, + "cachedValue": 24 + }, + "image_path": { + "isAuto": true, + "value": "X://insert/path/here.mp4", + "cachedValue": "X://insert/path/here.mp4" + }, + "enable_teacache": { + "isAuto": true, + "value": true, + "cachedValue": true + } + } + }, + { + "id": 1, + "type": "VideoGenerator", + "pos": [ + 818.804931640625, + 348.9299621582031 + ], + "size": [ + 400, + 436 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "inference_args", + "shape": 7, + "type": "INFERENCE_ARGS", + "link": 3 + }, + { + "name": "vae_config", + "shape": 7, + "type": "VAE_CONFIG", + "link": 2 + }, + { + "name": "text_encoder_config", + "shape": 7, + "type": "TEXT_ENCODER_CONFIG", + "link": 7 + }, + { + "name": "dit_config", + "shape": 7, + "type": "DIT_CONFIG", + "link": 6 + } + ], + "outputs": [ + { + "name": "video_path", + "type": "STRING", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VideoGenerator" + }, + "widgets_values": [ + "A woman crying from laughter.", + "/workspace/ComfyUI/outputs_video/", + 4, + "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + 6, + 2, + 2, + "fp16", + true, + true, + "fp16", + "fp16", + true + ], + "auto_widget_states": { + "embedded_cfg_scale": { + "isAuto": true, + "value": 6, + "cachedValue": 6 + }, + "sp_size": { + "isAuto": true, + "value": 2, + "cachedValue": 2 + }, + "tp_size": { + "isAuto": true, + "value": 2, + "cachedValue": 2 + }, + "vae_precision": { + "isAuto": true, + "value": "fp16", + "cachedValue": "fp16" + }, + "vae_tiling": { + "isAuto": true, + "value": true, + "cachedValue": true + }, + "vae_sp": { + "isAuto": true, + "value": true, + "cachedValue": true + }, + "text_encoder_precision": { + "isAuto": true, + "value": "fp16", + "cachedValue": "fp16" + }, + "precision": { + "isAuto": true, + "value": "fp16", + "cachedValue": "fp16" + }, + "use_cpu_offload": { + "isAuto": true, + "value": true, + "cachedValue": true + } + } + }, + { + "id": 5, + "type": "TextEncoderConfig", + "pos": [ + 416.4937744140625, + 953.6171875 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder_config", + "type": "TEXT_ENCODER_CONFIG", + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "TextEncoderConfig" + }, + "widgets_values": [ + "", + "", + "" + ], + "auto_widget_states": { + "prefix": { + "isAuto": true, + "value": "", + "cachedValue": "" + }, + "quant_config": { + "isAuto": true, + "value": "", + "cachedValue": "" + }, + "lora_config": { + "isAuto": true, + "value": "", + "cachedValue": "" + } + } + }, + { + "id": 6, + "type": "DITConfig", + "pos": [ + 415.1928405761719, + 1154.1573486328125 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "dit_config", + "type": "DIT_CONFIG", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "DITConfig" + }, + "widgets_values": [ + "", + "" + ], + "auto_widget_states": { + "prefix": { + "isAuto": true, + "value": "", + "cachedValue": "" + }, + "quant_config": { + "isAuto": true, + "value": "", + "cachedValue": "" + } + } + } + ], + "links": [ + [ + 1, + 7, + 0, + 2, + 0, + "STRING" + ], + [ + 2, + 4, + 0, + 1, + 1, + "VAE_CONFIG" + ], + [ + 3, + 2, + 0, + 1, + 0, + "INFERENCE_ARGS" + ], + [ + 4, + 1, + 0, + 3, + 2, + "STRING" + ], + [ + 5, + 3, + 0, + 8, + 0, + "IMAGE" + ], + [ + 6, + 6, + 0, + 1, + 3, + "DIT_CONFIG" + ], + [ + 7, + 5, + 0, + 1, + 2, + "TEXT_ENCODER_CONFIG" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 646.7950212991898, + 66.17259910028655 + ] + }, + "frontendVersion": "1.20.4", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/video_generator/dit_config.py b/comfyui/video_generator/dit_config.py new file mode 100644 index 000000000..fc0157418 --- /dev/null +++ b/comfyui/video_generator/dit_config.py @@ -0,0 +1,31 @@ +class DITConfig: + + @classmethod + def INPUT_TYPES(cls): + return { + "optional": { + "prefix": ("STRING", { + "default": "" + }), + "quant_config": ("STRING", { + "default": "" + }), + } + } + + @classmethod + def VALIDATE_INPUTS(cls, **kwargs): + return True + + RETURN_TYPES = ("DIT_CONFIG", ) + RETURN_NAMES = ("dit_config", ) + FUNCTION = "set_args" + CATEGORY = "fastvideo" + + def set_args(self, prefix, quant_config): + raw_args = {"prefix": prefix, "quant_config": quant_config} + + # Filter out keys where value is -99999 + args = {k: v for k, v in raw_args.items() if str(int(v)) != str(-99999)} + + return (args, ) diff --git a/comfyui/video_generator/inference_args.py b/comfyui/video_generator/inference_args.py new file mode 100644 index 000000000..2ba46157e --- /dev/null +++ b/comfyui/video_generator/inference_args.py @@ -0,0 +1,89 @@ +class InferenceArgs: + + @classmethod + def INPUT_TYPES(cls): + return { + "optional": { + "height": ("INT", { + "default": 720 + }), + "width": ("INT", { + "default": 1280 + }), + "num_frames": ("INT", { + "default": 45 + }), + "num_inference_steps": ("INT", { + "default": 6 + }), + "guidance_scale": ("FLOAT", { + "default": 1.0 + }), + "flow_shift": ("INT", { + "default": 17 + }), + "seed": ("INT", { + "default": 1024 + }), + "fps": ("INT", { + "default": 24 + }), + "image_path": ("STRING", { + "default": "X://insert/path/here.mp4" + }), + "enable_teacache": ([True, False], { + "default": False + }), + } + } + + @classmethod + def VALIDATE_INPUTS(cls, **kwargs): + return True + + RETURN_TYPES = ("INFERENCE_ARGS", ) + RETURN_NAMES = ("inference_args", ) + FUNCTION = "set_args" + CATEGORY = "fastvideo" + + def set_args( + self, + height, + width, + num_frames, + num_inference_steps, + guidance_scale, + flow_shift, + seed, + fps, + image_path, + enable_teacache, + ): + raw_args = { + "height": height, + "width": width, + "num_frames": num_frames, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "flow_shift": flow_shift, + "seed": seed, + "fps": fps, + "image_path": image_path, + "enable_teacache": enable_teacache, + } + + # Filter out keys where value is -99999, handling different types properly + args = {} + for k, v in raw_args.items(): + try: + if isinstance(v, str): + if v != "-99999": + args[k] = v + elif v != -99999: + # If it's not a string, compare directly + args[k] = v + except (ValueError, TypeError): + # Include any value that causes an error in comparison + args[k] = v + + return (args, ) diff --git a/comfyui/video_generator/load_image.py b/comfyui/video_generator/load_image.py new file mode 100644 index 000000000..54556fb34 --- /dev/null +++ b/comfyui/video_generator/load_image.py @@ -0,0 +1,104 @@ +import hashlib +import os +from typing import List + +import folder_paths +import numpy as np +import torch +from PIL import Image, ImageOps, ImageSequence + +from .node_helpers import pillow + + +class LoadImagePath: + + @classmethod + def INPUT_TYPES(s): + input_dir = folder_paths.get_input_directory() + files = [ + f for f in os.listdir(input_dir) + if os.path.isfile(os.path.join(input_dir, f)) + ] + files = folder_paths.filter_files_content_types(files, ["image"]) + return { + "required": { + "image": (sorted(files), { + "image_upload": True + }) + }, + } + + CATEGORY = "fastvideo" + + RETURN_TYPES = ("STRING", "IMAGE", "MASK") + RETURN_NAMES = ("image_path", "IMAGE", "MASK") + FUNCTION = "load_image" + + def load_image(self, image): + image_path = folder_paths.get_annotated_filepath(image) + + img = pillow(Image.open, image_path) + + output_images: List[torch.Tensor] = [] + output_masks: List[torch.Tensor] = [] + w, h = None, None + + excluded_formats = ['MPO'] + + for i in ImageSequence.Iterator(img): + processed_image = pillow(ImageOps.exif_transpose, i) + if processed_image is None: + continue + + if processed_image.mode == 'I': + processed_image = processed_image.point(lambda i: i * (1 / 255)) + image = processed_image.convert("RGB") + + if len(output_images) == 0: + w = image.size[0] + h = image.size[1] + + if image.size[0] != w or image.size[1] != h: + continue + + image = np.array(image).astype(np.float32) / 255.0 + image = torch.from_numpy(image)[ + None, + ] + if 'A' in processed_image.getbands(): + mask = np.array(processed_image.getchannel('A')).astype( + np.float32) / 255.0 + mask = 1. - torch.from_numpy(mask) + elif processed_image.mode == 'P' and 'transparency' in processed_image.info: + mask = np.array( + processed_image.convert('RGBA').getchannel('A')).astype( + np.float32) / 255.0 + mask = 1. - torch.from_numpy(mask) + else: + mask = torch.zeros((64, 64), dtype=torch.float32, device="cpu") + output_images.append(image) + output_masks.append(mask.unsqueeze(0)) + + if len(output_images) > 1 and img.format not in excluded_formats: + output_image = torch.cat(output_images, dim=0) + output_mask = torch.cat(output_masks, dim=0) + else: + output_image = output_images[0] + output_mask = output_masks[0] + + return (image_path, output_image, output_mask) + + @classmethod + def IS_CHANGED(s, image): + image_path = folder_paths.get_annotated_filepath(image) + m = hashlib.sha256() + with open(image_path, 'rb') as f: + m.update(f.read()) + return m.digest().hex() + + @classmethod + def VALIDATE_INPUTS(s, image): + if not folder_paths.exists_annotated_filepath(image): + return "Invalid image file: {}".format(image) + + return True diff --git a/comfyui/video_generator/node_helpers.py b/comfyui/video_generator/node_helpers.py new file mode 100644 index 000000000..7249c9efd --- /dev/null +++ b/comfyui/video_generator/node_helpers.py @@ -0,0 +1,68 @@ +import hashlib +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar + +import torch +from comfy.cli_args import args +from PIL import ImageFile, UnidentifiedImageError + +T = TypeVar('T') + + +def conditioning_set_values( + conditioning: List[Any], + values: Optional[Dict[str, Any]] = None) -> List[Any]: + if values is None: + values = {} + c = [] + for t in conditioning: + n = [t[0], t[1].copy()] + for k in values: + n[1][k] = values[k] + c.append(n) + + return c + + +def pillow(fn: Callable[[Any], T], arg: Any) -> T: + prev_value = None + try: + x = fn(arg) + except (OSError, UnidentifiedImageError, ValueError + ): #PIL issues #4472 and #2445, also fixes ComfyUI issue #3416 + prev_value = ImageFile.LOAD_TRUNCATED_IMAGES + ImageFile.LOAD_TRUNCATED_IMAGES = True + x = fn(arg) + finally: + if prev_value is not None: + ImageFile.LOAD_TRUNCATED_IMAGES = prev_value + return x + + +def hasher() -> Callable[[], Any]: + hashfuncs = { + "md5": hashlib.md5, + "sha1": hashlib.sha1, + "sha256": hashlib.sha256, + "sha512": hashlib.sha512 + } + return hashfuncs[args.default_hashing_function] + + +def string_to_torch_dtype(string: str) -> Optional[torch.dtype]: + if string == "fp32": + return torch.float32 + if string == "fp16": + return torch.float16 + if string == "bf16": + return torch.bfloat16 + return None + + +def image_alpha_fix(destination: torch.Tensor, + source: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + if destination.shape[-1] < source.shape[-1]: + source = source[..., :destination.shape[-1]] + elif destination.shape[-1] > source.shape[-1]: + destination = torch.nn.functional.pad(destination, (0, 1)) + destination[..., -1] = 1.0 + return destination, source diff --git a/comfyui/video_generator/nodes.py b/comfyui/video_generator/nodes.py new file mode 100644 index 000000000..6ed0e125e --- /dev/null +++ b/comfyui/video_generator/nodes.py @@ -0,0 +1,24 @@ +from .dit_config import DITConfig +from .inference_args import InferenceArgs +from .load_image import LoadImagePath +from .text_encoder_config import TextEncoderConfig +from .vae_config import VAEConfig +from .video_generator import VideoGenerator + +NODE_CLASS_MAPPINGS = { + "VideoGenerator": VideoGenerator, + "InferenceArgs": InferenceArgs, + "VAEConfig": VAEConfig, + "TextEncoderConfig": TextEncoderConfig, + "DITConfig": DITConfig, + "LoadImagePath": LoadImagePath +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "VideoGenerator": "Video Generator", + "InferenceArgs": "Inference Args", + "VAEConfig": "VAE Config", + "TextEncoderConfig": "Text Encoder Config", + "DITConfig": "DIT Config", + "LoadImagePath": "Load Image Path" +} \ No newline at end of file diff --git a/comfyui/video_generator/text_encoder_config.py b/comfyui/video_generator/text_encoder_config.py new file mode 100644 index 000000000..0ed9c1bd4 --- /dev/null +++ b/comfyui/video_generator/text_encoder_config.py @@ -0,0 +1,38 @@ +class TextEncoderConfig: + + @classmethod + def INPUT_TYPES(cls): + return { + "optional": { + "prefix": ("STRING", { + "default": "" + }), + "quant_config": ("STRING", { + "default": "" + }), + "lora_config": ("STRING", { + "default": "" + }), + } + } + + @classmethod + def VALIDATE_INPUTS(cls, **kwargs): + return True + + RETURN_TYPES = ("TEXT_ENCODER_CONFIG", ) + RETURN_NAMES = ("text_encoder_config", ) + FUNCTION = "set_args" + CATEGORY = "fastvideo" + + def set_args(self, prefix, quant_config, lora_config): + raw_args = { + "prefix": prefix, + "quant_config": quant_config, + "lora_config": lora_config + } + + # Filter out keys where value is -99999 + args = {k: v for k, v in raw_args.items() if str(int(v)) != str(-99999)} + + return (args, ) diff --git a/comfyui/video_generator/vae_config.py b/comfyui/video_generator/vae_config.py new file mode 100644 index 000000000..a20f22f40 --- /dev/null +++ b/comfyui/video_generator/vae_config.py @@ -0,0 +1,88 @@ +class VAEConfig: + + @classmethod + def INPUT_TYPES(cls): + return { + "optional": { + "load_encoder": ([True, False], { + "default": True + }), + "load_decoder": ([True, False], { + "default": True + }), + "tile_sample_min_height": ("INT", { + "default": 256 + }), + "tile_sample_min_width": ("INT", { + "default": 256 + }), + "tile_sample_min_num_frames": ("INT", { + "default": 16 + }), + "tile_sample_stride_height": ("INT", { + "default": 192 + }), + "tile_sample_stride_width": ("INT", { + "default": 192 + }), + "tile_sample_stride_num_frames": ("INT", { + "default": 12 + }), + "blend_num_frames": ("INT", { + "default": 0 + }), + "use_tiling": ([True, False], { + "default": True + }), + "use_temporal_tiling": ([True, False], { + "default": True + }), + "use_parallel_tiling": ([True, False], { + "default": True + }), + } + } + + @classmethod + def VALIDATE_INPUTS(cls, **kwargs): + return True + + RETURN_TYPES = ("VAE_CONFIG", ) + RETURN_NAMES = ("vae_config", ) + FUNCTION = "set_args" + CATEGORY = "fastvideo" + + def set_args( + self, + load_encoder, + load_decoder, + tile_sample_min_height, + tile_sample_min_width, + tile_sample_min_num_frames, + tile_sample_stride_height, + tile_sample_stride_width, + tile_sample_stride_num_frames, + blend_num_frames, + use_tiling, + use_temporal_tiling, + use_parallel_tiling, + ): + raw_args = { + "load_encoder": load_encoder, + "load_decoder": load_decoder, + "tile_sample_min_height": tile_sample_min_height, + "tile_sample_min_width": tile_sample_min_width, + "tile_sample_min_num_frames": tile_sample_min_num_frames, + "tile_sample_stride_height": tile_sample_stride_height, + "tile_sample_stride_width": tile_sample_stride_width, + "tile_sample_stride_num_frames": tile_sample_stride_num_frames, + "blend_num_frames": blend_num_frames, + "use_tiling": use_tiling, + "use_temporal_tiling": use_temporal_tiling, + "use_parallel_tiling": use_parallel_tiling, + } + + # Filter out any value explicitly set to -99999 + args = {k: v for k, v in raw_args.items() if str(int(v)) != str(-99999)} + + return (args, ) diff --git a/comfyui/video_generator/video_generator.py b/comfyui/video_generator/video_generator.py new file mode 100644 index 000000000..87bf6d945 --- /dev/null +++ b/comfyui/video_generator/video_generator.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import glob +import os +import signal +import sys +import threading +import time +from typing import Any + +from comfy.model_management import processing_interrupted + +from fastvideo import PipelineConfig +from fastvideo import VideoGenerator as FastVideoGenerator + +sys.path.insert( + 0, + os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) + + +# Custom exception for interruption +class GenerationInterruptedException(Exception): + pass + + +# Custom exception for interruption that ComfyUI will recognize +class GenerationCancelledException(Exception): + + def __init__(self, + message: str = "Generation was cancelled by user") -> None: + self.message = message + super().__init__(self.message) + + +def update_config_from_args(config: Any, args_dict: dict[str, Any]) -> None: + """ + Update configuration object from arguments dictionary. + + Args: + config: The configuration object to update + args_dict: Dictionary containing arguments + """ + for key, value in args_dict.items(): + if hasattr(config, key) and value is not None: + if key == "text_encoder_precisions" and isinstance(value, list): + setattr(config, key, tuple(value)) + else: + setattr(config, key, value) + + +class VideoGenerator: + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ("STRING", { + "multiline": + True, + "default": + "A ripe orange tumbles gently from a tree and lands on the head of a lounging capybara, " + "who blinks slowly in response. The moment is quietly humorous and oddly serene, framed by " + "lush green foliage and dappled sunlight. Mid-shot, warm and whimsical tones." + }), + "output_path": ("STRING", { + "default": "/workspace/ComfyUI/outputs_video/" + }), + "num_gpus": ("INT", { + "default": 2, + "min": 1, + "max": 16 + }), + "model_path": ("STRING", { + "default": "FastVideo/FastHunyuan-diffusers" + }) + }, + "optional": { + "inference_args": ("INFERENCE_ARGS", ), + "embedded_cfg_scale": ("FLOAT", { + "default": 6.0 + }), + "sp_size": ("INT", { + "default": 2 + }), + "tp_size": ("INT", { + "default": 2 + }), + "vae_config": ("VAE_CONFIG", ), + "vae_precision": (["fp16", "bf16"], { + "default": "fp16" + }), + "vae_tiling": ([True, False], { + "default": True + }), + "vae_sp": ([True, False], { + "default": False + }), + "text_encoder_config": ("TEXT_ENCODER_CONFIG", ), + "text_encoder_precision": (["fp16", "bf16"], { + "default": "fp16" + }), + "dit_config": ("DIT_CONFIG", ), + "precision": (["fp16", "bf16"], { + "default": "fp16" + }), + "use_cpu_offload": ([True, False], { + "default": False + }), + } + } + + @classmethod + def VALIDATE_INPUTS(cls, **kwargs): + return True + + RETURN_TYPES = ("STRING", ) + RETURN_NAMES = ("video_path", ) + FUNCTION = "launch_inference" + CATEGORY = "fastvideo" + + generator: FastVideoGenerator | None = None + _interrupt_thread: threading.Thread | None = None + _generation_active: bool = False + _generation_interrupted: bool = False + _interrupt_event: threading.Event = threading.Event() + _generation_thread: threading.Thread | None = None + _generation_result: str | None = None + _generation_exception: Exception | None = None + + def _monitor_for_interruption(self): + """Background thread that monitors for interruption requests""" + time.sleep(2) # Give the generation thread time to send execute_forward + + while self._generation_active and not self._interrupt_event.is_set(): + if processing_interrupted(): + print("Video generation interrupted by user") + self._generation_interrupted = True + + # Try to send interrupt signal to worker processes + if self.generator is not None and hasattr( + self.generator, 'executor'): + try: + # The MultiprocExecutor has a workers attribute + if hasattr(self.generator.executor, 'workers'): + for worker in self.generator.executor.workers: + if worker.is_alive(): + os.kill(worker.pid, signal.SIGINT) + print("Interrupt signal sent to worker processes") + except Exception as e: + print(f"Error sending interrupt signal: {e}") + + # Set the interrupt event to notify other threads + self._interrupt_event.set() + break + time.sleep(0.5) + + def _run_generation(self, prompt: str, output_path: str, + inference_args: dict[str, Any]) -> None: + """Thread function to run the generation""" + try: + if self.generator is not None: + self.generator.generate_video(prompt=prompt, + output_path=output_path, + **inference_args) + self._generation_result = os.path.join(output_path, + f"{prompt[:100]}.mp4") + else: + raise RuntimeError("Generator is not initialized") + except Exception as e: + self._generation_exception = e + self._interrupt_event.set() + + def load_output_video(self, output_dir): + video_extensions = ["*.mp4", "*.avi", "*.mov", "*.mkv"] + video_files = [] + + for ext in video_extensions: + video_files.extend(glob.glob(os.path.join(output_dir, ext))) + + if not video_files: + print("No video files found in output directory: %s", output_dir) + return "" + + video_files.sort() + return video_files[0] + + def launch_inference( + self, + prompt, + output_path, + num_gpus, + model_path, + embedded_cfg_scale, + sp_size, + tp_size, + vae_precision, + vae_tiling, + vae_sp, + text_encoder_precision, + precision, + inference_args=None, + vae_config=None, + text_encoder_config=None, + dit_config=None, + use_cpu_offload=None, + ): + print('Running FastVideo inference') + + # Reset interruption flag and event + self._generation_interrupted = False + self._interrupt_event.clear() + self._generation_result = None + self._generation_exception = None + + # Load pipeline config from model path + pipeline_config = PipelineConfig.from_pretrained(model_path) + print('pipeline_config', pipeline_config) + + # Update configs with provided config dictionaries + if dit_config is not None: + update_config_from_args(pipeline_config.dit_config, dit_config) + + if vae_config is not None: + update_config_from_args(pipeline_config.vae_config, vae_config) + + if text_encoder_config is not None: + update_config_from_args(pipeline_config.text_encoder_configs, + text_encoder_config) + + # Update top-level pipeline config with remaining arguments + raw_pipeline_args = {} + if embedded_cfg_scale is not None: + raw_pipeline_args['embedded_cfg_scale'] = embedded_cfg_scale + if precision is not None: + raw_pipeline_args['precision'] = precision + if vae_precision is not None: + raw_pipeline_args['vae_precision'] = vae_precision + if vae_tiling is not None: + raw_pipeline_args['vae_tiling'] = vae_tiling + if vae_sp is not None: + raw_pipeline_args['vae_sp'] = vae_sp + if text_encoder_precision is not None: + raw_pipeline_args['text_encoder_precision'] = text_encoder_precision + + # Filter out any value explicitly set to -99999 (auto values) + pipeline_args = { + k: v + for k, v in raw_pipeline_args.items() if str(int(v)) != str(-99999) + } + + update_config_from_args(pipeline_config, pipeline_args) + + raw_generation_args = {} + if num_gpus is not None: + raw_generation_args['num_gpus'] = num_gpus + if tp_size is not None: + raw_generation_args['tp_size'] = tp_size + if sp_size is not None: + raw_generation_args['sp_size'] = sp_size + if use_cpu_offload is not None: + raw_generation_args['use_cpu_offload'] = use_cpu_offload + + generation_args = { + k: v + for k, v in raw_generation_args.items() + if str(int(v)) != str(-99999) + } + + if self.generator is None: + print('generation_args', generation_args) + print('pipeline_config', pipeline_config) + self.generator = FastVideoGenerator.from_pretrained( + model_path=model_path, + **generation_args, + pipeline_config=pipeline_config) + + print('inference_args', inference_args) + + # Start a thread to run the generation + self._generation_thread = threading.Thread(target=self._run_generation, + args=(prompt, output_path, + inference_args), + daemon=True) + self._generation_thread.start() + + # Start a background thread to monitor for interruptions + self._generation_active = True + self._interrupt_thread = threading.Thread( + target=self._monitor_for_interruption, daemon=True) + self._interrupt_thread.start() + + # Wait for either completion or interruption + while self._generation_thread.is_alive( + ) and not self._interrupt_event.is_set(): + self._generation_thread.join(timeout=0.5) + + self._generation_active = False + if self._interrupt_thread: + self._interrupt_thread.join(timeout=1.0) + self._interrupt_thread = None + + if self._generation_interrupted: + print("Video generation was cancelled by user") + raise GenerationCancelledException() + elif self._generation_exception: + # Re-raise the exception from the generation thread + raise self._generation_exception + elif self._generation_result: + return (self._generation_result, ) + else: + # This shouldn't happen, but just in case + print("Generation completed but no result was produced") + raise Exception("Generation failed to produce a result") diff --git a/comfyui/web/js/FastVideo.core.js b/comfyui/web/js/FastVideo.core.js new file mode 100644 index 000000000..6be0dcbc5 --- /dev/null +++ b/comfyui/web/js/FastVideo.core.js @@ -0,0 +1,593 @@ +import { app } from '../../../scripts/app.js' + +function chainCallback(object, property, callback) { + if (object == undefined) { + console.error("Tried to add callback to non-existent object"); + return; + } + if (property in object && object[property]) { + const callback_orig = object[property]; + object[property] = function () { + const r = callback_orig.apply(this, arguments); + return callback.apply(this, arguments) ?? r; + }; + } else { + object[property] = callback; + } +} + +function drawAutoAnnotated(ctx, node, widget_width, y, H) { + const litegraph_base = LiteGraph; + const show_text = app.canvas.ds.scale >= 0.5; + const margin = 15; + + const autoTextWidth = 30; + const autoTextRightMargin = 5; + + ctx.textAlign = 'left'; + ctx.strokeStyle = litegraph_base.WIDGET_OUTLINE_COLOR; + ctx.fillStyle = litegraph_base.WIDGET_BGCOLOR; + + ctx.beginPath(); + if (show_text && ctx.roundRect) { + ctx.roundRect(margin, y, widget_width - margin * 2, H, [H * 0.5]); + } else { + ctx.rect(margin, y, widget_width - margin * 2, H); + } + ctx.fill(); + + if (show_text) { + if (!this.disabled) ctx.stroke(); + const isAuto = this.isAuto === true; + + ctx.save(); + if (isAuto) { + ctx.fillStyle = litegraph_base.WIDGET_TEXT_COLOR; + ctx.strokeStyle = litegraph_base.WIDGET_TEXT_COLOR; + } else { + ctx.fillStyle = litegraph_base.WIDGET_SECONDARY_TEXT_COLOR; + ctx.strokeStyle = litegraph_base.WIDGET_SECONDARY_TEXT_COLOR; + } + + // Position for the cog + const cogX = widget_width - autoTextRightMargin - autoTextWidth - 6; + const cogY = y + H * 0.5; + const cogRadius = 6; + const toothLength = 2; + const numTeeth = 8; + const holeRadius = 2; // Radius of the center hole + + // Draw the cog + ctx.beginPath(); + ctx.arc(cogX, cogY, cogRadius - toothLength, 0, Math.PI * 2); + ctx.fill(); + + // Draw the center hole (by clearing it) + ctx.beginPath(); + ctx.arc(cogX, cogY, holeRadius, 0, Math.PI * 2); + ctx.fillStyle = litegraph_base.WIDGET_BGCOLOR; + ctx.fill(); + + // Reset fill style for the teeth + if (isAuto) { + ctx.fillStyle = litegraph_base.WIDGET_TEXT_COLOR; + } else { + ctx.fillStyle = litegraph_base.WIDGET_SECONDARY_TEXT_COLOR; + } + + // Draw teeth + ctx.beginPath(); + for (let i = 0; i < numTeeth; i++) { + const angle = (i / numTeeth) * Math.PI * 2; + const innerX = cogX + (cogRadius - toothLength) * Math.cos(angle); + const innerY = cogY + (cogRadius - toothLength) * Math.sin(angle); + const outerX = cogX + cogRadius * Math.cos(angle); + const outerY = cogY + cogRadius * Math.sin(angle); + + ctx.moveTo(innerX, innerY); + ctx.lineTo(outerX, outerY); + } + ctx.lineWidth = 2; + ctx.stroke(); + + ctx.restore(); + + // Draw label + ctx.fillStyle = litegraph_base.WIDGET_SECONDARY_TEXT_COLOR; + const label = this.label || this.name; + if (label != null) { + ctx.fillText(label, margin * 2 + 5, y + H * 0.7); + } + + // Draw value + ctx.textAlign = 'right'; + const text = isAuto ? "auto" : this.displayValue(); + ctx.fillStyle = isAuto ? litegraph_base.WIDGET_SECONDARY_TEXT_COLOR : litegraph_base.WIDGET_TEXT_COLOR; + ctx.fillText(text, widget_width - autoTextRightMargin - autoTextWidth - 15, y + H * 0.7); + + // Draw increment/decrement buttons if not in AUTO mode and not a string widget + if (!isAuto && !this.disabled && this.config[0] !== "FVAUTOSTRING") { + // Draw decrement button (left triangle) + ctx.fillStyle = litegraph_base.WIDGET_TEXT_COLOR; + ctx.beginPath(); + ctx.moveTo(margin + 16, y + 5); + ctx.lineTo(margin + 6, y + H * 0.5); + ctx.lineTo(margin + 16, y + H - 5); + ctx.fill(); + + // Draw increment button (right triangle) + ctx.beginPath(); + ctx.moveTo(widget_width - margin - 16, y + 5); + ctx.lineTo(widget_width - margin - 6, y + H * 0.5); + ctx.lineTo(widget_width - margin - 16, y + H - 5); + ctx.fill(); + } + } +} + +function mouseAutoAnnotated(event, [x, y], node) { + const widget_width = node.size[0]; + const margin = 15; + const H = 20; // Widget height + + const autoTextWidth = 30; + const autoTextRightMargin = 5; + + const cogRadius = 6; + + if (this.isAuto) { + if (event.type === "pointerup" || event.type === "mouseup") { + const cogX = widget_width - autoTextRightMargin - autoTextWidth - 6; + const cogLeftEdge = cogX - cogRadius; + const cogRightEdge = cogX + cogRadius; + + if (x > cogLeftEdge && x < cogRightEdge) { + this.isAuto = false; + this.value = this.cachedValue !== undefined ? this.cachedValue : (this.options.default || 0); + + if (this.callback) { + this.callback(this.value); + } + node.graph.setDirtyCanvas(true, false); + } + } + + // Block ALL events in auto mode except cog clicks + event.preventDefault?.(); + event.stopPropagation?.(); + event.stopImmediatePropagation?.(); + return true; // Always return true to indicate event was handled + } + + // Determine if clicking on increment/decrement buttons + const delta = this.config[0] === "FVAUTOSTRING" ? 0 : + (x < 40 ? -1 : x > widget_width - 48 ? 1 : 0); + + if (event.type === "pointerdown" || event.type === "mousedown") { + // ComfyUI appears to intercept pointerdown events, so this code path is never reached + console.log("pointerdown received (unexpected)"); + return false; + } else if (event.type === "pointerup" || event.type === "mouseup") { + // Stop event propagation to prevent double handling + event.preventDefault?.(); + event.stopPropagation?.(); + event.stopImmediatePropagation?.(); + + const cogX = widget_width - autoTextRightMargin - autoTextWidth - 6; + const cogLeftEdge = widget_width - autoTextRightMargin - autoTextWidth - 6 - cogRadius; + const cogRightEdge = widget_width - autoTextRightMargin - autoTextWidth - 6 + cogRadius; + + if (x > cogLeftEdge && x < cogRightEdge) { + this.isAuto = !this.isAuto; + + if (this.isAuto) { + this.cachedValue = this.value; + this.value = -99999; + } else { + this.value = this.cachedValue !== undefined ? this.cachedValue : (this.options.default || 0); + } + + if (this.callback) { + this.callback(this.value); + } + + node.graph.setDirtyCanvas(true, false); + return true; + } + + // If in auto mode and NOT clicking the cog, block all other interactions + if (this.isAuto) { + return true; + } + + // Handle increment/decrement buttons if not in auto mode + if (delta !== 0 && !this.isAuto) { + if (this.config[0] === "FVAUTOCOMBO") { + const options = this.options.values || []; + if (options.length === 0) return true; + + let currentIndex = -1; + for (let i = 0; i < options.length; i++) { + const optValue = typeof options[i] === 'object' ? options[i].value : options[i]; + if (optValue == this.value || String(optValue) === String(this.value)) { + currentIndex = i; + break; + } + } + + if (currentIndex === -1) { + currentIndex = 0; + } + + let newIndex = currentIndex + delta; + if (newIndex < 0) { + newIndex = options.length - 1; + } else if (newIndex >= options.length) { + newIndex = 0; + } + + const newOption = options[newIndex]; + this.value = typeof newOption === 'object' ? newOption.value : newOption; + + if (this.callback) { + this.callback(this.value); + } + + node.graph.setDirtyCanvas(true, false); + return true; + } else { + let v = parseFloat(this.value); + const increment = delta * 0.1 * (this.options.step || 1); + + v += increment; + + // Apply min/max constraints + if (this.options.min != null) { + v = Math.max(this.options.min, v); + } + if (this.options.max != null) { + v = Math.min(this.options.max, v); + } + + // Round to precision or to integer + if (this.config[0] === "FVAUTOINT") { + v = Math.round(v); + } else if (this.options.precision !== undefined) { + const precision = Math.pow(10, this.options.precision); + v = Math.round(v * precision) / precision; + } + + this.value = v; + + if (this.callback) { + this.callback(this.value); + } + + node.graph.setDirtyCanvas(true, false); + return true; + } + } + + if (delta === 0 && !this.isAuto) { + if (this.config[0] === "FVAUTOCOMBO") { + const options = this.options.values || []; + + // Create menu items + const menuItems = options.map(opt => { + const value = typeof opt === 'object' ? opt.value : opt; + const label = typeof opt === 'object' ? opt.label : opt.toString(); + + return { + content: label, + callback: () => { + this.value = value; + + if (this.callback) { + this.callback(this.value); + } + + node.graph.setDirtyCanvas(true, false); + } + }; + }); + + new LiteGraph.ContextMenu(menuItems, { + event: event, + title: null, + callback: null, + extra: node + }); + + return true; + } else if (this.config[0] === "FVAUTOSTRING") { + const d_callback = (v) => { + this.value = v; + + if (this.callback) { + this.callback(this.value); + } + + node.graph.setDirtyCanvas(true, false); + }; + + const dialog = app.canvas.prompt( + 'Value', + this.value, + d_callback, + event + ); + + return true; + } else { + // For numeric widgets, show input dialog + const d_callback = (v) => { + this.value = this.parseValue?.(v) ?? Number(v); + + // Apply min/max constraints + if (this.options.min != null) { + this.value = Math.max(this.options.min, this.value); + } + if (this.options.max != null) { + this.value = Math.min(this.options.max, this.value); + } + + // Round to precision or to integer + if (this.config[0] === "FVAUTOINT") { + this.value = Math.round(this.value); + } else if (this.options.precision !== undefined) { + const precision = Math.pow(10, this.options.precision); + this.value = Math.round(this.value * precision) / precision; + } + + if (this.callback) { + this.callback(this.value); + } + + node.graph.setDirtyCanvas(true, false); + }; + + const dialog = app.canvas.prompt( + 'Value', + this.value, + d_callback, + event + ); + + return true; + } + } + + return true; + } + + return false; +} + +function makeAutoAnnotated(widget, inputData) { + const original = { + callback: widget.callback, + type: widget.type, + value: widget.value + }; + + // Add AUTO properties to the widget + Object.assign(widget, { + type: "BOOLEAN", + draw: drawAutoAnnotated, + mouse: mouseAutoAnnotated, + onMouse: null, // Explicitly disable original onMouse handler + + isAuto: true, + cachedValue: widget.value, + config: inputData, + options: Object.assign({}, inputData[1], widget.options), + original: original, // Store original properties for reference + + // Disable other potential mouse handlers with no-op functions + onClick: function () { + return false; + }, + onPointerUp: function () { + return false; + }, + onPointerDown: function () { + return false; + }, + onMouseUp: function () { + return false; + }, + onMouseDown: function () { + return false; + }, + + computeSize(width) { + return [width, 20]; + }, + displayValue: function () { + if (this.config[0] === "FVAUTOINT") { + return Math.round(this.value).toString(); + } + if (this.config[0] === "FVAUTOCOMBO") { + return this.value; + } + if (this.config[0] === "FVAUTOSTRING") { + return this.value; + } + // For FLOAT values, check if it's actually an integer + if (Number.isInteger(this.value)) { + return this.value.toString(); + } + return this.value.toFixed(this.options.precision || 2); + }, + parseValue: function (v) { + if (this.config[0] === "FVAUTOSTRING") { + return v; + } + if (typeof v === "string") { + return parseFloat(v); + } + return v; + }, + serializeValue: function () { + // Return special value for AUTO mode + return this.isAuto ? -99999 : this.value; + }, + deserializeValue: function (data) { + if (data === -99999) { + this.isAuto = true; + this.value = -99999; + } else { + this.isAuto = false; + this.value = data; + this.cachedValue = data; + } + } + }); + + // Override callback to handle AUTO mode + widget.callback = function (v) { + if (this.isAuto) { + return; // Don't call the original callback in AUTO mode + } + const result = original.callback?.call(this, v); + return result; + }; + + // Override any potential click handlers + const originalOnClick = widget.onClick; + if (originalOnClick) { + widget.onClick = function (...args) { + if (this.isAuto) { + return false; + } + return originalOnClick.call(this, ...args); + }; + } + + return widget; +} + +app.registerExtension({ + name: "FastVideo.AutoWidgets", + + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (nodeData?.name == "VideoGenerator" || nodeData?.name === "InferenceArgs" || nodeData?.name === "VAEConfig" || + nodeData?.name === "TextEncoderConfig" || nodeData?.name === "DITConfig") { + // Add serialization support + chainCallback(nodeType.prototype, "onSerialize", function (info) { + if (!this.widgets) { + return; + } + // Ensure widgets_values exists + if (!info.widgets_values) { + info.widgets_values = {}; + } + + // Store AUTO widget states in a separate property + if (!info.auto_widget_states) { + info.auto_widget_states = {}; + } + + // Handle AUTO widgets specially + for (const w of this.widgets) { + if (w.type === "BOOLEAN" && w.isAuto !== undefined) { + // Store the serialized value (for Python node) + info.widgets_values[w.name] = w.serializeValue(); + + // Store the full state (for UI restoration) + info.auto_widget_states[w.name] = { + isAuto: w.isAuto, + value: w.value, + cachedValue: w.cachedValue + }; + } + } + }); + + // Add deserialization support + chainCallback(nodeType.prototype, "onConfigure", function (info) { + if (!this.widgets) { + return; + } + + // First, restore from widgets_values (for backward compatibility) + if (info.widgets_values && Array.isArray(info.widgets_values)) { + for (let i = 0; i < this.widgets.length && i < info.widgets_values.length; i++) { + const w = this.widgets[i]; + const value = info.widgets_values[i]; + + if (w.type === "BOOLEAN" && w.isAuto !== undefined) { + w.deserializeValue(value); + } + } + } + + // Then, restore full state if available + if (info.auto_widget_states) { + for (const w of this.widgets) { + if (w.type === "BOOLEAN" && w.isAuto !== undefined && w.name in info.auto_widget_states) { + const state = info.auto_widget_states[w.name]; + + w.isAuto = state.isAuto; + w.cachedValue = state.cachedValue; + w.value = state.isAuto ? -99999 : state.value; + + w.callback?.(w.value); + } + } + } + + // Force a redraw + this.graph?.setDirtyCanvas(true, true); + }); + + + // Override addInput to handle AUTO widgets + chainCallback(nodeType.prototype, "onNodeCreated", function () { + // Convert any existing widgets to AUTO widgets if needed + let new_widgets = []; + const intWidgetNames = ["sp_size", "tp_size", "height", "width", "num_frames", "num_inference_steps", "flow_shift", "seed", "fps", "scale_factor", + "tile_sample_min_height", "tile_sample_min_width", "tile_sample_min_num_frames", "tile_sample_stride_height", "tile_sample_stride_width", + "tile_sample_stride_num_frames", "blend_num_frames" + ] + const floatWidgetNames = ["embedded_cfg_scale", "guidance_scale"] + const comboWidgetNames = ["vae_tiling", "vae_precision", "vae_sp", "text_encoder_precision", "precision", + "load_encoder", "load_decoder", "use_tiling", "use_temporal_tiling", "use_parallel_tiling", "use_cpu_offload", "enable_teacache" + ] + const stringWidgetNames = ["prefix", "quant_config", "lora_config", "image_path"] + + if (this.widgets) { + for (let w of this.widgets) { + if (intWidgetNames.includes(w.name)) { + new_widgets.push(makeAutoAnnotated(w, ["FVAUTOINT", { "default": 0 }])); + } else if (floatWidgetNames.includes(w.name)) { + new_widgets.push(makeAutoAnnotated(w, ["FVAUTOFLOAT", { "default": 0 }])); + } else if (comboWidgetNames.includes(w.name)) { + new_widgets.push(makeAutoAnnotated(w, ["FVAUTOCOMBO", { "default": 0 }])); + } else if (stringWidgetNames.includes(w.name)) { + new_widgets.push(makeAutoAnnotated(w, ["FVAUTOSTRING", { "default": "" }])); + } else { + new_widgets.push(w); + } + } + this.widgets = new_widgets; + + const autoWidgets = this.widgets.filter(w => w.type === "BOOLEAN" && w.isAuto !== undefined); + } + + this.graph?.setDirtyCanvas(true, true); + }); + } + }, + + async init() { + // Force a redraw of all nodes when the extension initializes + if (app.graph) { + setTimeout(() => { + app.graph.setDirtyCanvas(true, true); + }, 1000); + } + } +}); + +console.log("FastVideo.core.js loaded");