Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 kevin314
|
e331806a78 | ||
|
kevin314
|
1d64150044 | ||
|
kevin314
|
ecd7339386 | ||
|
kevin314
|
f7e6fceef9 |
+2
-2
@@ -1,7 +1,7 @@
|
||||
try:
|
||||
from .comfyui.video_generator.nodes import (NODE_CLASS_MAPPINGS,
|
||||
NODE_DISPLAY_NAME_MAPPINGS)
|
||||
WEB_DIRECTORY = "./web"
|
||||
WEB_DIRECTORY = "./comfyui/web"
|
||||
__all__ = [
|
||||
'NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY'
|
||||
]
|
||||
@@ -9,7 +9,7 @@ except ImportError:
|
||||
# ComfyUI environment not available, skip comfyui imports
|
||||
NODE_CLASS_MAPPINGS = {}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {}
|
||||
WEB_DIRECTORY = "./web"
|
||||
WEB_DIRECTORY = "./comfyui/web"
|
||||
__all__ = [
|
||||
'NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY'
|
||||
]
|
||||
|
||||
+12
-3
@@ -74,6 +74,7 @@ After installation, the following nodes will be available in the ComfyUI interfa
|
||||
You may have noticed many arguments on the nodes have 'auto' as the default value. This is because FastVideo will automatically detect the best values for these parameters based on the model and the hardware. However, you can also manually configure these parameters to get the best performance for your specific use case. We plan on releasing more optimized workflow files for different models and hardware configurations in the future.
|
||||
|
||||
You can see what some of the default configurations are by looking at the FastVideo repo:
|
||||
-[Wan-AI_Wan2.2-TI2V-5B-Diffusers]()
|
||||
- [Wan2.1-I2V-14B-480P-Diffusers](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/configs/wan_14B_i2v_480p_pipeline.json)
|
||||
- [FastHunyuan-diffusers](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/configs/fasthunyuan_t2v.json)
|
||||
|
||||
@@ -117,19 +118,27 @@ These parameters affect the model's distribution across GPUs and require a compl
|
||||
|
||||
## Example workflows
|
||||
|
||||
### Text and Image to Video
|
||||
|
||||
Wan-AI/Wan2.2-TI2V-5B-Diffusers
|
||||
|
||||
|
||||
|
||||
- [Wan2.2-TI2V-5B-Diffusers](./examples/Wan-AI_Wan2.2-TI2V-5B-Diffusers.json)
|
||||
|
||||
### Text to Video
|
||||
|
||||
FastVideo-FastHunyuan-diffusers
|
||||
|
||||
|
||||
|
||||
|
||||
- [FastHunyuan-diffusers.json](./examples/FastHunyuan-diffusers.json)
|
||||
|
||||
### Image to Video
|
||||
|
||||
Wan2.1-I2V-14B-480P-Diffusers
|
||||
Wan-AI/Wan2.1-I2V-14B-480P-Diffusers
|
||||
|
||||
|
||||
|
||||
|
||||
- [Wan2.1-I2V-14B-480P-Diffusers.json](./examples/Wan2.1-I2V-14B-480P-Diffusers.json)
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.6 MiB |
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"id": "23a1f065-bbba-4a8f-b144-944e1318fcbf",
|
||||
"revision": 0,
|
||||
"last_node_id": 9,
|
||||
"last_link_id": 8,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 4,
|
||||
"type": "VAEConfig",
|
||||
"pos": [
|
||||
374.2159423828125,
|
||||
554.85888671875
|
||||
],
|
||||
"size": [
|
||||
334.080078125,
|
||||
322
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "vae_config",
|
||||
"type": "VAE_CONFIG",
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "VAEConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"load_encoder": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"load_decoder": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"tile_sample_min_height": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 256
|
||||
},
|
||||
"tile_sample_min_width": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 256
|
||||
},
|
||||
"tile_sample_min_num_frames": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 16
|
||||
},
|
||||
"tile_sample_stride_height": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 192
|
||||
},
|
||||
"tile_sample_stride_width": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 192
|
||||
},
|
||||
"tile_sample_stride_num_frames": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 12
|
||||
},
|
||||
"blend_num_frames": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 0
|
||||
},
|
||||
"use_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"use_temporal_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"use_parallel_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "TextEncoderConfig",
|
||||
"pos": [
|
||||
416.4937744140625,
|
||||
953.6171875
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "text_encoder_config",
|
||||
"type": "TEXT_ENCODER_CONFIG",
|
||||
"links": [
|
||||
7
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "TextEncoderConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
-99999,
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"prefix": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
},
|
||||
"quant_config": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
},
|
||||
"lora_config": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "DITConfig",
|
||||
"pos": [
|
||||
415.1928405761719,
|
||||
1154.1573486328125
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
82
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "dit_config",
|
||||
"type": "DIT_CONFIG",
|
||||
"links": [
|
||||
6
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "DITConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"prefix": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
},
|
||||
"quant_config": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "VHS_LoadVideoPath",
|
||||
"pos": [
|
||||
1350.136962890625,
|
||||
331.20361328125
|
||||
],
|
||||
"size": [
|
||||
236.4912109375,
|
||||
286
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "video"
|
||||
},
|
||||
"link": 4
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
5
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "frame_count",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "AUDIO",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "video_info",
|
||||
"type": "VHS_VIDEOINFO",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfyui-videohelpersuite",
|
||||
"ver": "1.7.2",
|
||||
"Node name for S&R": "VHS_LoadVideoPath"
|
||||
},
|
||||
"widgets_values": {
|
||||
"video": "",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1,
|
||||
"format": "Wan",
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "",
|
||||
"type": "path",
|
||||
"format": "video/",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "VHS_VideoCombine",
|
||||
"pos": [
|
||||
1668.3499755859375,
|
||||
328.22625732421875
|
||||
],
|
||||
"size": [
|
||||
507.507080078125,
|
||||
616.12890625
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 5
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "Filenames",
|
||||
"type": "VHS_FILENAMES",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfyui-videohelpersuite",
|
||||
"ver": "1.7.2",
|
||||
"Node name for S&R": "VHS_VideoCombine"
|
||||
},
|
||||
"widgets_values": {
|
||||
"frame_rate": 24,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": false,
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "._00001.mp4",
|
||||
"subfolder": "",
|
||||
"type": "temp",
|
||||
"format": "video/h264-mp4",
|
||||
"frame_rate": 24,
|
||||
"workflow": "._00001.png",
|
||||
"fullpath": "/workspace/ComfyUI/temp/._00001.mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "LoadImagePath",
|
||||
"pos": [
|
||||
24.36871910095215,
|
||||
181.6505584716797
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
334.0000305175781
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "image_path",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
8
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "LoadImagePath"
|
||||
},
|
||||
"widgets_values": [
|
||||
"dog.jpeg",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"type": "VideoGenerator",
|
||||
"pos": [
|
||||
818.804931640625,
|
||||
348.9299621582031
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
460
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "inference_args",
|
||||
"shape": 7,
|
||||
"type": "INFERENCE_ARGS",
|
||||
"link": 3
|
||||
},
|
||||
{
|
||||
"name": "vae_config",
|
||||
"shape": 7,
|
||||
"type": "VAE_CONFIG",
|
||||
"link": 2
|
||||
},
|
||||
{
|
||||
"name": "text_encoder_config",
|
||||
"shape": 7,
|
||||
"type": "TEXT_ENCODER_CONFIG",
|
||||
"link": 7
|
||||
},
|
||||
{
|
||||
"name": "dit_config",
|
||||
"shape": 7,
|
||||
"type": "DIT_CONFIG",
|
||||
"link": 6
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "video_path",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
4
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "VideoGenerator"
|
||||
},
|
||||
"widgets_values": [
|
||||
"A dog taking a stroll on flat ground in front of scenic mountains.",
|
||||
"/workspace/ComfyUI/outputs_video/",
|
||||
2,
|
||||
"Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
"bf16",
|
||||
-99999,
|
||||
true,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"embedded_cfg_scale": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 6
|
||||
},
|
||||
"sp_size": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 2
|
||||
},
|
||||
"tp_size": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 2
|
||||
},
|
||||
"vae_precision": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "fp16"
|
||||
},
|
||||
"vae_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"vae_sp": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"text_encoder_precisions": {
|
||||
"isAuto": false,
|
||||
"value": "bf16",
|
||||
"cachedValue": "fp32"
|
||||
},
|
||||
"precision": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "fp16"
|
||||
},
|
||||
"dit_cpu_offload": {
|
||||
"isAuto": false,
|
||||
"value": true,
|
||||
"cachedValue": true
|
||||
},
|
||||
"image_encoder_precision": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "fp32"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "InferenceArgs",
|
||||
"pos": [
|
||||
411.46307373046875,
|
||||
178.18182373046875
|
||||
],
|
||||
"size": [
|
||||
278.73828125,
|
||||
298
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image_path",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "image_path"
|
||||
},
|
||||
"link": 8
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "inference_args",
|
||||
"type": "INFERENCE_ARGS",
|
||||
"links": [
|
||||
3
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "InferenceArgs"
|
||||
},
|
||||
"widgets_values": [
|
||||
704,
|
||||
1280,
|
||||
21,
|
||||
50,
|
||||
5,
|
||||
-99999,
|
||||
1025,
|
||||
"fixed",
|
||||
24,
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"height": {
|
||||
"isAuto": false,
|
||||
"value": 704,
|
||||
"cachedValue": 720
|
||||
},
|
||||
"width": {
|
||||
"isAuto": false,
|
||||
"value": 1280,
|
||||
"cachedValue": 1280
|
||||
},
|
||||
"num_frames": {
|
||||
"isAuto": false,
|
||||
"value": 21,
|
||||
"cachedValue": 45
|
||||
},
|
||||
"num_inference_steps": {
|
||||
"isAuto": false,
|
||||
"value": 50,
|
||||
"cachedValue": 6
|
||||
},
|
||||
"guidance_scale": {
|
||||
"isAuto": false,
|
||||
"value": 5,
|
||||
"cachedValue": 0.9
|
||||
},
|
||||
"flow_shift": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 17
|
||||
},
|
||||
"seed": {
|
||||
"isAuto": false,
|
||||
"value": 1025,
|
||||
"cachedValue": 1024
|
||||
},
|
||||
"fps": {
|
||||
"isAuto": false,
|
||||
"value": 24,
|
||||
"cachedValue": 24
|
||||
},
|
||||
"image_path": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "X://insert/path/here.mp4"
|
||||
},
|
||||
"enable_teacache": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
2,
|
||||
4,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
"VAE_CONFIG"
|
||||
],
|
||||
[
|
||||
3,
|
||||
2,
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
"INFERENCE_ARGS"
|
||||
],
|
||||
[
|
||||
4,
|
||||
1,
|
||||
0,
|
||||
3,
|
||||
2,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
5,
|
||||
3,
|
||||
0,
|
||||
8,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
6,
|
||||
6,
|
||||
0,
|
||||
1,
|
||||
3,
|
||||
"DIT_CONFIG"
|
||||
],
|
||||
[
|
||||
7,
|
||||
5,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
"TEXT_ENCODER_CONFIG"
|
||||
],
|
||||
[
|
||||
8,
|
||||
9,
|
||||
0,
|
||||
2,
|
||||
0,
|
||||
"STRING"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.9229599817706479,
|
||||
"offset": [
|
||||
-266.82059483107133,
|
||||
-98.19223155358443
|
||||
]
|
||||
},
|
||||
"frontendVersion": "1.24.4",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -44,10 +44,7 @@ def update_config_from_args(config: Any, args_dict: dict[str, Any]) -> None:
|
||||
"""
|
||||
for key, value in args_dict.items():
|
||||
if hasattr(config, key) and value is not None:
|
||||
if key == "text_encoder_precisions" and isinstance(value, list):
|
||||
setattr(config, key, tuple(value))
|
||||
else:
|
||||
setattr(config, key, value)
|
||||
setattr(config, key, value)
|
||||
|
||||
|
||||
class VideoGenerator:
|
||||
@@ -88,8 +85,8 @@ class VideoGenerator:
|
||||
"default": 2
|
||||
}),
|
||||
"vae_config": ("VAE_CONFIG", ),
|
||||
"vae_precision": (["fp16", "bf16"], {
|
||||
"default": "fp16"
|
||||
"vae_precision": (["fp16", "bf16", "fp32"], {
|
||||
"default": "fp32"
|
||||
}),
|
||||
"vae_tiling": ([True, False], {
|
||||
"default": True
|
||||
@@ -98,16 +95,19 @@ class VideoGenerator:
|
||||
"default": False
|
||||
}),
|
||||
"text_encoder_config": ("TEXT_ENCODER_CONFIG", ),
|
||||
"text_encoder_precision": (["fp16", "bf16"], {
|
||||
"default": "fp16"
|
||||
"text_encoder_precisions": (["fp16", "bf16", "fp32"], {
|
||||
"default": "fp32"
|
||||
}),
|
||||
"dit_config": ("DIT_CONFIG", ),
|
||||
"precision": (["fp16", "bf16"], {
|
||||
"default": "fp16"
|
||||
"precision": (["fp16", "bf16", "fp32"], {
|
||||
"default": "bf16"
|
||||
}),
|
||||
"dit_cpu_offload": ([True, False], {
|
||||
"default": False
|
||||
}),
|
||||
"image_encoder_precision": (["fp16", "bf16", "fp32"], {
|
||||
"default": "fp32"
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,8 +198,9 @@ class VideoGenerator:
|
||||
vae_precision,
|
||||
vae_tiling,
|
||||
vae_sp,
|
||||
text_encoder_precision,
|
||||
text_encoder_precisions,
|
||||
precision,
|
||||
image_encoder_precision,
|
||||
inference_args=None,
|
||||
vae_config=None,
|
||||
text_encoder_config=None,
|
||||
@@ -216,7 +217,7 @@ class VideoGenerator:
|
||||
|
||||
# Load pipeline config from model path
|
||||
pipeline_config = PipelineConfig.from_pretrained(model_path)
|
||||
print('pipeline_config', pipeline_config)
|
||||
print('Default Pipeline Config:', pipeline_config)
|
||||
|
||||
# Update configs with provided config dictionaries
|
||||
if dit_config is not None:
|
||||
@@ -241,14 +242,19 @@ class VideoGenerator:
|
||||
raw_pipeline_args['vae_tiling'] = vae_tiling
|
||||
if vae_sp is not None:
|
||||
raw_pipeline_args['vae_sp'] = vae_sp
|
||||
if text_encoder_precision is not None:
|
||||
raw_pipeline_args['text_encoder_precision'] = text_encoder_precision
|
||||
if text_encoder_precisions is not None and str(text_encoder_precisions) != str(-99999):
|
||||
raw_pipeline_args['text_encoder_precisions'] = tuple([text_encoder_precisions])
|
||||
if image_encoder_precision is not None:
|
||||
raw_pipeline_args['image_encoder_precision'] = image_encoder_precision
|
||||
|
||||
# Filter out any value explicitly set to -99999 (auto values)
|
||||
pipeline_args = {
|
||||
k: v
|
||||
for k, v in raw_pipeline_args.items() if str(int(v)) != str(-99999)
|
||||
}
|
||||
pipeline_args = {}
|
||||
for k, v in raw_pipeline_args.items():
|
||||
try:
|
||||
if str(int(v)) != str(-99999):
|
||||
pipeline_args[k] = v
|
||||
except (ValueError, TypeError):
|
||||
pipeline_args[k] = v
|
||||
|
||||
update_config_from_args(pipeline_config, pipeline_args)
|
||||
|
||||
@@ -269,14 +275,14 @@ class VideoGenerator:
|
||||
}
|
||||
|
||||
if self.generator is None:
|
||||
print('generation_args', generation_args)
|
||||
print('pipeline_config', pipeline_config)
|
||||
print('Generation Args:', generation_args)
|
||||
print('Updated Pipeline Config:', pipeline_config)
|
||||
self.generator = FastVideoGenerator.from_pretrained(
|
||||
model_path=model_path,
|
||||
**generation_args,
|
||||
pipeline_config=pipeline_config)
|
||||
|
||||
print('inference_args', inference_args)
|
||||
print('Inference Args:', inference_args)
|
||||
|
||||
# Start a thread to run the generation
|
||||
self._generation_thread = threading.Thread(target=self._run_generation,
|
||||
|
||||
@@ -551,8 +551,9 @@ app.registerExtension({
|
||||
"tile_sample_stride_num_frames", "blend_num_frames"
|
||||
]
|
||||
const floatWidgetNames = ["embedded_cfg_scale", "guidance_scale"]
|
||||
const comboWidgetNames = ["vae_tiling", "vae_precision", "vae_sp", "text_encoder_precision", "precision",
|
||||
"load_encoder", "load_decoder", "use_tiling", "use_temporal_tiling", "use_parallel_tiling", "dit_cpu_offload", "enable_teacache"
|
||||
const comboWidgetNames = ["vae_tiling", "vae_precision", "vae_sp", "text_encoder_precisions", "precision",
|
||||
"load_encoder", "load_decoder", "use_tiling", "use_temporal_tiling", "use_parallel_tiling", "dit_cpu_offload", "enable_teacache",
|
||||
"image_encoder_precision"
|
||||
]
|
||||
const stringWidgetNames = ["prefix", "quant_config", "lora_config", "image_path"]
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from typing import Any
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from tqdm.auto import tqdm
|
||||
import time
|
||||
|
||||
from fastvideo.attention import get_attn_backend
|
||||
from fastvideo.configs.pipelines.base import STA_Mode
|
||||
from fastvideo.distributed import (get_local_torch_device, get_sp_parallel_rank,
|
||||
@@ -88,11 +88,6 @@ class DenoisingStage(PipelineStage):
|
||||
Returns:
|
||||
The batch with denoised latents.
|
||||
"""
|
||||
# Start timing for the entire DenoisingStage
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
stage_start_time = time.time()
|
||||
|
||||
pipeline = self.pipeline() if self.pipeline else None
|
||||
if not fastvideo_args.model_loaded["transformer"]:
|
||||
loader = TransformerLoader()
|
||||
@@ -606,11 +601,6 @@ class DmdDenoisingStage(DenoisingStage):
|
||||
Returns:
|
||||
The batch with denoised latents.
|
||||
"""
|
||||
# Start timing for the entire DmdDenoisingStage
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
dmd_stage_start_time = time.time()
|
||||
|
||||
# Setup precision and autocast settings
|
||||
# TODO(will): make the precision configurable for inference
|
||||
# target_dtype = PRECISION_TO_TYPE[fastvideo_args.precision]
|
||||
@@ -691,13 +681,7 @@ class DmdDenoisingStage(DenoisingStage):
|
||||
image_latent = image_latent[:, :, rank_in_sp_group, :, :, :]
|
||||
batch.image_latent = image_latent
|
||||
|
||||
|
||||
|
||||
# Synchronize CUDA before starting timer
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
denoise_start_time = time.time()
|
||||
|
||||
# Run denoising loop
|
||||
with self.progress_bar(total=len(timesteps)) as progress_bar:
|
||||
for i, t in enumerate(timesteps):
|
||||
# Skip if interrupted
|
||||
@@ -731,7 +715,9 @@ class DmdDenoisingStage(DenoisingStage):
|
||||
with torch.autocast(device_type="cuda",
|
||||
dtype=target_dtype,
|
||||
enabled=autocast_enabled):
|
||||
if (vsa_available and self.attn_backend
|
||||
if (st_attn_available
|
||||
and self.attn_backend == SlidingTileAttentionBackend
|
||||
) or (vsa_available and self.attn_backend
|
||||
== VideoSparseAttentionBackend):
|
||||
self.attn_metadata_builder_cls = self.attn_backend.get_builder_cls(
|
||||
)
|
||||
@@ -812,20 +798,6 @@ class DmdDenoisingStage(DenoisingStage):
|
||||
if sp_group:
|
||||
latents = sequence_model_parallel_all_gather(latents, dim=1)
|
||||
latents = latents.permute(0, 2, 1, 3, 4)
|
||||
|
||||
# Calculate total denoising time
|
||||
# Synchronize CUDA before ending timer
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
denoise_end_time = time.time()
|
||||
total_denoise_time = denoise_end_time - denoise_start_time
|
||||
print(f"Total denoising time: {total_denoise_time:.2f} seconds")
|
||||
|
||||
# Calculate total stage time
|
||||
dmd_stage_end_time = time.time()
|
||||
total_dmd_stage_time = dmd_stage_end_time - dmd_stage_start_time
|
||||
print(f"Total DmdDenoisingStage time: {total_dmd_stage_time:.2f} seconds")
|
||||
|
||||
# Update batch with final latents
|
||||
batch.latents = latents
|
||||
|
||||
|
||||
Reference in New Issue
Block a user