Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 “BrianChen1129”
|
30bf8ca964 | ||
|
“BrianChen1129”
|
7e8c607e44 | ||
|
“BrianChen1129”
|
78352fcb77 | ||
|
“BrianChen1129”
|
3187698cc2 | ||
|
“BrianChen1129”
|
5ae466be21 | ||
|
“BrianChen1129”
|
f70cdc4c2e | ||
|
“BrianChen1129”
|
62fe15b64f | ||
|
“BrianChen1129”
|
6b43af5274 | ||
|
“BrianChen1129”
|
3efe6da6b0 | ||
|
“BrianChen1129”
|
3fc247ddf4 | ||
|
“BrianChen1129”
|
76006d24cf | ||
|
“BrianChen1129”
|
708fe18bb8 | ||
|
“BrianChen1129”
|
a0522cd7f0 | ||
|
“BrianChen1129”
|
df6afbb8b0 | ||
|
“BrianChen1129”
|
6d446c6eb9 | ||
|
“BrianChen1129”
|
ebae1321cf | ||
|
“BrianChen1129”
|
5c94cbad94 | ||
|
“BrianChen1129”
|
862ac8a6a1 | ||
|
“BrianChen1129”
|
e1b2c40857 | ||
|
“BrianChen1129”
|
93521970f6 | ||
|
“BrianChen1129”
|
e7c363d313 | ||
|
“BrianChen1129”
|
fc63becc92 | ||
|
“BrianChen1129”
|
3973469f70 | ||
|
“BrianChen1129”
|
9c135a3845 | ||
|
“BrianChen1129”
|
bc8b24ca9b |
+2
-2
@@ -1,7 +1,7 @@
|
||||
try:
|
||||
from .comfyui.video_generator.nodes import (NODE_CLASS_MAPPINGS,
|
||||
NODE_DISPLAY_NAME_MAPPINGS)
|
||||
WEB_DIRECTORY = "./comfyui/web"
|
||||
WEB_DIRECTORY = "./web"
|
||||
__all__ = [
|
||||
'NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY'
|
||||
]
|
||||
@@ -9,7 +9,7 @@ except ImportError:
|
||||
# ComfyUI environment not available, skip comfyui imports
|
||||
NODE_CLASS_MAPPINGS = {}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {}
|
||||
WEB_DIRECTORY = "./comfyui/web"
|
||||
WEB_DIRECTORY = "./web"
|
||||
__all__ = [
|
||||
'NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS', 'WEB_DIRECTORY'
|
||||
]
|
||||
|
||||
+3
-12
@@ -74,7 +74,6 @@ After installation, the following nodes will be available in the ComfyUI interfa
|
||||
You may have noticed many arguments on the nodes have 'auto' as the default value. This is because FastVideo will automatically detect the best values for these parameters based on the model and the hardware. However, you can also manually configure these parameters to get the best performance for your specific use case. We plan on releasing more optimized workflow files for different models and hardware configurations in the future.
|
||||
|
||||
You can see what some of the default configurations are by looking at the FastVideo repo:
|
||||
-[Wan-AI_Wan2.2-TI2V-5B-Diffusers]()
|
||||
- [Wan2.1-I2V-14B-480P-Diffusers](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/configs/wan_14B_i2v_480p_pipeline.json)
|
||||
- [FastHunyuan-diffusers](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/configs/fasthunyuan_t2v.json)
|
||||
|
||||
@@ -118,27 +117,19 @@ These parameters affect the model's distribution across GPUs and require a compl
|
||||
|
||||
## Example workflows
|
||||
|
||||
### Text and Image to Video
|
||||
|
||||
Wan-AI/Wan2.2-TI2V-5B-Diffusers
|
||||
|
||||
|
||||
|
||||
- [Wan2.2-TI2V-5B-Diffusers](./examples/Wan-AI_Wan2.2-TI2V-5B-Diffusers.json)
|
||||
|
||||
### Text to Video
|
||||
|
||||
FastVideo-FastHunyuan-diffusers
|
||||
|
||||
|
||||
|
||||
|
||||
- [FastHunyuan-diffusers.json](./examples/FastHunyuan-diffusers.json)
|
||||
|
||||
### Image to Video
|
||||
|
||||
Wan-AI/Wan2.1-I2V-14B-480P-Diffusers
|
||||
Wan2.1-I2V-14B-480P-Diffusers
|
||||
|
||||
|
||||
|
||||
|
||||
- [Wan2.1-I2V-14B-480P-Diffusers.json](./examples/Wan2.1-I2V-14B-480P-Diffusers.json)
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 MiB |
@@ -1,727 +0,0 @@
|
||||
{
|
||||
"id": "23a1f065-bbba-4a8f-b144-944e1318fcbf",
|
||||
"revision": 0,
|
||||
"last_node_id": 9,
|
||||
"last_link_id": 8,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 4,
|
||||
"type": "VAEConfig",
|
||||
"pos": [
|
||||
374.2159423828125,
|
||||
554.85888671875
|
||||
],
|
||||
"size": [
|
||||
334.080078125,
|
||||
322
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "vae_config",
|
||||
"type": "VAE_CONFIG",
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "VAEConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"load_encoder": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"load_decoder": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"tile_sample_min_height": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 256
|
||||
},
|
||||
"tile_sample_min_width": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 256
|
||||
},
|
||||
"tile_sample_min_num_frames": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 16
|
||||
},
|
||||
"tile_sample_stride_height": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 192
|
||||
},
|
||||
"tile_sample_stride_width": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 192
|
||||
},
|
||||
"tile_sample_stride_num_frames": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 12
|
||||
},
|
||||
"blend_num_frames": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 0
|
||||
},
|
||||
"use_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"use_temporal_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"use_parallel_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "TextEncoderConfig",
|
||||
"pos": [
|
||||
416.4937744140625,
|
||||
953.6171875
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
106
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "text_encoder_config",
|
||||
"type": "TEXT_ENCODER_CONFIG",
|
||||
"links": [
|
||||
7
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "TextEncoderConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
-99999,
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"prefix": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
},
|
||||
"quant_config": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
},
|
||||
"lora_config": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "DITConfig",
|
||||
"pos": [
|
||||
415.1928405761719,
|
||||
1154.1573486328125
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
82
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "dit_config",
|
||||
"type": "DIT_CONFIG",
|
||||
"links": [
|
||||
6
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "DITConfig"
|
||||
},
|
||||
"widgets_values": [
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"prefix": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
},
|
||||
"quant_config": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "VHS_LoadVideoPath",
|
||||
"pos": [
|
||||
1350.136962890625,
|
||||
331.20361328125
|
||||
],
|
||||
"size": [
|
||||
236.4912109375,
|
||||
286
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "video"
|
||||
},
|
||||
"link": 4
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
5
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "frame_count",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "AUDIO",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "video_info",
|
||||
"type": "VHS_VIDEOINFO",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfyui-videohelpersuite",
|
||||
"ver": "1.7.2",
|
||||
"Node name for S&R": "VHS_LoadVideoPath"
|
||||
},
|
||||
"widgets_values": {
|
||||
"video": "",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1,
|
||||
"format": "Wan",
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "",
|
||||
"type": "path",
|
||||
"format": "video/",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "VHS_VideoCombine",
|
||||
"pos": [
|
||||
1668.3499755859375,
|
||||
328.22625732421875
|
||||
],
|
||||
"size": [
|
||||
507.507080078125,
|
||||
616.12890625
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 5
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"shape": 7,
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "Filenames",
|
||||
"type": "VHS_FILENAMES",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfyui-videohelpersuite",
|
||||
"ver": "1.7.2",
|
||||
"Node name for S&R": "VHS_VideoCombine"
|
||||
},
|
||||
"widgets_values": {
|
||||
"frame_rate": 24,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": false,
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "._00001.mp4",
|
||||
"subfolder": "",
|
||||
"type": "temp",
|
||||
"format": "video/h264-mp4",
|
||||
"frame_rate": 24,
|
||||
"workflow": "._00001.png",
|
||||
"fullpath": "/workspace/ComfyUI/temp/._00001.mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "LoadImagePath",
|
||||
"pos": [
|
||||
24.36871910095215,
|
||||
181.6505584716797
|
||||
],
|
||||
"size": [
|
||||
270,
|
||||
334.0000305175781
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "image_path",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
8
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "LoadImagePath"
|
||||
},
|
||||
"widgets_values": [
|
||||
"dog.jpeg",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"type": "VideoGenerator",
|
||||
"pos": [
|
||||
818.804931640625,
|
||||
348.9299621582031
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
460
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "inference_args",
|
||||
"shape": 7,
|
||||
"type": "INFERENCE_ARGS",
|
||||
"link": 3
|
||||
},
|
||||
{
|
||||
"name": "vae_config",
|
||||
"shape": 7,
|
||||
"type": "VAE_CONFIG",
|
||||
"link": 2
|
||||
},
|
||||
{
|
||||
"name": "text_encoder_config",
|
||||
"shape": 7,
|
||||
"type": "TEXT_ENCODER_CONFIG",
|
||||
"link": 7
|
||||
},
|
||||
{
|
||||
"name": "dit_config",
|
||||
"shape": 7,
|
||||
"type": "DIT_CONFIG",
|
||||
"link": 6
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "video_path",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
4
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "VideoGenerator"
|
||||
},
|
||||
"widgets_values": [
|
||||
"A dog taking a stroll on flat ground in front of scenic mountains.",
|
||||
"/workspace/ComfyUI/outputs_video/",
|
||||
2,
|
||||
"Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
-99999,
|
||||
"bf16",
|
||||
-99999,
|
||||
true,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"embedded_cfg_scale": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 6
|
||||
},
|
||||
"sp_size": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 2
|
||||
},
|
||||
"tp_size": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 2
|
||||
},
|
||||
"vae_precision": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "fp16"
|
||||
},
|
||||
"vae_tiling": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"vae_sp": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
},
|
||||
"text_encoder_precisions": {
|
||||
"isAuto": false,
|
||||
"value": "bf16",
|
||||
"cachedValue": "fp32"
|
||||
},
|
||||
"precision": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "fp16"
|
||||
},
|
||||
"dit_cpu_offload": {
|
||||
"isAuto": false,
|
||||
"value": true,
|
||||
"cachedValue": true
|
||||
},
|
||||
"image_encoder_precision": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "fp32"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "InferenceArgs",
|
||||
"pos": [
|
||||
411.46307373046875,
|
||||
178.18182373046875
|
||||
],
|
||||
"size": [
|
||||
278.73828125,
|
||||
298
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image_path",
|
||||
"shape": 7,
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "image_path"
|
||||
},
|
||||
"link": 8
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "inference_args",
|
||||
"type": "INFERENCE_ARGS",
|
||||
"links": [
|
||||
3
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "fastvideo",
|
||||
"ver": "1d6415004410b367a30daeefa684917d38d95a86",
|
||||
"Node name for S&R": "InferenceArgs"
|
||||
},
|
||||
"widgets_values": [
|
||||
704,
|
||||
1280,
|
||||
21,
|
||||
50,
|
||||
5,
|
||||
-99999,
|
||||
1025,
|
||||
"fixed",
|
||||
24,
|
||||
-99999,
|
||||
-99999
|
||||
],
|
||||
"auto_widget_states": {
|
||||
"height": {
|
||||
"isAuto": false,
|
||||
"value": 704,
|
||||
"cachedValue": 720
|
||||
},
|
||||
"width": {
|
||||
"isAuto": false,
|
||||
"value": 1280,
|
||||
"cachedValue": 1280
|
||||
},
|
||||
"num_frames": {
|
||||
"isAuto": false,
|
||||
"value": 21,
|
||||
"cachedValue": 45
|
||||
},
|
||||
"num_inference_steps": {
|
||||
"isAuto": false,
|
||||
"value": 50,
|
||||
"cachedValue": 6
|
||||
},
|
||||
"guidance_scale": {
|
||||
"isAuto": false,
|
||||
"value": 5,
|
||||
"cachedValue": 0.9
|
||||
},
|
||||
"flow_shift": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": 17
|
||||
},
|
||||
"seed": {
|
||||
"isAuto": false,
|
||||
"value": 1025,
|
||||
"cachedValue": 1024
|
||||
},
|
||||
"fps": {
|
||||
"isAuto": false,
|
||||
"value": 24,
|
||||
"cachedValue": 24
|
||||
},
|
||||
"image_path": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": "X://insert/path/here.mp4"
|
||||
},
|
||||
"enable_teacache": {
|
||||
"isAuto": true,
|
||||
"value": -99999,
|
||||
"cachedValue": true
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
2,
|
||||
4,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
"VAE_CONFIG"
|
||||
],
|
||||
[
|
||||
3,
|
||||
2,
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
"INFERENCE_ARGS"
|
||||
],
|
||||
[
|
||||
4,
|
||||
1,
|
||||
0,
|
||||
3,
|
||||
2,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
5,
|
||||
3,
|
||||
0,
|
||||
8,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
6,
|
||||
6,
|
||||
0,
|
||||
1,
|
||||
3,
|
||||
"DIT_CONFIG"
|
||||
],
|
||||
[
|
||||
7,
|
||||
5,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
"TEXT_ENCODER_CONFIG"
|
||||
],
|
||||
[
|
||||
8,
|
||||
9,
|
||||
0,
|
||||
2,
|
||||
0,
|
||||
"STRING"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.9229599817706479,
|
||||
"offset": [
|
||||
-266.82059483107133,
|
||||
-98.19223155358443
|
||||
]
|
||||
},
|
||||
"frontendVersion": "1.24.4",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -44,7 +44,10 @@ def update_config_from_args(config: Any, args_dict: dict[str, Any]) -> None:
|
||||
"""
|
||||
for key, value in args_dict.items():
|
||||
if hasattr(config, key) and value is not None:
|
||||
setattr(config, key, value)
|
||||
if key == "text_encoder_precisions" and isinstance(value, list):
|
||||
setattr(config, key, tuple(value))
|
||||
else:
|
||||
setattr(config, key, value)
|
||||
|
||||
|
||||
class VideoGenerator:
|
||||
@@ -85,8 +88,8 @@ class VideoGenerator:
|
||||
"default": 2
|
||||
}),
|
||||
"vae_config": ("VAE_CONFIG", ),
|
||||
"vae_precision": (["fp16", "bf16", "fp32"], {
|
||||
"default": "fp32"
|
||||
"vae_precision": (["fp16", "bf16"], {
|
||||
"default": "fp16"
|
||||
}),
|
||||
"vae_tiling": ([True, False], {
|
||||
"default": True
|
||||
@@ -95,19 +98,16 @@ class VideoGenerator:
|
||||
"default": False
|
||||
}),
|
||||
"text_encoder_config": ("TEXT_ENCODER_CONFIG", ),
|
||||
"text_encoder_precisions": (["fp16", "bf16", "fp32"], {
|
||||
"default": "fp32"
|
||||
"text_encoder_precision": (["fp16", "bf16"], {
|
||||
"default": "fp16"
|
||||
}),
|
||||
"dit_config": ("DIT_CONFIG", ),
|
||||
"precision": (["fp16", "bf16", "fp32"], {
|
||||
"default": "bf16"
|
||||
"precision": (["fp16", "bf16"], {
|
||||
"default": "fp16"
|
||||
}),
|
||||
"dit_cpu_offload": ([True, False], {
|
||||
"default": False
|
||||
}),
|
||||
"image_encoder_precision": (["fp16", "bf16", "fp32"], {
|
||||
"default": "fp32"
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,9 +198,8 @@ class VideoGenerator:
|
||||
vae_precision,
|
||||
vae_tiling,
|
||||
vae_sp,
|
||||
text_encoder_precisions,
|
||||
text_encoder_precision,
|
||||
precision,
|
||||
image_encoder_precision,
|
||||
inference_args=None,
|
||||
vae_config=None,
|
||||
text_encoder_config=None,
|
||||
@@ -217,7 +216,7 @@ class VideoGenerator:
|
||||
|
||||
# Load pipeline config from model path
|
||||
pipeline_config = PipelineConfig.from_pretrained(model_path)
|
||||
print('Default Pipeline Config:', pipeline_config)
|
||||
print('pipeline_config', pipeline_config)
|
||||
|
||||
# Update configs with provided config dictionaries
|
||||
if dit_config is not None:
|
||||
@@ -242,19 +241,14 @@ class VideoGenerator:
|
||||
raw_pipeline_args['vae_tiling'] = vae_tiling
|
||||
if vae_sp is not None:
|
||||
raw_pipeline_args['vae_sp'] = vae_sp
|
||||
if text_encoder_precisions is not None and str(text_encoder_precisions) != str(-99999):
|
||||
raw_pipeline_args['text_encoder_precisions'] = tuple([text_encoder_precisions])
|
||||
if image_encoder_precision is not None:
|
||||
raw_pipeline_args['image_encoder_precision'] = image_encoder_precision
|
||||
if text_encoder_precision is not None:
|
||||
raw_pipeline_args['text_encoder_precision'] = text_encoder_precision
|
||||
|
||||
# Filter out any value explicitly set to -99999 (auto values)
|
||||
pipeline_args = {}
|
||||
for k, v in raw_pipeline_args.items():
|
||||
try:
|
||||
if str(int(v)) != str(-99999):
|
||||
pipeline_args[k] = v
|
||||
except (ValueError, TypeError):
|
||||
pipeline_args[k] = v
|
||||
pipeline_args = {
|
||||
k: v
|
||||
for k, v in raw_pipeline_args.items() if str(int(v)) != str(-99999)
|
||||
}
|
||||
|
||||
update_config_from_args(pipeline_config, pipeline_args)
|
||||
|
||||
@@ -275,14 +269,14 @@ class VideoGenerator:
|
||||
}
|
||||
|
||||
if self.generator is None:
|
||||
print('Generation Args:', generation_args)
|
||||
print('Updated Pipeline Config:', pipeline_config)
|
||||
print('generation_args', generation_args)
|
||||
print('pipeline_config', pipeline_config)
|
||||
self.generator = FastVideoGenerator.from_pretrained(
|
||||
model_path=model_path,
|
||||
**generation_args,
|
||||
pipeline_config=pipeline_config)
|
||||
|
||||
print('Inference Args:', inference_args)
|
||||
print('inference_args', inference_args)
|
||||
|
||||
# Start a thread to run the generation
|
||||
self._generation_thread = threading.Thread(target=self._run_generation,
|
||||
|
||||
@@ -551,9 +551,8 @@ app.registerExtension({
|
||||
"tile_sample_stride_num_frames", "blend_num_frames"
|
||||
]
|
||||
const floatWidgetNames = ["embedded_cfg_scale", "guidance_scale"]
|
||||
const comboWidgetNames = ["vae_tiling", "vae_precision", "vae_sp", "text_encoder_precisions", "precision",
|
||||
"load_encoder", "load_decoder", "use_tiling", "use_temporal_tiling", "use_parallel_tiling", "dit_cpu_offload", "enable_teacache",
|
||||
"image_encoder_precision"
|
||||
const comboWidgetNames = ["vae_tiling", "vae_precision", "vae_sp", "text_encoder_precision", "precision",
|
||||
"load_encoder", "load_decoder", "use_tiling", "use_temporal_tiling", "use_parallel_tiling", "dit_cpu_offload", "enable_teacache"
|
||||
]
|
||||
const stringWidgetNames = ["prefix", "quant_config", "lora_config", "image_path"]
|
||||
|
||||
|
||||
@@ -187,3 +187,6 @@ class TrainingBatch:
|
||||
# Distillation losses
|
||||
generator_loss: float = 0.0
|
||||
fake_score_loss: float = 0.0
|
||||
|
||||
dmd_latent_vis_dict: dict[str, Any] = field(default_factory=dict)
|
||||
fake_score_latent_vis_dict: dict[str, Any] = field(default_factory=dict)
|
||||
@@ -192,6 +192,7 @@ class DistillationPipeline(TrainingPipeline):
|
||||
device=self.device,
|
||||
dtype=torch.long)
|
||||
timestep = self.denoising_step_list[index]
|
||||
training_batch.dmd_latent_vis_dict["generator_timestep"] = timestep
|
||||
|
||||
noise = torch.randn(self.video_latent_shape,
|
||||
device=self.device,
|
||||
@@ -258,7 +259,7 @@ class DistillationPipeline(TrainingPipeline):
|
||||
fake_score_pred_noise = self.fake_score_transformer(
|
||||
**training_batch.input_kwargs).permute(0, 2, 1, 3, 4)
|
||||
|
||||
pred_fake_video = pred_noise_to_pred_video(
|
||||
faker_score_pred_video = pred_noise_to_pred_video(
|
||||
pred_noise=fake_score_pred_noise.flatten(0, 1),
|
||||
noise_input_latent=training_batch.noise_latents.flatten(0, 1),
|
||||
timestep=timestep,
|
||||
@@ -297,7 +298,7 @@ class DistillationPipeline(TrainingPipeline):
|
||||
pred_real_video_cond -
|
||||
pred_real_video_uncond) * self.real_score_guidance_scale
|
||||
|
||||
grad = (pred_fake_video - real_score_pred_video) / torch.abs(
|
||||
grad = (faker_score_pred_video - real_score_pred_video) / torch.abs(
|
||||
generator_pred_video - real_score_pred_video).mean()
|
||||
grad = torch.nan_to_num(grad)
|
||||
|
||||
@@ -305,6 +306,19 @@ class DistillationPipeline(TrainingPipeline):
|
||||
generator_pred_video.float(),
|
||||
(generator_pred_video.float() - grad.float()).detach())
|
||||
|
||||
training_batch.dmd_latent_vis_dict.update({
|
||||
"training_batch_dmd_fwd_clean_latent":
|
||||
training_batch.latents,
|
||||
"generator_pred_video":
|
||||
generator_pred_video,
|
||||
"real_score_pred_video":
|
||||
real_score_pred_video,
|
||||
"faker_score_pred_video":
|
||||
faker_score_pred_video,
|
||||
"dmd_timestep":
|
||||
timestep,
|
||||
})
|
||||
|
||||
return dmd_loss
|
||||
|
||||
def faker_score_forward(
|
||||
@@ -353,9 +367,16 @@ class DistillationPipeline(TrainingPipeline):
|
||||
**training_batch.input_kwargs).permute(0, 2, 1, 3, 4)
|
||||
|
||||
target = fake_score_noise - generator_pred_video
|
||||
denoising_loss = torch.mean((fake_score_pred_noise - target)**2)
|
||||
flow_matching_loss = torch.mean((fake_score_pred_noise - target)**2)
|
||||
|
||||
return training_batch, denoising_loss
|
||||
training_batch.fake_score_latent_vis_dict = {
|
||||
"training_batch_fakerscore_fwd_clean_latent":
|
||||
training_batch.latents,
|
||||
"generator_pred_video": generator_pred_video,
|
||||
"fake_score_timestep": fake_score_timestep,
|
||||
}
|
||||
|
||||
return training_batch, flow_matching_loss
|
||||
|
||||
def _clip_model_grad_norm_(self, training_batch: TrainingBatch,
|
||||
transformer) -> TrainingBatch:
|
||||
@@ -389,6 +410,9 @@ class DistillationPipeline(TrainingPipeline):
|
||||
"encoder_attention_mask": self.negative_prompt_attention_mask,
|
||||
}
|
||||
|
||||
training_batch.dmd_latent_vis_dict = {}
|
||||
training_batch.fake_score_latent_vis_dict = {}
|
||||
|
||||
training_batch.conditional_dict = conditional_dict
|
||||
training_batch.unconditional_dict = unconditional_dict
|
||||
training_batch.raw_latent_shape = training_batch.latents.shape
|
||||
@@ -425,27 +449,29 @@ class DistillationPipeline(TrainingPipeline):
|
||||
|
||||
self.optimizer.zero_grad()
|
||||
total_dmd_loss = 0.0
|
||||
dmd_latent_vis_dict = {}
|
||||
fake_score_latent_vis_dict = {}
|
||||
if (self.current_trainstep % self.generator_update_interval == 0):
|
||||
for batch in batches:
|
||||
batch_stu = copy.deepcopy(batch)
|
||||
batch_gen = copy.deepcopy(batch)
|
||||
|
||||
with set_forward_context(
|
||||
current_timestep=batch_stu.timesteps,
|
||||
attn_metadata=batch_stu.attn_metadata_vsa):
|
||||
generator_pred_video = self._generator_forward(batch_stu)
|
||||
current_timestep=batch_gen.timesteps,
|
||||
attn_metadata=batch_gen.attn_metadata_vsa):
|
||||
generator_pred_video = self._generator_forward(batch_gen)
|
||||
|
||||
with set_forward_context(current_timestep=batch_stu.timesteps,
|
||||
attn_metadata=batch_stu.attn_metadata):
|
||||
with set_forward_context(current_timestep=batch_gen.timesteps,
|
||||
attn_metadata=batch_gen.attn_metadata):
|
||||
dmd_loss = self._dmd_forward(
|
||||
generator_pred_video=generator_pred_video,
|
||||
training_batch=batch_stu)
|
||||
training_batch=batch_gen)
|
||||
|
||||
with set_forward_context(
|
||||
current_timestep=batch_stu.timesteps,
|
||||
attn_metadata=batch_stu.attn_metadata_vsa):
|
||||
current_timestep=batch_gen.timesteps,
|
||||
attn_metadata=batch_gen.attn_metadata_vsa):
|
||||
(dmd_loss / gradient_accumulation_steps).backward()
|
||||
total_dmd_loss += dmd_loss.detach().item()
|
||||
self._clip_model_grad_norm_(batch_stu, self.transformer)
|
||||
self._clip_model_grad_norm_(batch_gen, self.transformer)
|
||||
self.optimizer.step()
|
||||
self.lr_scheduler.step()
|
||||
self.optimizer.zero_grad(set_to_none=True)
|
||||
@@ -456,20 +482,22 @@ class DistillationPipeline(TrainingPipeline):
|
||||
world_group.all_reduce(avg_dmd_loss,
|
||||
op=torch.distributed.ReduceOp.AVG)
|
||||
training_batch.generator_loss = avg_dmd_loss.item()
|
||||
dmd_latent_vis_dict = batch_gen.dmd_latent_vis_dict
|
||||
else:
|
||||
training_batch.generator_loss = 0.0
|
||||
|
||||
self.fake_score_optimizer.zero_grad()
|
||||
total_fake_score_loss = 0.0
|
||||
for batch in batches:
|
||||
batch_critic = copy.deepcopy(batch)
|
||||
batch_critic, fake_score_loss = self.faker_score_forward(
|
||||
batch_critic)
|
||||
with set_forward_context(current_timestep=batch_critic.timesteps,
|
||||
attn_metadata=batch_critic.attn_metadata):
|
||||
batch_fake = copy.deepcopy(batch)
|
||||
batch_fake, fake_score_loss = self.faker_score_forward(batch_fake)
|
||||
with set_forward_context(current_timestep=batch_fake.timesteps,
|
||||
attn_metadata=batch_fake.attn_metadata):
|
||||
(fake_score_loss / gradient_accumulation_steps).backward()
|
||||
total_fake_score_loss += fake_score_loss.detach().item()
|
||||
self._clip_model_grad_norm_(batch_critic, self.fake_score_transformer)
|
||||
fake_score_latent_vis_dict.update(
|
||||
batch_fake.fake_score_latent_vis_dict)
|
||||
self._clip_model_grad_norm_(batch_fake, self.fake_score_transformer)
|
||||
self.fake_score_optimizer.step()
|
||||
self.fake_score_lr_scheduler.step()
|
||||
self.fake_score_optimizer.zero_grad(set_to_none=True)
|
||||
@@ -480,6 +508,8 @@ class DistillationPipeline(TrainingPipeline):
|
||||
world_group.all_reduce(avg_fake_score_loss,
|
||||
op=torch.distributed.ReduceOp.AVG)
|
||||
training_batch.fake_score_loss = avg_fake_score_loss.item()
|
||||
training_batch.dmd_latent_vis_dict = dmd_latent_vis_dict
|
||||
training_batch.fake_score_latent_vis_dict = batch_fake.fake_score_latent_vis_dict
|
||||
|
||||
training_batch.total_loss = training_batch.generator_loss + training_batch.fake_score_loss
|
||||
return training_batch
|
||||
@@ -665,6 +695,88 @@ class DistillationPipeline(TrainingPipeline):
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def visualize_intermediate_latents(self, training_batch: TrainingBatch,
|
||||
training_args: TrainingArgs, step: int):
|
||||
"""Add visualization data to wandb logging and save frames to disk."""
|
||||
wandb_loss_dict = {}
|
||||
torch.cuda.empty_cache()
|
||||
dmd_latents_vis_dict = training_batch.dmd_latent_vis_dict
|
||||
fake_score_latents_vis_dict = training_batch.fake_score_latent_vis_dict
|
||||
fake_score_log_keys = [
|
||||
'training_batch_fakerscore_fwd_clean_latent', 'generator_pred_video'
|
||||
]
|
||||
dmd_log_keys = [
|
||||
'training_batch_dmd_fwd_clean_latent', 'generator_pred_video',
|
||||
'real_score_pred_video', 'faker_score_pred_video'
|
||||
]
|
||||
|
||||
for latent_key in fake_score_log_keys:
|
||||
latents = fake_score_latents_vis_dict[latent_key] # bs, t,c, h, w
|
||||
latents = latents.permute(0, 2, 1, 3, 4)
|
||||
# decoded_latent = decode_stage(ForwardBatch(data_type="video", latents=latents), training_args)
|
||||
|
||||
if isinstance(self.vae.scaling_factor, torch.Tensor):
|
||||
latents = latents / self.vae.scaling_factor.to(
|
||||
latents.device, latents.dtype)
|
||||
else:
|
||||
latents = latents / self.vae.scaling_factor
|
||||
|
||||
# Apply shifting if needed
|
||||
if (hasattr(self.vae, "shift_factor")
|
||||
and self.vae.shift_factor is not None):
|
||||
if isinstance(self.vae.shift_factor, torch.Tensor):
|
||||
latents += self.vae.shift_factor.to(latents.device,
|
||||
latents.dtype)
|
||||
else:
|
||||
latents += self.vae.shift_factor
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
video = self.vae.decode(latents)
|
||||
video = (video / 2 + 0.5).clamp(0, 1)
|
||||
video = video.cpu().float()
|
||||
video = video.permute(0, 2, 1, 3, 4)
|
||||
video = (video * 255).numpy().astype(np.uint8)
|
||||
wandb_loss_dict[latent_key] = wandb.Video(
|
||||
video, fps=24, format="mp4") # change to 16 for Wan2.1
|
||||
# Clean up references
|
||||
del video, latents
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# Process DMD training data if available - use decode_stage instead of self.vae.decode
|
||||
if 'generator_pred_video' in dmd_latents_vis_dict:
|
||||
for latent_key in dmd_log_keys:
|
||||
latents = dmd_latents_vis_dict[latent_key]
|
||||
latents = latents.permute(0, 2, 1, 3, 4)
|
||||
# decoded_latent = decode_stage(ForwardBatch(data_type="video", latents=latents), training_args)
|
||||
if isinstance(self.vae.scaling_factor, torch.Tensor):
|
||||
latents = latents / self.vae.scaling_factor.to(
|
||||
latents.device, latents.dtype)
|
||||
else:
|
||||
latents = latents / self.vae.scaling_factor
|
||||
|
||||
# Apply shifting if needed
|
||||
if (hasattr(self.vae, "shift_factor")
|
||||
and self.vae.shift_factor is not None):
|
||||
if isinstance(self.vae.shift_factor, torch.Tensor):
|
||||
latents += self.vae.shift_factor.to(
|
||||
latents.device, latents.dtype)
|
||||
else:
|
||||
latents += self.vae.shift_factor
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
video = self.vae.decode(latents)
|
||||
video = (video / 2 + 0.5).clamp(0, 1)
|
||||
video = video.cpu().float()
|
||||
video = video.permute(0, 2, 1, 3, 4)
|
||||
video = (video * 255).numpy().astype(np.uint8)
|
||||
wandb_loss_dict[latent_key] = wandb.Video(
|
||||
video, fps=24, format="mp4") # change to 16 for Wan2.1
|
||||
# Clean up references
|
||||
del video, latents
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# Log to wandb
|
||||
if self.global_rank == 0:
|
||||
wandb.log(wandb_loss_dict, step=step)
|
||||
|
||||
def train(self) -> None:
|
||||
"""Main training loop with distillation-specific logging."""
|
||||
assert self.training_args.seed is not None, "seed must be set"
|
||||
@@ -768,6 +880,25 @@ class DistillationPipeline(TrainingPipeline):
|
||||
log_data["train_generator_loss"] = generator_loss
|
||||
if use_vsa:
|
||||
log_data["VSA_train_sparsity"] = current_vsa_sparsity
|
||||
|
||||
if training_batch.dmd_latent_vis_dict:
|
||||
dmd_additional_logs = {
|
||||
"generator_timestep":
|
||||
training_batch.
|
||||
dmd_latent_vis_dict["generator_timestep"].item(),
|
||||
"dmd_timestep":
|
||||
training_batch.dmd_latent_vis_dict["dmd_timestep"].item(
|
||||
),
|
||||
}
|
||||
log_data.update(dmd_additional_logs)
|
||||
|
||||
faker_score_additional_logs = {
|
||||
"fake_score_timestep":
|
||||
training_batch.
|
||||
fake_score_latent_vis_dict["fake_score_timestep"].item(),
|
||||
}
|
||||
log_data.update(faker_score_additional_logs)
|
||||
|
||||
wandb.log(log_data, step=step)
|
||||
|
||||
# Save training state checkpoint (for resuming training)
|
||||
@@ -801,6 +932,10 @@ class DistillationPipeline(TrainingPipeline):
|
||||
only_save_generator_weight=True)
|
||||
|
||||
if self.training_args.log_validation and step % self.training_args.validation_steps == 0:
|
||||
if self.training_args.log_visualization:
|
||||
self.visualize_intermediate_latents(training_batch,
|
||||
self.training_args,
|
||||
step)
|
||||
self._log_validation(self.transformer, self.training_args, step)
|
||||
|
||||
wandb.finish()
|
||||
|
||||
Reference in New Issue
Block a user