From 336e236105fbb3bda868d8b4134ddff1fbe094a1 Mon Sep 17 00:00:00 2001 From: John Pollock Date: Fri, 29 Aug 2025 20:50:55 -0500 Subject: [PATCH] Fix for wanvideo bug and add new example --- ...1_3B_control_lora_example_01_MultiGPU.json | 1165 +++++++++++++++++ wanvideo.py | 31 + 2 files changed, 1196 insertions(+) create mode 100755 examples/wannvideowrapper/wanvideo_1_3B_control_lora_example_01_MultiGPU.json diff --git a/examples/wannvideowrapper/wanvideo_1_3B_control_lora_example_01_MultiGPU.json b/examples/wannvideowrapper/wanvideo_1_3B_control_lora_example_01_MultiGPU.json new file mode 100755 index 0000000..9bf08bf --- /dev/null +++ b/examples/wannvideowrapper/wanvideo_1_3B_control_lora_example_01_MultiGPU.json @@ -0,0 +1,1165 @@ +{ + "id": "04f1de76-e363-4c27-bec7-eb184ac6e476", + "revision": 0, + "last_node_id": 111, + "last_link_id": 161, + "nodes": [ + { + "id": 36, + "type": "Note", + "pos": [ + 160, + -1010 + ], + "size": [ + 374.3061828613281, + 171.9547576904297 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "fp8_fast seems to cause huge quality degradation\n\nfp_16_fast enables \"Full FP16 Accmumulation in FP16 GEMMs\" feature available in the very latest pytorch nightly, this is around 20% speed boost. \n\nSageattn if you have it installed can be used for almost double inference speed" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 35, + "type": "WanVideoTorchCompileSettings", + "pos": [ + -276.8500671386719, + -1050.6326904296875 + ], + "size": [ + 390.5999755859375, + 202 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 44, + "type": "Note", + "pos": [ + -620.9041137695312, + -1049.732421875 + ], + "size": [ + 303.0501403808594, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "If you have Triton installed, connect this for ~30% speed increase" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 53, + "type": "Note", + "pos": [ + 1329.0076904296875, + -810.7708740234375 + ], + "size": [ + 324.64129638671875, + 159.47401428222656 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "TeaCache could be considered to be sort of an automated step skipper \n\nThe relative l1 threshold -value determines how aggressive this is, higher values are faster but quality suffers more. Very first steps should NEVER be skipped with this model or it kills the motion. When using the pre-calculated coefficients, the treshold value should be much higher than with the default coefficients." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 1704.86572265625, + -604.0441284179688 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 159 + }, + { + "name": "samples", + "type": "LATENT", + "link": 154 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 145 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 30, + "type": "VHS_VideoCombine", + "pos": [ + 1709.29345703125, + -334.3517150878906 + ], + "size": [ + 1037.934326171875, + 856.9671630859375 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 146 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "WanVideoWrapper_I2V", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideoWrapper_I2V_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16, + "workflow": "WanVideoWrapper_I2V_00004.png", + "fullpath": "/home/johnj/ComfyUI/output/WanVideoWrapper_I2V_00004.mp4" + } + } + } + }, + { + "id": 97, + "type": "VHS_LoadVideo", + "pos": [ + -257.8260498046875, + 26.503103256225586 + ], + "size": [ + 247.455078125, + 551.455078125 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 147 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "wolf_interpolated.mp4", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "AnimateDiff", + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "wolf_interpolated.mp4", + "type": "input", + "format": "video/mp4", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 95, + "type": "WanVideoEncode", + "pos": [ + 280.9074401855469, + -116.20671844482422 + ], + "size": [ + 315, + 242 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 158 + }, + { + "name": "image", + "type": "IMAGE", + "link": 148 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 132 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoEncode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + 0, + 1.0000000000000002 + ] + }, + { + "id": 104, + "type": "ImageBlur", + "pos": [ + 89.50601959228516, + 217.970947265625 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 147 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 148, + 149 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlur" + }, + "widgets_values": [ + 4, + 1 + ] + }, + { + "id": 103, + "type": "ImageConcatMulti", + "pos": [ + 2055.855224609375, + -543.0326538085938 + ], + "size": [ + 315, + 150 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 149 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 145 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 146 + ] + } + ], + "properties": {}, + "widgets_values": [ + 2, + "right", + false, + null + ] + }, + { + "id": 33, + "type": "Note", + "pos": [ + 170, + -1150 + ], + "size": [ + 359.0753479003906, + 88 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Models:\nhttps://huggingface.co/Kijai/WanVideo_comfy/tree/main" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "WanVideoControlEmbeds", + "pos": [ + 784.249755859375, + -17.228534698486328 + ], + "size": [ + 289.79998779296875, + 102 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "latents", + "type": "LATENT", + "link": 132 + }, + { + "name": "fun_ref_image", + "shape": 7, + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "slot_index": 0, + "links": [ + 151 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoControlEmbeds" + }, + "widgets_values": [ + 0, + 0.7 + ] + }, + { + "id": 52, + "type": "WanVideoTeaCache", + "pos": [ + 1334.42626953125, + -588.2476196289062 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "cache_args", + "type": "CACHEARGS", + "links": [ + 153 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTeaCache" + }, + "widgets_values": [ + 0.1, + 1, + -1, + "offload_device", + "true", + "e" + ] + }, + { + "id": 64, + "type": "WanVideoTorchCompileSettings", + "pos": [ + -276.8500671386719, + -1050.6326904296875 + ], + "size": [ + 390.5999755859375, + 202 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "slot_index": 0, + "links": [ + 157 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ] + }, + { + "id": 109, + "type": "WanVideoVAELoaderMultiGPU", + "pos": [ + -246.82862854003906, + -338.9017333984375 + ], + "size": [ + 368.710693359375, + 106 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "links": [ + 158, + 159 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoVAELoaderMultiGPU" + }, + "widgets_values": [ + "wan_2.1_vae.safetensors", + "cuda:0", + "bf16" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 111, + "type": "LoadWanVideoT5TextEncoderMultiGPU", + "pos": [ + 334.64263916015625, + -356.91937255859375 + ], + "size": [ + 348.9195251464844, + 130 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wan_t5_model", + "type": "WANTEXTENCODER", + "links": [ + 161 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVideoT5TextEncoderMultiGPU" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "cuda:0", + "disabled" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 110, + "type": "WanVideoTextEncodeMultiGPU", + "pos": [ + 750.6812744140625, + -392.9539794921875 + ], + "size": [ + 400, + 234 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "t5", + "shape": 7, + "type": "WANTEXTENCODER", + "link": 161 + }, + { + "name": "model_to_offload", + "shape": 7, + "type": "WANVIDEOMODEL", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 160 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTextEncodeMultiGPU" + }, + "widgets_values": [ + "video of a wolf", + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + "cuda:0", + false, + false + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 108, + "type": "WanVideoSamplerMultiGPU", + "pos": [ + 1322.1207275390625, + -347.0115051269531 + ], + "size": [ + 327.80859375, + 902 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 155 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 151 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 160 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": 153 + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": null + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 154 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "WanVideoSamplerMultiGPU" + }, + "widgets_values": [ + 30, + 6, + 5, + 0, + "fixed", + false, + "unipc", + 0, + 1, + false, + "comfy", + 0, + -1, + false + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 107, + "type": "WanVideoModelLoaderMultiGPU", + "pos": [ + 295.29010009765625, + -723.5640258789062 + ], + "size": [ + 400.63916015625, + 274 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": 157 + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": null + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 156 + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 155 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoModelLoaderMultiGPU" + }, + "widgets_values": [ + "wan2.1_t2v_1.3B_bf16.safetensors", + "bf16", + "disabled", + "cuda:1", + "sdpa" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 98, + "type": "WanVideoLoraSelect", + "pos": [ + -161.7538604736328, + -680.8223876953125 + ], + "size": [ + 315, + 150 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 156 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoLoraSelect" + }, + "widgets_values": [ + "wan2.1-1.3b-control-lora-tile-v1.1_comfy.safetensors", + 1, + false, + true + ] + } + ], + "links": [ + [ + 132, + 95, + 0, + 96, + 0, + "LATENT" + ], + [ + 145, + 28, + 0, + 103, + 1, + "IMAGE" + ], + [ + 146, + 103, + 0, + 30, + 0, + "IMAGE" + ], + [ + 147, + 97, + 0, + 104, + 0, + "IMAGE" + ], + [ + 148, + 104, + 0, + 95, + 1, + "IMAGE" + ], + [ + 149, + 104, + 0, + 103, + 0, + "IMAGE" + ], + [ + 151, + 96, + 0, + 108, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 153, + 52, + 0, + 108, + 6, + "CACHEARGS" + ], + [ + 154, + 108, + 0, + 28, + 1, + "LATENT" + ], + [ + 155, + 107, + 0, + 108, + 0, + "WANVIDEOMODEL" + ], + [ + 156, + 98, + 0, + 107, + 2, + "WANVIDLORA" + ], + [ + 157, + 64, + 0, + 107, + 0, + "WANCOMPILEARGS" + ], + [ + 158, + 109, + 0, + 95, + 0, + "WANVAE" + ], + [ + 159, + 109, + 0, + 28, + 0, + "WANVAE" + ], + [ + 160, + 110, + 0, + 108, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 161, + 111, + 0, + 110, + 0, + "WANTEXTENCODER" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.814027493868451, + "offset": [ + 714.8717021081054, + 1253.694882650272 + ] + }, + "node_versions": { + "ComfyUI-WanVideoWrapper": "5a2383621a05825d0d0437781afcb8552d9590fd", + "ComfyUI-VideoHelperSuite": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "comfy-core": "0.3.26", + "ComfyUI-KJNodes": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f" + }, + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "frontendVersion": "1.25.11" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/wanvideo.py b/wanvideo.py index a49c2b0..3ddb952 100644 --- a/wanvideo.py +++ b/wanvideo.py @@ -158,6 +158,10 @@ class WanVideoVAELoader: result = original_loader.loadmodel(model_name, precision, compile_args) + # Attach device info to VAE object for downstream nodes + if result and len(result) > 0: + result[0].load_device = selected_device + logging.info(f"[MultiGPU] WanVideo VAE loaded on {selected_device}") return result else: @@ -383,6 +387,33 @@ class WanVideoSampler: original_sampler = NODE_CLASS_MAPPINGS["WanVideoSampler"]() return original_sampler.process(model, **kwargs) +class WanVideoVACEEncode: + @classmethod + def INPUT_TYPES(s): + from nodes import NODE_CLASS_MAPPINGS + original_types = NODE_CLASS_MAPPINGS["WanVideoVACEEncode"].INPUT_TYPES() + return original_types + + RETURN_TYPES = ("LATENT",) + RETURN_NAMES = ("latent",) + FUNCTION = "process" + CATEGORY = "WanVideoWrapper" + DESCRIPTION = "MultiGPU-aware VACE encoder that uses device from input VAE" + + def process(self, vae, **kwargs): + # Get device from VAE object + vae_device = vae.load_device + logging.info(f"[MultiGPU] WanVideoVACEEncode: Processing on device: {vae_device}") + + # Patch all WanVideo modules to use the VAE's device + for module_name in sys.modules.keys(): + if 'WanVideoWrapper' in module_name and hasattr(sys.modules[module_name], 'device'): + sys.modules[module_name].device = vae_device + + from nodes import NODE_CLASS_MAPPINGS + original_encoder = NODE_CLASS_MAPPINGS["WanVideoVACEEncode"]() + return original_encoder.process(vae, **kwargs) + class WanVideoBlockSwap: @classmethod def INPUT_TYPES(s):