From cae24fc7e042adb023fb8a97c193d22953edb028 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Tue, 19 Aug 2025 20:02:24 +0300 Subject: [PATCH 01/24] uni3c adjustment --- nodes.py | 27 +++++++++++++++++++-------- uni3c/nodes.py | 18 +++++++++--------- wanvideo/modules/model.py | 2 +- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/nodes.py b/nodes.py index 4d37441..8b6eb06 100644 --- a/nodes.py +++ b/nodes.py @@ -2094,7 +2094,7 @@ class WanVideoSampler: if uni3c_embeds is not None: transformer.controlnet = uni3c_embeds["controlnet"] pcd_data = { - "render_latent": uni3c_embeds["render_latent"].to(dtype), + "render_latent": uni3c_embeds["render_latent"], "render_mask": uni3c_embeds["render_mask"], "camera_embedding": uni3c_embeds["camera_embedding"], "controlnet_weight": uni3c_embeds["controlnet_weight"], @@ -3047,7 +3047,7 @@ class WanVideoSampler: if uni3c_embeds is not None: transformer.controlnet = uni3c_embeds["controlnet"] pcd_data = { - "render_latent": uni3c_embeds["render_latent"].to(dtype), + "render_latent": uni3c_embeds["render_latent"], "render_mask": uni3c_embeds["render_mask"], "camera_embedding": uni3c_embeds["camera_embedding"], "controlnet_weight": uni3c_embeds["controlnet_weight"], @@ -3066,7 +3066,7 @@ class WanVideoSampler: if mode == "infinitetalk": cond_image = original_images[:, :, current_condframe_index:current_condframe_index+1] log.info(f"current_condframe_index: {current_condframe_index}") - log.info(f"audio_start_idx: {audio_start_idx}") + log.info(f"audio_indices: {audio_start_idx}-{audio_end_idx}|{clip_length}") if multitalk_embeds is not None: audio_embs = [] # split audio with window size @@ -3079,7 +3079,20 @@ class WanVideoSampler: if uni3c_embeds is not None: vae.to(device) - render_latent = vae.encode(original_images[:, :, audio_start_idx:audio_end_idx].to(device, vae.dtype), device=device, tiled=tiled_vae).to(dtype) + print("original_images", original_images.shape) + # Pad original_images if needed + num_frames = original_images.shape[2] + required_frames = audio_end_idx - audio_start_idx + if audio_end_idx > num_frames: + pad_len = audio_end_idx - num_frames + last_frame = original_images[:, :, -1:].repeat(1, 1, pad_len, 1, 1) + padded_images = torch.cat([original_images, last_frame], dim=2) + else: + padded_images = original_images + render_latent = vae.encode( + padded_images[:, :, audio_start_idx:audio_end_idx].to(device, vae.dtype), + device=device, tiled=tiled_vae + ).to(dtype) pcd_data['render_latent'] = render_latent h, w = cond_image.shape[-2], cond_image.shape[-1] @@ -3309,10 +3322,8 @@ class WanVideoSampler: del noise, latent if force_offload: - if model["manual_offloading"]: - transformer.to(offload_device) - mm.soft_empty_cache() - gc.collect() + if not model["auto_cpu_offload"]: + offload_transformer(transformer) try: print_memory(device) torch.cuda.reset_peak_memory_stats(device) diff --git a/uni3c/nodes.py b/uni3c/nodes.py index 3720356..3015ad6 100644 --- a/uni3c/nodes.py +++ b/uni3c/nodes.py @@ -138,12 +138,12 @@ class WanVideoUni3C_embeds: def INPUT_TYPES(s): return {"required": { "controlnet": ("WANVIDEOCONTROLNET",), - "render_latent": ("LATENT",), "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}), "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Start percent of the steps to apply the controlnet"}), "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "End percent of the steps to apply the controlnet"}), }, "optional": { + "render_latent": ("LATENT",), "render_mask": ("MASK", {"tooltip": "NOT IMPLEMENTED!"}), }, } @@ -153,16 +153,16 @@ class WanVideoUni3C_embeds: FUNCTION = "process" CATEGORY = "WanVideoWrapper" - def process(self, controlnet, render_latent, strength, start_percent, end_percent, render_mask=None): + def process(self, controlnet, strength, start_percent, end_percent, render_latent=None, render_mask=None): device = mm.get_torch_device() - latent_mask = None - - latents = render_latent["samples"] - nframe = latents.shape[2] * 4 - height = latents.shape[3] * 8 - width = latents.shape[4] * 8 + latent_mask = latents = None + if render_latent is not None: + latents = render_latent["samples"] + # nframe = latents.shape[2] * 4 + # height = latents.shape[3] * 8 + # width = latents.shape[4] * 8 if render_mask is not None: raise NotImplementedError("render_mask is not implemented at this time") @@ -224,7 +224,7 @@ class WanVideoUni3C_embeds: "controlnet_weight": strength, "start": start_percent, "end": end_percent, - "render_latent": latents.to(device), + "render_latent": latents, "render_mask": latent_mask, "camera_embedding": None } diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index b17da61..3271a51 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -1595,7 +1595,7 @@ class WanModel(torch.nn.Module): #uni3c controlnet if pcd_data is not None: hidden_states = x[0].unsqueeze(0).clone().float() - render_latent = torch.cat([hidden_states[:, :20], pcd_data["render_latent"]], dim=1) + render_latent = torch.cat([hidden_states[:, :20], pcd_data["render_latent"].to(x[0].dtype)], dim=1) # embeddings if control_lora_enabled: From 04a99fd4e25b6ae7b9bf2591e3a935835a294500 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Tue, 19 Aug 2025 20:19:01 +0300 Subject: [PATCH 02/24] Add tooltips --- multitalk/nodes.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/multitalk/nodes.py b/multitalk/nodes.py index e4ee204..f48fd2e 100644 --- a/multitalk/nodes.py +++ b/multitalk/nodes.py @@ -75,8 +75,8 @@ class MultiTalkWav2VecEmbeds: return {"required": { "wav2vec_model": ("WAV2VECMODEL",), "audio_1": ("AUDIO",), - "normalize_loudness": ("BOOLEAN", {"default": True}), - "num_frames": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 1}), + "normalize_loudness": ("BOOLEAN", {"default": True, "tooltip": "Normalize the audio loudness to -23 LUFS"}), + "num_frames": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 1, "tooltip": "The total frame count to generate."}), "fps": ("FLOAT", {"default": 25.0, "min": 1.0, "max": 60.0, "step": 0.1}), "audio_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "tooltip": "Strength of the audio conditioning"}), "audio_cfg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "tooltip": "When not 1.0, an extra model pass without audio conditioning is done: slower inference but more motion is allowed"}), @@ -243,11 +243,11 @@ class WanVideoImageToVideoMultiTalk: def INPUT_TYPES(s): return {"required": { "vae": ("WANVAE",), - "width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8, "tooltip": "Width of the image to encode"}), - "height": ("INT", {"default": 480, "min": 64, "max": 29048, "step": 8, "tooltip": "Height of the image to encode"}), - "frame_window_size": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 4, "tooltip": "Number of frames to encode"}), - "motion_frame": ("INT", {"default": 25, "min": 1, "max": 10000, "step": 1, "tooltip": "Driven frame length used in the long video generation."}), - "force_offload": ("BOOLEAN", {"default": True}), + "width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8, "tooltip": "Width of the generation"}), + "height": ("INT", {"default": 480, "min": 64, "max": 29048, "step": 8, "tooltip": "Height of the generation"}), + "frame_window_size": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 4, "tooltip": "The number of frames to process at once, should be a value the model is generally good at."}), + "motion_frame": ("INT", {"default": 25, "min": 1, "max": 10000, "step": 1, "tooltip": "Driven frame length used in the long video generation. Basically the overlap length."}), + "force_offload": ("BOOLEAN", {"default": False, "tooltip": "Whether to force offload the model within the loop for VAE operations, enable if you encounter memory issues."}), "colormatch": ( [ 'disabled', @@ -258,11 +258,11 @@ class WanVideoImageToVideoMultiTalk: 'hm-mvgd-hm', 'hm-mkl-hm', ], { - "default": 'disabled' - }), + "default": 'disabled', "tooltip": "Color matching method to use between the windows" + },), }, "optional": { - "start_image": ("IMAGE", {"tooltip": "Image to encode"}), + "start_image": ("IMAGE", {"tooltip": "Images to encode"}), "tiled_vae": ("BOOLEAN", {"default": False, "tooltip": "Use tiled VAE encoding for reduced memory use"}), "clip_embeds": ("WANVIDIMAGE_CLIPEMBEDS", {"tooltip": "Clip vision encoded image"}), "mode": ([ @@ -276,6 +276,7 @@ class WanVideoImageToVideoMultiTalk: RETURN_NAMES = ("image_embeds",) FUNCTION = "process" CATEGORY = "WanVideoWrapper" + DESCRIPTION = "Enables Multi/InfiniteTalk long video generation sampling method, the video is created in windows with overlapping frames. Not compatible or necessary to be used with context windows and many other features besides Multi/InfiniteTalk." def process(self, vae, width, height, frame_window_size, motion_frame, force_offload, colormatch, start_image=None, tiled_vae=False, clip_embeds=None, mode="multitalk"): From 5375a01fb6dae02db244b8848087e87fbd69f44a Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Tue, 19 Aug 2025 20:30:59 +0300 Subject: [PATCH 03/24] Add error --- nodes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nodes.py b/nodes.py index 8b6eb06..a2ad21a 100644 --- a/nodes.py +++ b/nodes.py @@ -1614,6 +1614,10 @@ class WanVideoSampler: transformer = compile_model(transformer, model["compile_args"]) multitalk_sampling = image_embeds.get("multitalk_sampling", False) + + if multitalk_sampling and context_options is not None: + raise Exception("context_options are not compatible or necessary with 'WanVideoImageToVideoMultiTalk' node, since it's already an alternative method that creates the video in a loop.") + if not multitalk_sampling and scheduler == "multitalk": raise Exception("multitalk scheduler is only for multitalk sampling when using ImagetoVideoMultiTalk -node") From 35adf24d978e21cb9f1ec7e58195ac7882bbd4e4 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:11:34 +0300 Subject: [PATCH 04/24] Support Fun 5B control --- nodes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nodes.py b/nodes.py index a2ad21a..2c215f7 100644 --- a/nodes.py +++ b/nodes.py @@ -826,9 +826,9 @@ class WanVideoImageToVideoEncode: H = height W = width - lat_h = H // 8 - lat_w = W // 8 - + lat_h = H // vae.upsampling_factor + lat_w = W // vae.upsampling_factor + num_frames = ((num_frames - 1) // 4) * 4 + 1 two_ref_images = start_image is not None and end_image is not None @@ -1746,7 +1746,7 @@ class WanVideoSampler: control_embeds = image_embeds.get("control_embeds", None) if control_embeds is not None: - if transformer.in_dim not in [52, 48, 36, 32]: + if transformer.in_dim not in [148, 52, 48, 36, 32]: raise ValueError("Control signal only works with Fun-Control model") control_latents = control_embeds.get("control_images", None) @@ -1843,7 +1843,7 @@ class WanVideoSampler: patcher = apply_lora(patcher, device, device, low_mem_load=False, control_lora=True) patcher.model.is_patched = True else: - if transformer.in_dim not in [48, 36, 32, 52]: + if transformer.in_dim not in [148, 48, 36, 32, 52]: raise ValueError("Control signal only works with Fun-Control model") image_cond = torch.zeros_like(noise).to(device) #fun control if transformer.in_dim == 52 or transformer.control_adapter is not None: #fun 2.2 control From b9f32539fd51ecc2069e8ac8575768712816b7e3 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:11:45 +0300 Subject: [PATCH 05/24] This should never have been here... --- uni3c/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uni3c/nodes.py b/uni3c/nodes.py index 3015ad6..064b6dc 100644 --- a/uni3c/nodes.py +++ b/uni3c/nodes.py @@ -21,7 +21,7 @@ class WanVideoUni3C_ControlnetLoader: "model": (folder_paths.get_filename_list("controlnet"), {"tooltip": "These models are loaded from the 'ComfyUI/models/controlnet' -folder",}), "base_precision": (["fp32", "bf16", "fp16"], {"default": "fp16"}), - "quantization": (['disabled', 'fp8_e4m3fn', 'fp8_e4m3fn_fast', 'fp8_e5m2', 'fp8_e4m3fn_fast_no_ffn'], {"default": 'disabled', "tooltip": "optional quantization method"}), + "quantization": (['disabled', 'fp8_e4m3fn', 'fp8_e5m2'], {"default": 'disabled', "tooltip": "optional quantization method"}), "load_device": (["main_device", "offload_device"], {"default": "main_device", "tooltip": "Initial device to load the model to, NOT recommended with the larger models unless you have 48GB+ VRAM"}), "attention_mode": ([ "sdpa", From ff779c91714d8ee3484cd4119b082c72a1734b72 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:18:53 +0300 Subject: [PATCH 06/24] Update nodes.py --- nodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodes.py b/nodes.py index 2c215f7..093c7b5 100644 --- a/nodes.py +++ b/nodes.py @@ -1846,7 +1846,7 @@ class WanVideoSampler: if transformer.in_dim not in [148, 48, 36, 32, 52]: raise ValueError("Control signal only works with Fun-Control model") image_cond = torch.zeros_like(noise).to(device) #fun control - if transformer.in_dim == 52 or transformer.control_adapter is not None: #fun 2.2 control + if transformer.in_dim in [148, 52] or transformer.control_adapter is not None: #fun 2.2 control mask_latents = torch.tile( torch.zeros_like(noise[:1]), [4, 1, 1, 1] ) @@ -1860,7 +1860,7 @@ class WanVideoSampler: control_start_percent = control_embeds.get("start_percent", 0.0) control_end_percent = control_embeds.get("end_percent", 1.0) else: - if transformer.in_dim == 36: #fun inp + if transformer.in_dim in [148, 52]: #fun inp mask_latents = torch.tile( torch.zeros_like(noise[:1]), [4, 1, 1, 1] ) From a1220f7f36f152a0da3a841bd1a00152bc32cf09 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 13:33:44 +0300 Subject: [PATCH 07/24] Handle InfiniteTalk first frame when not using the looping sampling --- multitalk/nodes.py | 12 +++++++----- nodes.py | 22 +++++++++++++++++++++- nodes_model_loading.py | 4 +++- wanvideo/modules/model.py | 2 ++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/multitalk/nodes.py b/multitalk/nodes.py index f48fd2e..dea56d4 100644 --- a/multitalk/nodes.py +++ b/multitalk/nodes.py @@ -52,7 +52,8 @@ class MultiTalkModelLoader: multitalk = { "proj_model": multitalk_proj_model, "sd": sd, - "is_gguf": model_path.endswith(".gguf") + "is_gguf": model_path.endswith(".gguf"), + "model_type": "InfiniteTalk" if "infinite" in model.lower() else "MultiTalk", } return (multitalk,) @@ -266,9 +267,10 @@ class WanVideoImageToVideoMultiTalk: "tiled_vae": ("BOOLEAN", {"default": False, "tooltip": "Use tiled VAE encoding for reduced memory use"}), "clip_embeds": ("WANVIDIMAGE_CLIPEMBEDS", {"tooltip": "Clip vision encoded image"}), "mode": ([ + "auto", "multitalk", "infinitetalk" - ], {"default": "multitalk", "tooltip": "The sampling strategy to use in the long video generation loop, should match the model used"}) + ], {"default": "auto", "tooltip": "The sampling strategy to use in the long video generation loop, should match the model used"}) } } @@ -321,7 +323,7 @@ NODE_CLASS_MAPPINGS = { } NODE_DISPLAY_NAME_MAPPINGS = { - "MultiTalkModelLoader": "MultiTalk Model Loader", - "MultiTalkWav2VecEmbeds": "MultiTalk Wav2Vec Embeds", - "WanVideoImageToVideoMultiTalk": "WanVideo Image To Video MultiTalk" + "MultiTalkModelLoader": "Multi/InfiniteTalk Model Loader", + "MultiTalkWav2VecEmbeds": "Multi/InfiniteTalk Wav2Vec Embeds", + "WanVideoImageToVideoMultiTalk": "WanVideo Image To Video Multi/InfiniteTalk" } \ No newline at end of file diff --git a/nodes.py b/nodes.py index 093c7b5..0eb3773 100644 --- a/nodes.py +++ b/nodes.py @@ -2069,7 +2069,7 @@ class WanVideoSampler: # extra latents (Pusa) and 5b latents_to_insert = add_index = None - if (extra_latents := image_embeds.get("extra_latents", None)) is not None: + if (extra_latents := image_embeds.get("extra_latents", None)) is not None and transformer.multitalk_model_type.lower() != "infinitetalk": all_indices = [] for entry in extra_latents: add_index = entry["index"] @@ -2725,6 +2725,15 @@ class WanVideoSampler: latent_flipped = torch.flip(latent, dims=[1]) latent_model_input_flipped = latent_flipped.to(device) + #InfiniteTalk first frame handling + if (extra_latents is not None + and not multitalk_sampling + and transformer.multitalk_model_type=="InfiniteTalk"): + for entry in extra_latents: + add_index = entry["index"] + num_extra_frames = entry["samples"].shape[2] + latent[:, add_index:add_index+num_extra_frames] = entry["samples"].to(latent) + latent_model_input = latent.to(device) current_step_percentage = idx / len(timesteps) @@ -3002,6 +3011,8 @@ class WanVideoSampler: #region multitalk elif multitalk_sampling: mode = image_embeds.get("multitalk_mode", "multitalk") + if mode == "auto": + mode = transformer.multitalk_model_type.lower() log.info(f"Multitalk mode: {mode}") original_images = cond_image = image_embeds.get("multitalk_start_image", None) offload = image_embeds.get("force_offload", False) @@ -3399,6 +3410,15 @@ class WanVideoSampler: **scheduler_step_args)[0].squeeze(0) latent_backwards = torch.flip(latent_backwards, dims=[1]) latent = latent * 0.5 + latent_backwards * 0.5 + + #InfiniteTalk first frame handling + if (extra_latents is not None + and not multitalk_sampling + and transformer.multitalk_model_type=="InfiniteTalk"): + for entry in extra_latents: + add_index = entry["index"] + num_extra_frames = entry["samples"].shape[2] + latent[:, add_index:add_index+num_extra_frames] = entry["samples"].to(latent) if freeinit_args is not None: current_latent = latent.clone() diff --git a/nodes_model_loading.py b/nodes_model_loading.py index 1c7ce4c..117151b 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -1014,6 +1014,7 @@ class WanVideoModelLoader: if multitalk_model is not None: if multitalk_model["is_gguf"] and not gguf: raise ValueError("Multitalk/InfiniteTalk model is a GGUF model, main model also has to be a GGUF model.") + multitalk_model_type = multitalk_model.get("model_type", "MultiTalk") # init audio module from .multitalk.multitalk import SingleStreamMultiAttention from .wanvideo.modules.model import WanRMSNorm, WanLayerNorm @@ -1033,8 +1034,9 @@ class WanVideoModelLoader: attention_mode=attention_mode, ) block.norm_x = WanLayerNorm(dim, transformer.eps, elementwise_affine=True) if norm_input_visual else nn.Identity() - log.info("MultiTalk model detected, patching model...") + log.info(f"{multitalk_model_type} detected, patching model...") transformer.audio_proj = multitalk_model["proj_model"] + transformer.multitalk_model_type = multitalk_model_type sd.update(multitalk_model["sd"]) # Additional cond latents diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index 3271a51..ca6f752 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -1260,6 +1260,8 @@ class WanModel(torch.nn.Module): self.video_attention_split_steps = [] self.lora_scheduling_enabled = False + self.multitalk_model_type = None + # embeddings self.patch_embedding = nn.Conv3d( in_dim, dim, kernel_size=patch_size, stride=patch_size) From 8b4546896ed8a6a801885660b417d2b19dab688e Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 13:39:46 +0300 Subject: [PATCH 08/24] Update nodes.py --- multitalk/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multitalk/nodes.py b/multitalk/nodes.py index dea56d4..a7672bc 100644 --- a/multitalk/nodes.py +++ b/multitalk/nodes.py @@ -325,5 +325,5 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "MultiTalkModelLoader": "Multi/InfiniteTalk Model Loader", "MultiTalkWav2VecEmbeds": "Multi/InfiniteTalk Wav2Vec Embeds", - "WanVideoImageToVideoMultiTalk": "WanVideo Image To Video Multi/InfiniteTalk" + "WanVideoImageToVideoMultiTalk": "WanVideo Long I2V Multi/InfiniteTalk" } \ No newline at end of file From edf2a24519e8a06fa4661d30423675432ef43c74 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 13:58:14 +0300 Subject: [PATCH 09/24] Fix encode progress bar --- wanvideo/wan_video_vae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wanvideo/wan_video_vae.py b/wanvideo/wan_video_vae.py index 320a262..7ac9a05 100644 --- a/wanvideo/wan_video_vae.py +++ b/wanvideo/wan_video_vae.py @@ -1037,7 +1037,7 @@ class VideoVAE_(nn.Module): feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) out = torch.cat([out, out_], 2) - pbar.update(1) + pbar.update(iter_) mu = self.conv1(out).chunk(2, dim=1)[0] mu = (mu - self.mean.to(mu)) * self.inv_std.to(mu) From 053f26ce828eabd0c7a67fde6138742db5442cd9 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:40:13 +0300 Subject: [PATCH 10/24] Multi/InfiniteTalk v2v --- nodes.py | 72 ++++++++++++++++++++++++++++++++------- nodes_model_loading.py | 1 + wanvideo/wan_video_vae.py | 61 +++++++++++++++++++-------------- 3 files changed, 96 insertions(+), 38 deletions(-) diff --git a/nodes.py b/nodes.py index 0eb3773..ec5b869 100644 --- a/nodes.py +++ b/nodes.py @@ -2033,7 +2033,7 @@ class WanVideoSampler: context = get_context_scheduler(context_schedule) # vid2vid - if samples is not None: + if samples is not None and not multitalk_sampling: saved_generator_state = samples.get("generator_state", None) if saved_generator_state is not None: seed_g.set_state(saved_generator_state) @@ -2623,7 +2623,7 @@ class WanVideoSampler: # diff diff prep masks = None - if samples is not None and mask is not None: + if not multitalk_sampling and samples is not None and mask is not None: mask = 1 - mask thresholds = torch.arange(len(timesteps), dtype=original_image.dtype) / len(timesteps) thresholds = thresholds.unsqueeze(1).unsqueeze(1).unsqueeze(1).unsqueeze(1).to(device) @@ -2704,7 +2704,7 @@ class WanVideoSampler: try: pbar = ProgressBar(len(timesteps)) #region main loop start - for idx, t in enumerate(tqdm(timesteps)): + for idx, t in enumerate(tqdm(timesteps, disable=multitalk_sampling)): if flowedit_args is not None: if idx < skip_steps: continue @@ -3071,17 +3071,16 @@ class WanVideoSampler: } estimated_iterations = total_frames // (frame_num - motion_frame) + 1 - loop_pbar = tqdm(total=estimated_iterations, desc="Generating video clips") + loop_pbar = tqdm(total=estimated_iterations, desc="Total progress", position=1, leave=True) callback = prepare_callback(patcher, estimated_iterations) audio_embedding = multitalk_audio_embedding human_num = len(audio_embedding) audio_embs = None while True: # start video generation iteratively + cur_motion_frames_latent_num = int(1 + (cur_motion_frames_num-1) // 4) if mode == "infinitetalk": cond_image = original_images[:, :, current_condframe_index:current_condframe_index+1] - log.info(f"current_condframe_index: {current_condframe_index}") - log.info(f"audio_indices: {audio_start_idx}-{audio_end_idx}|{clip_length}") if multitalk_embeds is not None: audio_embs = [] # split audio with window size @@ -3094,7 +3093,6 @@ class WanVideoSampler: if uni3c_embeds is not None: vae.to(device) - print("original_images", original_images.shape) # Pad original_images if needed num_frames = original_images.shape[2] required_frames = audio_end_idx - audio_start_idx @@ -3117,6 +3115,34 @@ class WanVideoSampler: noise = torch.randn( 16, (frame_num - 1) // 4 + 1, lat_h, lat_w, dtype=torch.float32, device=torch.device("cpu"), generator=seed_g).to(device) + + if samples is not None: + input_samples = samples["samples"].squeeze(0).to(noise) + # Calculate the correct slice based on current iteration + if is_first_clip: + latent_start_idx = 0 + latent_end_idx = noise.shape[1] + else: + new_frames_per_iteration = frame_num - motion_frame + new_latent_frames_per_iteration = ((new_frames_per_iteration - 1) // 4 + 1) + latent_start_idx = iteration_count * new_latent_frames_per_iteration + latent_end_idx = latent_start_idx + noise.shape[1] + + # Check if we have enough frames in input_samples + if latent_end_idx > input_samples.shape[1]: + # We need more frames than available - pad the input_samples at the end + pad_length = latent_end_idx - input_samples.shape[1] + last_frame = input_samples[:, -1:].repeat(1, pad_length, 1, 1) + input_samples = torch.cat([input_samples, last_frame], dim=1) + input_samples = input_samples[:, latent_start_idx:latent_end_idx] + + assert input_samples.shape[1] == noise.shape[1], f"Slice mismatch: {input_samples.shape[1]} vs {noise.shape[1]}" + + if add_noise_to_samples: + latent_timestep = timesteps[0] + noise = noise * latent_timestep / 1000 + (1 - latent_timestep / 1000) * input_samples + else: + noise = input_samples # get mask msk = torch.ones(1, frame_num, lat_h, lat_w, device=device) @@ -3138,8 +3164,7 @@ class WanVideoSampler: # encode vae.to(device) - y = vae.encode(padding_frames_pixels_values, device=device, tiled=tiled_vae).to(dtype) - cur_motion_frames_latent_num = int(1 + (cur_motion_frames_num-1) // 4) + y = vae.encode(padding_frames_pixels_values, device=device, tiled=tiled_vae, pbar=False).to(dtype) if mode == "multitalk": latent_motion_frames = y[:, :, :cur_motion_frames_latent_num][0] # C T H W @@ -3162,6 +3187,24 @@ class WanVideoSampler: else: sample_scheduler, timesteps = get_scheduler(scheduler, steps, shift, device, transformer.dim, flowedit_args, denoise_strength, sigmas=sigmas) + steps = len(timesteps) + if end_step != -1 and start_step >= end_step: + raise ValueError("start_step must be less than end_step") + if denoise_strength < 1.0: + if start_step != 0: + raise ValueError("start_step must be 0 when denoise_strength is used") + start_step = steps - int(steps * denoise_strength) - 1 + if (end_step != -1 or end_step >= steps): + timesteps = timesteps[:end_step] + sample_scheduler.sigmas = sample_scheduler.sigmas[:end_step+1] + if start_step > 0: + timesteps = timesteps[start_step:] + sample_scheduler.sigmas = sample_scheduler.sigmas[start_step:] + + if sample_scheduler is not None: + if hasattr(sample_scheduler, 'timesteps'): + sample_scheduler.timesteps = timesteps + transformed_timesteps = [] for t in timesteps: t_tensor = torch.tensor([t.item()], device=device) @@ -3214,8 +3257,8 @@ class WanVideoSampler: elif model["manual_offloading"]: transformer.to(device) - comfy_pbar = ProgressBar(len(timesteps)-1) - for i in tqdm(range(len(timesteps)-1)): + sampling_pbar = tqdm(total=len(timesteps)-1, desc=f"Sampling audio indices {audio_start_idx}-{audio_end_idx}", position=0, leave=True) + for i in range(len(timesteps)-1): timestep = timesteps[i] latent_model_input = latent.to(device) if mode == "infinitetalk": @@ -3229,6 +3272,8 @@ class WanVideoSampler: timestep, idx, y.squeeze(0), clip_embeds, control_latents, vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, cache_state=self.cache_state, multitalk_audio_embeds=audio_embs) + sampling_pbar.update(1) + if callback is not None: callback_latent = (latent_model_input.to(device) - noise_pred.to(device) * t.to(device) / 1000).detach().permute(1,0,2,3) callback(iteration_count, callback_latent, None, estimated_iterations) @@ -3261,13 +3306,14 @@ class WanVideoSampler: x0 = latent.to(device) del latent_model_input, timestep - comfy_pbar.update(1) if offload: transformer.to(offload_device) vae.to(device) - videos = vae.decode(x0.unsqueeze(0).to(vae.dtype), device=device, tiled=tiled_vae) + videos = vae.decode(x0.unsqueeze(0).to(vae.dtype), device=device, tiled=tiled_vae, pbar=False) vae.to(offload_device) + + sampling_pbar.close() # cache generated samples videos = torch.stack(videos).cpu() # B C T H W diff --git a/nodes_model_loading.py b/nodes_model_loading.py index 117151b..1c9e16b 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -1098,6 +1098,7 @@ class WanVideoModelLoader: #for name, param in transformer.named_parameters(): # print(name, param.dtype, param.device, param.shape) pbar.update_absolute(param_count) + pbar.update_absolute(0) comfy_model.diffusion_model = transformer comfy_model.load_device = transformer_load_device diff --git a/wanvideo/wan_video_vae.py b/wanvideo/wan_video_vae.py index 7ac9a05..472dad7 100644 --- a/wanvideo/wan_video_vae.py +++ b/wanvideo/wan_video_vae.py @@ -1019,10 +1019,11 @@ class VideoVAE_(nn.Module): return mu - def encode(self, x): + def encode(self, x, pbar=True): self.clear_cache() ## cache - pbar = ProgressBar(x.shape[2]) + if pbar: + pbar = ProgressBar(x.shape[2]) t = x.shape[2] iter_ = 1 + (t - 1) // 4 @@ -1037,10 +1038,13 @@ class VideoVAE_(nn.Module): feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) out = torch.cat([out, out_], 2) - pbar.update(iter_) + if pbar: + pbar.update(iter_) mu = self.conv1(out).chunk(2, dim=1)[0] mu = (mu - self.mean.to(mu)) * self.inv_std.to(mu) + if pbar: + pbar.update_absolute(0) return mu @@ -1076,11 +1080,12 @@ class VideoVAE_(nn.Module): - def decode(self, z): + def decode(self, z, pbar=True): self.clear_cache() # z: [b,c,t,h,w] - pbar = ProgressBar(z.shape[2]) - + if pbar: + pbar = ProgressBar(z.shape[2]) + z = z / self.inv_std.to(z) + self.mean.to(z) iter_ = z.shape[2] @@ -1096,7 +1101,11 @@ class VideoVAE_(nn.Module): feat_cache=self._feat_map, feat_idx=self._conv_idx) out = torch.cat([out, out_], 2) # may add tensor offload - pbar.update(1) + + if pbar: + pbar.update(1) + if pbar: + pbar.update_absolute(0) return out def reparameterize(self, mu, log_var): @@ -1167,7 +1176,7 @@ class WanVideoVAE(nn.Module): return mask - def tiled_decode(self, hidden_states, device, tile_size, tile_stride): + def tiled_decode(self, hidden_states, device, tile_size, tile_stride, pbar=True): _, _, T, H, W = hidden_states.shape size_h, size_w = tile_size stride_h, stride_w = tile_stride @@ -1187,8 +1196,8 @@ class WanVideoVAE(nn.Module): out_T = T * 4 - 3 weight = torch.zeros((1, 1, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) values = torch.zeros((1, 3, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) - - pbar = ProgressBar(len(tasks)) + if pbar: + pbar = ProgressBar(len(tasks)) for h, h_, w, w_ in tqdm(tasks, desc="VAE decoding"): hidden_states_batch = hidden_states[:, :, :, h:h_, w:w_].to(computation_device) hidden_states_batch = self.model.decode(hidden_states_batch).to(data_device) @@ -1215,13 +1224,14 @@ class WanVideoVAE(nn.Module): target_h: target_h + hidden_states_batch.shape[3], target_w: target_w + hidden_states_batch.shape[4], ] += mask - pbar.update(1) + if pbar: + pbar.update(1) values = values / weight values = values.float().clamp_(-1, 1) return values - def tiled_encode(self, video, device, tile_size, tile_stride, end_=False): + def tiled_encode(self, video, device, tile_size, tile_stride, end_=False, pbar=True): _, _, T, H, W = video.shape if tile_size is None and tile_stride is None: @@ -1248,8 +1258,8 @@ class WanVideoVAE(nn.Module): out_T += 1 weight = torch.zeros((1, 1, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) values = torch.zeros((1, self.z_dim, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) - - pbar = ProgressBar(len(tasks)) + if pbar: + pbar = ProgressBar(len(tasks)) for h, h_, w, w_ in tqdm(tasks, desc="VAE encoding"): hidden_states_batch = video[:, :, :, h:h_, w:w_].to(computation_device) if end_: @@ -1279,21 +1289,22 @@ class WanVideoVAE(nn.Module): target_h: target_h + hidden_states_batch.shape[3], target_w: target_w + hidden_states_batch.shape[4], ] += mask - pbar.update(1) + if pbar: + pbar.update(1) values = values / weight values = values.float() return values - def single_encode(self, video, device): + def single_encode(self, video, device, pbar=True): video = video.to(device) - x = self.model.encode(video) + x = self.model.encode(video, pbar=pbar) return x.float() - def single_decode(self, hidden_state, device): + def single_decode(self, hidden_state, device, pbar=True): hidden_state = hidden_state.to(device) - video = self.model.decode(hidden_state) + video = self.model.decode(hidden_state, pbar=pbar) return video def double_encode(self, video, device): @@ -1308,36 +1319,36 @@ class WanVideoVAE(nn.Module): video = self.model.decode_2(hidden_state) return video - def encode(self, videos, device, tiled=False,end_=False, tile_size=None, tile_stride=None): + def encode(self, videos, device, tiled=False,end_=False, tile_size=None, tile_stride=None, pbar=True): videos = [video.to("cpu") for video in videos] hidden_states = [] for video in videos: video = video.unsqueeze(0) if tiled: - hidden_state = self.tiled_encode(video, device, tile_size, tile_stride, end_=end_) + hidden_state = self.tiled_encode(video, device, tile_size, tile_stride, end_=end_, pbar=pbar) else: if end_: hidden_state = self.double_encode(video, device) else: - hidden_state = self.single_encode(video, device) + hidden_state = self.single_encode(video, device, pbar=pbar) hidden_state = hidden_state.squeeze(0) hidden_states.append(hidden_state) hidden_states = torch.stack(hidden_states) return hidden_states - def decode(self, hidden_states, device, tiled=False, end_=False, tile_size=(34, 34), tile_stride=(18, 16)): + def decode(self, hidden_states, device, tiled=False, end_=False, tile_size=(34, 34), tile_stride=(18, 16), pbar=True): hidden_states = [hidden_state.to("cpu") for hidden_state in hidden_states] videos = [] for hidden_state in hidden_states: hidden_state = hidden_state.unsqueeze(0) if tiled: - video = self.tiled_decode(hidden_state, device, tile_size, tile_stride) + video = self.tiled_decode(hidden_state, device, tile_size, tile_stride, pbar=pbar) else: if end_: video = self.double_decode(hidden_state, device) else: - video = self.single_decode(hidden_state, device) + video = self.single_decode(hidden_state, device, pbar=pbar) video = video.squeeze(0) videos.append(video) return videos From a0cf985bc3c03f6bb2a11eee60265036653a79fe Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:51:23 +0300 Subject: [PATCH 11/24] Update nodes.py --- nodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodes.py b/nodes.py index ec5b869..570d3dd 100644 --- a/nodes.py +++ b/nodes.py @@ -3170,9 +3170,9 @@ class WanVideoSampler: latent_motion_frames = y[:, :, :cur_motion_frames_latent_num][0] # C T H W else: if is_first_clip: - latent_motion_frames = vae.encode(cond_image.to(device, vae.dtype), device=device, tiled=tiled_vae).to(dtype) + latent_motion_frames = vae.encode(cond_image.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) else: - latent_motion_frames = vae.encode(cond_frame.to(device, vae.dtype), device=device, tiled=tiled_vae).to(dtype) + latent_motion_frames = vae.encode(cond_frame.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) latent_motion_frames = latent_motion_frames[0] vae.to(offload_device) From b39d6abc1bdf0069cf20d0832fb9782790fd544d Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 17:11:46 +0300 Subject: [PATCH 12/24] Fix bug --- wanvideo/modules/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index ca6f752..01da52c 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -1260,7 +1260,7 @@ class WanModel(torch.nn.Module): self.video_attention_split_steps = [] self.lora_scheduling_enabled = False - self.multitalk_model_type = None + self.multitalk_model_type = "none" # embeddings self.patch_embedding = nn.Conv3d( From 6db2d113ba696ed5268a082cad543279eede2fba Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 17:12:04 +0300 Subject: [PATCH 13/24] Update nodes.py --- nodes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nodes.py b/nodes.py index 570d3dd..ef89013 100644 --- a/nodes.py +++ b/nodes.py @@ -3050,7 +3050,7 @@ class WanVideoSampler: is_first_clip = True arrive_last_frame = False cur_motion_frames_num = 1 - audio_start_idx = iteration_count = 0 + audio_start_idx = iteration_count = step_iteration_count= 0 audio_end_idx = audio_start_idx + clip_length indices = (torch.arange(4 + 1) - 2) * 1 current_condframe_index = 0 @@ -3273,10 +3273,12 @@ class WanVideoSampler: cache_state=self.cache_state, multitalk_audio_embeds=audio_embs) sampling_pbar.update(1) - + if callback is not None: callback_latent = (latent_model_input.to(device) - noise_pred.to(device) * t.to(device) / 1000).detach().permute(1,0,2,3) - callback(iteration_count, callback_latent, None, estimated_iterations) + callback(step_iteration_count, callback_latent, None, estimated_iterations*(len(timesteps)-1)) + + step_iteration_count += 1 # update latent if scheduler == "multitalk": From 68ef3ac468dcc5c8da1369f119267643a2ce93aa Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 18:58:55 +0300 Subject: [PATCH 14/24] Fix 2.2VAE progressbar issue --- wanvideo/wan_video_vae.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/wanvideo/wan_video_vae.py b/wanvideo/wan_video_vae.py index 472dad7..c26acd3 100644 --- a/wanvideo/wan_video_vae.py +++ b/wanvideo/wan_video_vae.py @@ -1022,10 +1022,10 @@ class VideoVAE_(nn.Module): def encode(self, x, pbar=True): self.clear_cache() ## cache - if pbar: - pbar = ProgressBar(x.shape[2]) t = x.shape[2] iter_ = 1 + (t - 1) // 4 + if pbar: + pbar = ProgressBar(iter_) for i in range(iter_): self._enc_conv_idx = [0] @@ -1039,7 +1039,7 @@ class VideoVAE_(nn.Module): feat_idx=self._enc_conv_idx) out = torch.cat([out, out_], 2) if pbar: - pbar.update(iter_) + pbar.update(1) mu = self.conv1(out).chunk(2, dim=1)[0] mu = (mu - self.mean.to(mu)) * self.inv_std.to(mu) @@ -1083,12 +1083,10 @@ class VideoVAE_(nn.Module): def decode(self, z, pbar=True): self.clear_cache() # z: [b,c,t,h,w] - if pbar: - pbar = ProgressBar(z.shape[2]) - z = z / self.inv_std.to(z) + self.mean.to(z) - iter_ = z.shape[2] + if pbar: + pbar = ProgressBar(iter_) x = self.conv2(z) for i in range(iter_): self._conv_idx = [0] @@ -1408,11 +1406,13 @@ class VideoVAE38_(VideoVAE_): attn_scales, self.temperal_upsample, dropout) - def encode(self, x): + def encode(self, x, pbar=False): self.clear_cache() x = patchify(x, patch_size=2) t = x.shape[2] iter_ = 1 + (t - 1) // 4 + if pbar: + pbar = ProgressBar(iter_) for i in range(iter_): self._enc_conv_idx = [0] if i == 0: @@ -1424,6 +1424,8 @@ class VideoVAE38_(VideoVAE_): feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) out = torch.cat([out, out_], 2) + if pbar: + pbar.update(1) mu = self.conv1(out).chunk(2, dim=1)[0] mu = (mu - self.mean.to(mu)) * self.inv_std.to(mu) @@ -1432,12 +1434,13 @@ class VideoVAE38_(VideoVAE_): return mu - def decode(self, z): + def decode(self, z, pbar=False): self.clear_cache() - z = z / self.inv_std.to(z) + self.mean.to(z) iter_ = z.shape[2] + if pbar: + pbar = ProgressBar(iter_) x = self.conv2(z) for i in range(iter_): self._conv_idx = [0] @@ -1451,6 +1454,8 @@ class VideoVAE38_(VideoVAE_): feat_cache=self._feat_map, feat_idx=self._conv_idx) out = torch.cat([out, out_], 2) + if pbar: + pbar.update(1) out = unpatchify(out, patch_size=2) self.clear_cache() return out From 70bced06f4637c9eda7a2dab218c171ec98050f7 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 21:08:25 +0300 Subject: [PATCH 15/24] Return exact audio frame count if num_frames above it --- fantasytalking/nodes.py | 4 ++-- multitalk/nodes.py | 24 +++++++++++++++++++++--- nodes.py | 8 ++++++-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/fantasytalking/nodes.py b/fantasytalking/nodes.py index 2458407..928aa1b 100644 --- a/fantasytalking/nodes.py +++ b/fantasytalking/nodes.py @@ -20,8 +20,8 @@ class DownloadAndLoadWav2VecModel: "required": { "model": ( [ - "facebook/wav2vec2-base-960h", - "TencentGameMate/chinese-wav2vec2-base" + "TencentGameMate/chinese-wav2vec2-base", + "facebook/wav2vec2-base-960h" ], ), diff --git a/multitalk/nodes.py b/multitalk/nodes.py index a7672bc..628c5d2 100644 --- a/multitalk/nodes.py +++ b/multitalk/nodes.py @@ -91,8 +91,8 @@ class MultiTalkWav2VecEmbeds: } } - RETURN_TYPES = ("MULTITALK_EMBEDS", "AUDIO", ) - RETURN_NAMES = ("multitalk_embeds", "audio", ) + RETURN_TYPES = ("MULTITALK_EMBEDS", "AUDIO", "INT", ) + RETURN_NAMES = ("multitalk_embeds", "audio", "num_frames", ) FUNCTION = "process" CATEGORY = "WanVideoWrapper" @@ -231,12 +231,30 @@ class MultiTalkWav2VecEmbeds: offset += w.shape[-1] out_audio = {"waveform": mixed, "sample_rate": sr} + # Calculate actual frames based on audio duration + actual_num_frames = num_frames + if len(audio_outputs) > 0: + if multi_audio_type == "para": + # For parallel mode, use the longest audio duration + max_audio_duration = max([ao["waveform"].shape[-1] / sr for ao in audio_outputs]) + actual_frames_from_audio = int(max_audio_duration * fps) + else: # "add" + # For sequential mode, use the total audio duration + total_audio_duration = sum([ao["waveform"].shape[-1] / sr for ao in audio_outputs]) + actual_frames_from_audio = int(total_audio_duration * fps) + + # Use the smaller of requested frames or actual audio frames + actual_num_frames = min(num_frames, actual_frames_from_audio) + + if actual_frames_from_audio < num_frames: + log.info(f"[MultiTalk] Audio duration ({actual_frames_from_audio} frames) is shorter than requested ({num_frames} frames). Using {actual_num_frames} frames.") + # Debug: log final mixed audio length and mode total_samples_raw = sum([ao["waveform"].shape[-1] for ao in audio_outputs]) log.info(f"[MultiTalk] total raw duration = {total_samples_raw/sr:.3f}s") log.info(f"[MultiTalk] multi_audio_type={multi_audio_type} | final waveform shape={out_audio['waveform'].shape} | length={out_audio['waveform'].shape[-1]} samples | seconds={out_audio['waveform'].shape[-1]/sr:.3f}s (expected {'sum' if multi_audio_type=='add' else 'max'} of raw)") - return (multitalk_embeds, out_audio) + return (multitalk_embeds, out_audio, actual_num_frames) class WanVideoImageToVideoMultiTalk: diff --git a/nodes.py b/nodes.py index ef89013..d956ec5 100644 --- a/nodes.py +++ b/nodes.py @@ -2616,8 +2616,9 @@ class WanVideoSampler: from .latent_preview import prepare_callback #custom for tiny VAE previews callback = prepare_callback(patcher, len(timesteps)) - log.info(f"Input sequence length: {seq_len}") - log.info(f"Sampling {(latent_video_length-1) * 4 + 1} frames at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") + if not multitalk_sampling: + log.info(f"Input sequence length: {seq_len}") + log.info(f"Sampling {(latent_video_length-1) * 4 + 1} frames at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") intermediate_device = device @@ -3077,6 +3078,9 @@ class WanVideoSampler: audio_embedding = multitalk_audio_embedding human_num = len(audio_embedding) audio_embs = None + + log.info(f"Sampling {total_frames} frames in {estimated_iterations} windows, at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") + while True: # start video generation iteratively cur_motion_frames_latent_num = int(1 + (cur_motion_frames_num-1) // 4) if mode == "infinitetalk": From 655e6557eba9c4b64896a82fa533fb61a059a7dc Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 21:13:49 +0300 Subject: [PATCH 16/24] Update nodes_model_loading.py --- nodes_model_loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes_model_loading.py b/nodes_model_loading.py index 1c9e16b..866fa9f 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -1074,7 +1074,7 @@ class WanVideoModelLoader: dtype = torch.float8_e5m2 else: dtype = base_dtype - params_to_keep = {"norm", "bias", "time_in", "patch_embedding", "time_", "img_emb", "modulation", "text_embedding", "adapter", "add", "ref_conv"} + params_to_keep = {"norm", "bias", "time_in", "patch_embedding", "time_", "img_emb", "modulation", "text_embedding", "adapter", "add", "ref_conv", "audio_proj"} if not lora_low_mem_load: log.info("Using accelerate to load and assign model weights to device...") param_count = sum(1 for _ in transformer.named_parameters()) From 1edc6ddb32f05664ee54ec3ee02381e0d41f0a64 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 21:31:25 +0300 Subject: [PATCH 17/24] Create wanvideo_I2V_InfiniteTalk_example_01.json --- .../wanvideo_I2V_InfiniteTalk_example_01.json | 2429 +++++++++++++++++ 1 file changed, 2429 insertions(+) create mode 100644 example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json diff --git a/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json b/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json new file mode 100644 index 0000000..1a2ac56 --- /dev/null +++ b/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json @@ -0,0 +1,2429 @@ +{ + "id": "8b7a9a57-2303-4ef5-9fc2-bf41713bd1fc", + "revision": 0, + "last_node_id": 299, + "last_link_id": 540, + "nodes": [ + { + "id": 120, + "type": "MultiTalkModelLoader", + "pos": [ + 592.73583984375, + -2520.13525390625 + ], + "size": [ + 511.7737121582031, + 66.51400756835938 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "MULTITALKMODEL", + "links": [ + 355 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "MultiTalkModelLoader" + }, + "widgets_values": [ + "WanVideo\\InfiniteTalk\\InfiniteTalk\\Wan2_1-InfiniteTalk_Single_Q8.gguf" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 239, + "type": "MarkdownNote", + "pos": [ + 623.767822265625, + -2378.682861328125 + ], + "size": [ + 452.8328857421875, + 214.95587158203125 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": {}, + "widgets_values": [ + "You can't mix GGUF MultiTalk with non-GGUF main model, but you can mix different GGUF Qtypes with eachother.\n\nGGUF:\n\n[https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main](https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main)\n\n[https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk)\n\nFp8:\n\n[https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled](https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled/tree/main/InfiniteTalk)\n\nfp16:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk)\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 129, + "type": "WanVideoVAELoader", + "pos": [ + 1379.0877685546875, + -3038.031494140625 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 436 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoVAELoader" + }, + "widgets_values": [ + "wanvideo\\Wan2_1_VAE_bf16.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 240, + "type": "SetNode", + "pos": [ + 1761.82666015625, + -3007.7861328125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "link": 436 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_VAE", + "properties": { + "previousName": "VAE" + }, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 260, + "type": "SetNode", + "pos": [ + 1843.7393798828125, + -2738.34326171875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "link": 463 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_wanmodel", + "properties": { + "previousName": "wanmodel" + }, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 238, + "type": "CLIPVisionLoader", + "pos": [ + 1406.407470703125, + -2358.7119140625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 466 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 264, + "type": "SetNode", + "pos": [ + 1711.6546630859375, + -2336.708984375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "link": 466 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_clip_vision_model", + "properties": { + "previousName": "clip_vision_model" + }, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 247, + "type": "SetNode", + "pos": [ + 2497.053955078125, + -2320.50537109375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 441 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_width", + "properties": { + "previousName": "width" + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 248, + "type": "SetNode", + "pos": [ + 2495.547607421875, + -2143.893310546875 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 442 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_height", + "properties": { + "previousName": "height" + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 159, + "type": "AudioCrop", + "pos": [ + 676.1865234375, + -1132.495361328125 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 26, + "mode": 4, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 257 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 301, + 448 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioCrop" + }, + "widgets_values": [ + "0:02", + "2:00" + ] + }, + { + "id": 170, + "type": "AudioSeparation", + "pos": [ + 1064.037353515625, + -1347.1170654296875 + ], + "size": [ + 270, + 166 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 301 + } + ], + "outputs": [ + { + "name": "Bass", + "type": "AUDIO", + "links": null + }, + { + "name": "Drums", + "type": "AUDIO", + "links": null + }, + { + "name": "Other", + "type": "AUDIO", + "links": null + }, + { + "name": "Vocals", + "type": "AUDIO", + "links": [ + 335 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioSeparation" + }, + "widgets_values": [ + "linear", + 10, + 0.1 + ] + }, + { + "id": 137, + "type": "DownloadAndLoadWav2VecModel", + "pos": [ + 1400.8470458984375, + -1411.5140380859375 + ], + "size": [ + 330.96728515625, + 106 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "links": [ + 334 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "DownloadAndLoadWav2VecModel" + }, + "widgets_values": [ + "TencentGameMate/chinese-wav2vec2-base", + "fp16", + "main_device" + ] + }, + { + "id": 253, + "type": "SetNode", + "pos": [ + 1002.430908203125, + -1102.810302734375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "link": 448 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_input_audio", + "properties": { + "previousName": "input_audio" + }, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 282, + "type": "GetNode", + "pos": [ + 1040.8619384765625, + -1797.7919921875 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 495 + ] + } + ], + "title": "Get_height", + "properties": {}, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 261, + "type": "GetNode", + "pos": [ + 3279.65771484375, + -2056.792724609375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "links": [ + 464 + ] + } + ], + "title": "Get_wanmodel", + "properties": {}, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 138, + "type": "WanVideoLoraSelect", + "pos": [ + 589.2518920898438, + -2738.91064453125 + ], + "size": [ + 503.4073486328125, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 349 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoLoraSelect" + }, + "widgets_values": [ + "WanVideo\\Lightx2v\\lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 1, + false, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 125, + "type": "LoadAudio", + "pos": [ + 536.47412109375, + -1357.4945068359375 + ], + "size": [ + 404.9698791503906, + 136 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 257 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "hush.mp3", + null, + null + ] + }, + { + "id": 134, + "type": "WanVideoBlockSwap", + "pos": [ + 587.2186279296875, + -3026.727294921875 + ], + "size": [ + 281.404296875, + 202 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "links": [ + 362 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 20, + false, + false, + true, + 0, + 1, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 245, + "type": "INTConstant", + "pos": [ + 2401.922607421875, + -2438.11865234375 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Width", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 246, + "type": "INTConstant", + "pos": [ + 2404.592529296875, + -2258.70654296875 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 442 + ] + } + ], + "title": "Height", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 237, + "type": "WanVideoClipVisionEncode", + "pos": [ + 2538.0830078125, + -1204.498779296875 + ], + "size": [ + 280.9771423339844, + 262 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 467 + }, + { + "name": "image_1", + "type": "IMAGE", + "link": 539 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "negative_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_CLIPEMBEDS", + "links": [ + 433 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoClipVisionEncode" + }, + "widgets_values": [ + 1, + 1, + "center", + "average", + true, + 0, + 0.5 + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 265, + "type": "GetNode", + "pos": [ + 2549.920166015625, + -1268.2757568359375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 467 + ] + } + ], + "title": "Get_clip_vision_model", + "properties": {}, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 242, + "type": "GetNode", + "pos": [ + 3670.50830078125, + -1982.0732421875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 438 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 270, + "type": "INTConstant", + "pos": [ + 2655.538330078125, + -2261.7578125 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 472 + ] + } + ], + "title": "Max frames", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 1000 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 294, + "type": "SetNode", + "pos": [ + 1751.949462890625, + -1184.374267578125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 519 + } + ], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 520 + ] + } + ], + "title": "Set_actual_audio_frames", + "properties": { + "previousName": "actual_audio_frames" + }, + "widgets_values": [ + "actual_audio_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 293, + "type": "PreviewAny", + "pos": [ + 1772.05078125, + -1110.6668701171875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 520 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [] + }, + { + "id": 271, + "type": "SetNode", + "pos": [ + 2660.10888671875, + -2152.97998046875 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 472 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_max_frames", + "properties": { + "previousName": "max_frames" + }, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 194, + "type": "MultiTalkWav2VecEmbeds", + "pos": [ + 1400.6778564453125, + -1237.9234619140625 + ], + "size": [ + 291.08203125, + 278 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "link": 334 + }, + { + "name": "audio_1", + "type": "AUDIO", + "link": 335 + }, + { + "name": "audio_2", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_3", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_4", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "ref_target_masks", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 529 + } + ], + "outputs": [ + { + "name": "multitalk_embeds", + "type": "MULTITALK_EMBEDS", + "links": [ + 357 + ] + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "num_frames", + "type": "INT", + "links": [ + 519 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3d7801cee4c8e3106078dd9b9f146caee95069ba", + "Node name for S&R": "MultiTalkWav2VecEmbeds" + }, + "widgets_values": [ + true, + 400, + 25, + 1, + 1, + "para" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 263, + "type": "Note", + "pos": [ + 1719.4892578125, + -965.56591796875 + ], + "size": [ + 344.82928466796875, + 143.7332305908203 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "increase audio_scale for stronger effect\n\naudio_cfg is only used when sampler is using cfg as well\n\nnum_frames is the maximum frame count to generate, if the value is higher than your audio length would be, the audio length is used " + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "WanVideoModelLoader", + "pos": [ + 1121.4686279296875, + -2764.419921875 + ], + "size": [ + 668.1777954101562, + 314 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 362 + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 349 + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": 355 + }, + { + "name": "fantasyportrait_model", + "shape": 7, + "type": "FANTASYPORTRAITMODEL", + "link": null + }, + { + "name": "extra_model", + "shape": 7, + "type": "VACEPATH", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 463 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoModelLoader" + }, + "widgets_values": [ + "WanVideo\\wan2.1-i2v-14b-480p-Q8_0.gguf", + "fp16_fast", + "disabled", + "offload_device", + "sageattn" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 177, + "type": "WanVideoTorchCompileSettings", + "pos": [ + 970.6024169921875, + -3048.0654296875 + ], + "size": [ + 342.74609375, + 202 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "links": [] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "f3614e6720744247f3211d60f7b9333f43572384", + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 272, + "type": "GetNode", + "pos": [ + 1174.7862548828125, + -1059.2822265625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 529 + ] + } + ], + "title": "Get_max_frames", + "properties": {}, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 130, + "type": "WanVideoDecode", + "pos": [ + 3675.3330078125, + -1893.888671875 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 438 + }, + { + "name": "samples", + "type": "LATENT", + "link": 530 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 493 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 131, + "type": "VHS_VideoCombine", + "pos": [ + 3689.34814453125, + -1589.9613037109375 + ], + "size": [ + 991.5499877929688, + 334 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 493 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 449 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 25, + "loop_count": 0, + "filename_prefix": "WanVideo2_1_InfiniteTalk", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_1_InfiniteTalk_00004-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 25, + "workflow": "WanVideo2_1_InfiniteTalk_00004.png", + "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_1_InfiniteTalk_00004-audio.mp4" + } + } + } + }, + { + "id": 254, + "type": "GetNode", + "pos": [ + 3683.283935546875, + -1643.584228515625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 18, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 449 + ] + } + ], + "title": "Get_input_audio", + "properties": {}, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 291, + "type": "GetImageSizeAndCount", + "pos": [ + 2239.1484375, + -1265.372314453125 + ], + "size": [ + 190.86483764648438, + 86 + ], + "flags": { + "collapsed": false + }, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 538 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 539, + 540 + ] + }, + { + "label": "640 width", + "name": "width", + "type": "INT", + "links": [ + 516 + ] + }, + { + "label": "640 height", + "name": "height", + "type": "INT", + "links": [ + 517 + ] + }, + { + "label": "1 count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3", + "Node name for S&R": "GetImageSizeAndCount" + }, + "widgets_values": [] + }, + { + "id": 241, + "type": "WanVideoTextEncodeCached", + "pos": [ + 2281.288330078125, + -1946.174072265625 + ], + "size": [ + 465.5179138183594, + 386.354248046875 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "extender_args", + "shape": 7, + "type": "WANVIDEOPROMPTEXTENDER_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 437 + ] + }, + { + "name": "negative_text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + }, + { + "name": "positive_prompt", + "type": "STRING", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoTextEncodeCached" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "a woman is singing a lullaby", + "bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards", + "disabled", + false, + "gpu" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 192, + "type": "WanVideoImageToVideoMultiTalk", + "pos": [ + 2836.906982421875, + -1288.3162841796875 + ], + "size": [ + 349.6015625, + 266 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 440 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 540 + }, + { + "name": "clip_embeds", + "shape": 7, + "type": "WANVIDIMAGE_CLIPEMBEDS", + "link": 433 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 516 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 517 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 413 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "49430f900bf812a1e281560bafafd04c43dfdf13", + "Node name for S&R": "WanVideoImageToVideoMultiTalk" + }, + "widgets_values": [ + 832, + 480, + 81, + 9, + false, + "disabled", + false, + "infinitetalk" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 244, + "type": "GetNode", + "pos": [ + 2825.2236328125, + -1325.1824951171875 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 440 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 128, + "type": "WanVideoSampler", + "pos": [ + 3279.373046875, + -1999.322021484375 + ], + "size": [ + 315, + 999 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 464 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 413 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 437 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": null + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": 357 + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 530 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 7, + 1.0000000000000002, + 11.000000000000002, + 2, + "fixed", + true, + "dpm++_sde", + 0, + 1, + false, + "comfy", + 0, + -1, + true + ] + }, + { + "id": 283, + "type": "GetNode", + "pos": [ + 1043.0416259765625, + -1848.6282958984375 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 494 + ] + } + ], + "title": "Get_width", + "properties": {}, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 284, + "type": "LoadImage", + "pos": [ + 638.8305053710938, + -1938.324951171875 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 496 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "woman.png", + "image" + ] + }, + { + "id": 281, + "type": "ImageResizeKJv2", + "pos": [ + 1211.8646240234375, + -1936.6402587890625 + ], + "size": [ + 270, + 336 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 496 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 494 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 495 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 538 + ] + }, + { + "name": "width", + "type": "INT", + "links": [] + }, + { + "name": "height", + "type": "INT", + "links": [] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 832, + 480, + "lanczos", + "crop", + "0, 0, 0", + "center", + 16, + "cpu", + "Output: 1 x 640 x 640 | 4.69MB" + ] + }, + { + "id": 299, + "type": "Note", + "pos": [ + 1424.5263671875, + -2250.27001953125 + ], + "size": [ + 290.9361267089844, + 88 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Clip vision is not strictly necessary\n\nAny I2V model should work, MAGREF can be interesting to play with as well." + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 257, + 125, + 0, + 159, + 0, + "AUDIO" + ], + [ + 301, + 159, + 0, + 170, + 0, + "AUDIO" + ], + [ + 334, + 137, + 0, + 194, + 0, + "WAV2VECMODEL" + ], + [ + 335, + 170, + 3, + 194, + 1, + "AUDIO" + ], + [ + 349, + 138, + 0, + 122, + 2, + "WANVIDLORA" + ], + [ + 355, + 120, + 0, + 122, + 6, + "MULTITALKMODEL" + ], + [ + 357, + 194, + 0, + 128, + 15, + "MULTITALK_EMBEDS" + ], + [ + 362, + 134, + 0, + 122, + 1, + "BLOCKSWAPARGS" + ], + [ + 413, + 192, + 0, + 128, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 433, + 237, + 0, + 192, + 2, + "WANVIDIMAGE_CLIPEMBEDS" + ], + [ + 436, + 129, + 0, + 240, + 0, + "*" + ], + [ + 437, + 241, + 0, + 128, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 438, + 242, + 0, + 130, + 0, + "WANVAE" + ], + [ + 440, + 244, + 0, + 192, + 0, + "WANVAE" + ], + [ + 441, + 245, + 0, + 247, + 0, + "*" + ], + [ + 442, + 246, + 0, + 248, + 0, + "*" + ], + [ + 448, + 159, + 0, + 253, + 0, + "*" + ], + [ + 449, + 254, + 0, + 131, + 1, + "AUDIO" + ], + [ + 463, + 122, + 0, + 260, + 0, + "*" + ], + [ + 464, + 261, + 0, + 128, + 0, + "WANVIDEOMODEL" + ], + [ + 466, + 238, + 0, + 264, + 0, + "*" + ], + [ + 467, + 265, + 0, + 237, + 0, + "CLIP_VISION" + ], + [ + 472, + 270, + 0, + 271, + 0, + "*" + ], + [ + 493, + 130, + 0, + 131, + 0, + "IMAGE" + ], + [ + 494, + 283, + 0, + 281, + 2, + "INT" + ], + [ + 495, + 282, + 0, + 281, + 3, + "INT" + ], + [ + 496, + 284, + 0, + 281, + 0, + "IMAGE" + ], + [ + 516, + 291, + 1, + 192, + 3, + "INT" + ], + [ + 517, + 291, + 2, + 192, + 4, + "INT" + ], + [ + 519, + 194, + 2, + 294, + 0, + "*" + ], + [ + 520, + 294, + 0, + 293, + 0, + "*" + ], + [ + 529, + 272, + 0, + 194, + 6, + "INT" + ], + [ + 530, + 128, + 0, + 130, + 1, + "LATENT" + ], + [ + 538, + 281, + 0, + 291, + 0, + "IMAGE" + ], + [ + 539, + 291, + 0, + 237, + 1, + "IMAGE" + ], + [ + 540, + 291, + 0, + 192, + 1, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 480.1058044433594, + -3147.42529296875, + 1615.13037109375, + 1070.3359375 + ], + "color": "#88A", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Audio", + "bounding": [ + 462.57562255859375, + -1529.8046875, + 1656.1551513671875, + 753.9797973632812 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "Input image", + "bounding": [ + 466.7610778808594, + -2043.424560546875, + 1652.696533203125, + 495.40948486328125 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.45949729863576383, + "offset": [ + 34.34672984465828, + 2847.63895576458 + ] + }, + "frontendVersion": "1.26.3", + "node_versions": { + "ComfyUI-WanVideoWrapper": "0a11c67a0c0062b534178920a0d6dcaa75e7b5fe", + "comfy-core": "0.3.43", + "audio-separation-nodes-comfyui": "31a4567726e035097cc2d1f767767908a6fda2ea", + "ComfyUI-KJNodes": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "comfyui-videohelpersuite": "a7ce59e381934733bfae03b1be029756d6ce936d" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file From ad008d369b89fb98b2af331a6ac7ab8d6ef341f3 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:03:04 +0300 Subject: [PATCH 18/24] Create wanvideo_InfiniteTalk_V2V_example_01.json --- .../wanvideo_InfiniteTalk_V2V_example_01.json | 2891 +++++++++++++++++ 1 file changed, 2891 insertions(+) create mode 100644 example_workflows/wanvideo_InfiniteTalk_V2V_example_01.json diff --git a/example_workflows/wanvideo_InfiniteTalk_V2V_example_01.json b/example_workflows/wanvideo_InfiniteTalk_V2V_example_01.json new file mode 100644 index 0000000..dd7babc --- /dev/null +++ b/example_workflows/wanvideo_InfiniteTalk_V2V_example_01.json @@ -0,0 +1,2891 @@ +{ + "id": "8b7a9a57-2303-4ef5-9fc2-bf41713bd1fc", + "revision": 0, + "last_node_id": 301, + "last_link_id": 551, + "nodes": [ + { + "id": 234, + "type": "Note", + "pos": [ + 539.4052734375, + -832.4791870117188 + ], + "size": [ + 289.1911926269531, + 100.4728012084961 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "If audio is longer than the video, the result will extend from the last video frame" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 120, + "type": "MultiTalkModelLoader", + "pos": [ + 592.73583984375, + -2520.13525390625 + ], + "size": [ + 511.7737121582031, + 66.51400756835938 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "MULTITALKMODEL", + "links": [ + 355 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "MultiTalkModelLoader" + }, + "widgets_values": [ + "WanVideo\\InfiniteTalk\\InfiniteTalk\\Wan2_1-InfiniteTalk_Single_Q8.gguf" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 239, + "type": "MarkdownNote", + "pos": [ + 623.767822265625, + -2378.682861328125 + ], + "size": [ + 452.8328857421875, + 214.95587158203125 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": {}, + "widgets_values": [ + "You can't mix GGUF MultiTalk with non-GGUF main model, but you can mix different GGUF Qtypes with eachother.\n\nGGUF:\n\n[https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main](https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main)\n\n[https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk)\n\nFp8:\n\n[https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled](https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled/tree/main/InfiniteTalk)\n\nfp16:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk)\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 243, + "type": "GetNode", + "pos": [ + 1715.7088623046875, + -924.8023071289062 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 439 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 129, + "type": "WanVideoVAELoader", + "pos": [ + 1379.0877685546875, + -3038.031494140625 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 436 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoVAELoader" + }, + "widgets_values": [ + "wanvideo\\Wan2_1_VAE_bf16.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 240, + "type": "SetNode", + "pos": [ + 1761.82666015625, + -3007.7861328125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "link": 436 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_VAE", + "properties": { + "previousName": "VAE" + }, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 260, + "type": "SetNode", + "pos": [ + 1843.7393798828125, + -2738.34326171875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "link": 463 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_wanmodel", + "properties": { + "previousName": "wanmodel" + }, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 238, + "type": "CLIPVisionLoader", + "pos": [ + 1406.407470703125, + -2358.7119140625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 466 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 264, + "type": "SetNode", + "pos": [ + 1711.6546630859375, + -2336.708984375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "link": 466 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_clip_vision_model", + "properties": { + "previousName": "clip_vision_model" + }, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 247, + "type": "SetNode", + "pos": [ + 2497.053955078125, + -2320.50537109375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 441 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_width", + "properties": { + "previousName": "width" + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 248, + "type": "SetNode", + "pos": [ + 2495.547607421875, + -2143.893310546875 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 442 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_height", + "properties": { + "previousName": "height" + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 170, + "type": "AudioSeparation", + "pos": [ + 1099.1485595703125, + -1843.935302734375 + ], + "size": [ + 270, + 166 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 301 + } + ], + "outputs": [ + { + "name": "Bass", + "type": "AUDIO", + "links": null + }, + { + "name": "Drums", + "type": "AUDIO", + "links": null + }, + { + "name": "Other", + "type": "AUDIO", + "links": null + }, + { + "name": "Vocals", + "type": "AUDIO", + "links": [ + 335 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioSeparation" + }, + "widgets_values": [ + "linear", + 10, + 0.1 + ] + }, + { + "id": 137, + "type": "DownloadAndLoadWav2VecModel", + "pos": [ + 1435.958251953125, + -1908.332275390625 + ], + "size": [ + 330.96728515625, + 106 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "links": [ + 334 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "DownloadAndLoadWav2VecModel" + }, + "widgets_values": [ + "TencentGameMate/chinese-wav2vec2-base", + "fp16", + "main_device" + ] + }, + { + "id": 253, + "type": "SetNode", + "pos": [ + 1037.5419921875, + -1599.6285400390625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "link": 448 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_input_audio", + "properties": { + "previousName": "input_audio" + }, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 261, + "type": "GetNode", + "pos": [ + 3379.9423828125, + -1989.2093505859375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "links": [ + 464 + ] + } + ], + "title": "Get_wanmodel", + "properties": {}, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 138, + "type": "WanVideoLoraSelect", + "pos": [ + 589.2518920898438, + -2738.91064453125 + ], + "size": [ + 503.4073486328125, + 150 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 349 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoLoraSelect" + }, + "widgets_values": [ + "WanVideo\\Lightx2v\\lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 1, + false, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 134, + "type": "WanVideoBlockSwap", + "pos": [ + 587.2186279296875, + -3026.727294921875 + ], + "size": [ + 281.404296875, + 202 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "links": [ + 362 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 20, + false, + false, + true, + 0, + 1, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 245, + "type": "INTConstant", + "pos": [ + 2401.922607421875, + -2438.11865234375 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Width", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 246, + "type": "INTConstant", + "pos": [ + 2404.592529296875, + -2258.70654296875 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 442 + ] + } + ], + "title": "Height", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 241, + "type": "WanVideoTextEncodeCached", + "pos": [ + 2799.24609375, + -1790.9327392578125 + ], + "size": [ + 465.5179138183594, + 386.354248046875 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "extender_args", + "shape": 7, + "type": "WANVIDEOPROMPTEXTENDER_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 437 + ] + }, + { + "name": "negative_text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + }, + { + "name": "positive_prompt", + "type": "STRING", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoTextEncodeCached" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "a woman is singing a lullaby", + "bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards", + "disabled", + false, + "gpu" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 254, + "type": "GetNode", + "pos": [ + 3936.826171875, + -1674.468505859375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 449 + ] + } + ], + "title": "Get_input_audio", + "properties": {}, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 249, + "type": "GetNode", + "pos": [ + 897.5274047851562, + -921.2979125976562 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 443, + 511 + ] + } + ], + "title": "Get_height", + "properties": {}, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 250, + "type": "GetNode", + "pos": [ + 895.64990234375, + -977.6257934570312 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 444, + 510 + ] + } + ], + "title": "Get_width", + "properties": {}, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 192, + "type": "WanVideoImageToVideoMultiTalk", + "pos": [ + 2848.897705078125, + -1217.5718994140625 + ], + "size": [ + 349.6015625, + 266 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 440 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 543 + }, + { + "name": "clip_embeds", + "shape": 7, + "type": "WANVIDIMAGE_CLIPEMBEDS", + "link": 433 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 516 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 517 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 413 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "49430f900bf812a1e281560bafafd04c43dfdf13", + "Node name for S&R": "WanVideoImageToVideoMultiTalk" + }, + "widgets_values": [ + 832, + 480, + 81, + 9, + false, + "disabled", + false, + "infinitetalk" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 244, + "type": "GetNode", + "pos": [ + 2865.99169921875, + -1271.2249755859375 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 440 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 237, + "type": "WanVideoClipVisionEncode", + "pos": [ + 2538.0830078125, + -1204.498779296875 + ], + "size": [ + 280.9771423339844, + 262 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 467 + }, + { + "name": "image_1", + "type": "IMAGE", + "link": 542 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "negative_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_CLIPEMBEDS", + "links": [ + 433 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoClipVisionEncode" + }, + "widgets_values": [ + 1, + 1, + "center", + "average", + true, + 0, + 0.5 + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 265, + "type": "GetNode", + "pos": [ + 2549.920166015625, + -1268.2757568359375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 467 + ] + } + ], + "title": "Get_clip_vision_model", + "properties": {}, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 242, + "type": "GetNode", + "pos": [ + 3764.5048828125, + -2040.7044677734375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 18, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 438 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 270, + "type": "INTConstant", + "pos": [ + 2655.538330078125, + -2261.7578125 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 472 + ] + } + ], + "title": "Max frames", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 1000 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 294, + "type": "SetNode", + "pos": [ + 1787.0606689453125, + -1681.1925048828125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 519 + } + ], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 520 + ] + } + ], + "title": "Set_actual_audio_frames", + "properties": { + "previousName": "actual_audio_frames" + }, + "widgets_values": [ + "actual_audio_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 293, + "type": "PreviewAny", + "pos": [ + 1807.1619873046875, + -1607.485107421875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 520 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [] + }, + { + "id": 271, + "type": "SetNode", + "pos": [ + 2660.10888671875, + -2152.97998046875 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 472 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_max_frames", + "properties": { + "previousName": "max_frames" + }, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 263, + "type": "Note", + "pos": [ + 1754.6004638671875, + -1462.3841552734375 + ], + "size": [ + 344.82928466796875, + 143.7332305908203 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "increase audio_scale for stronger effect\n\naudio_cfg is only used when sampler is using cfg as well\n\nnum_frames is the maximum frame count to generate, if the value is higher than your audio length would be, the audio length is used " + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "WanVideoModelLoader", + "pos": [ + 1121.4686279296875, + -2764.419921875 + ], + "size": [ + 668.1777954101562, + 314 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 362 + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 349 + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": 355 + }, + { + "name": "fantasyportrait_model", + "shape": 7, + "type": "FANTASYPORTRAITMODEL", + "link": null + }, + { + "name": "extra_model", + "shape": 7, + "type": "VACEPATH", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 463 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoModelLoader" + }, + "widgets_values": [ + "WanVideo\\wan2.1-i2v-14b-480p-Q8_0.gguf", + "fp16_fast", + "disabled", + "offload_device", + "sageattn" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 177, + "type": "WanVideoTorchCompileSettings", + "pos": [ + 970.6024169921875, + -3048.0654296875 + ], + "size": [ + 342.74609375, + 202 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "links": [] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "f3614e6720744247f3211d60f7b9333f43572384", + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 229, + "type": "WanVideoEncode", + "pos": [ + 1707.7315673828125, + -876.6842041015625 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 439 + }, + { + "name": "image", + "type": "IMAGE", + "link": 417 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 528 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5375a01fb6dae02db244b8848087e87fbd69f44a", + "Node name for S&R": "WanVideoEncode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + 0, + 1 + ] + }, + { + "id": 228, + "type": "VHS_LoadVideo", + "pos": [ + 892.2421264648438, + -853.955810546875 + ], + "size": [ + 247.455078125, + 451.9747314453125 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "custom_width", + "type": "INT", + "widget": { + "name": "custom_width" + }, + "link": 510 + }, + { + "name": "custom_height", + "type": "INT", + "widget": { + "name": "custom_height" + }, + "link": 511 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 416 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "8e4d79471bf1952154768e8435a9300077b534fa", + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "10.mp4", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "Wan", + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "10.mp4", + "type": "input", + "format": "video/mp4", + "force_rate": 0, + "custom_width": null, + "custom_height": 480, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 291, + "type": "GetImageSizeAndCount", + "pos": [ + 2225.578125, + -1042.865478515625 + ], + "size": [ + 190.86483764648438, + 86 + ], + "flags": { + "collapsed": true + }, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 540 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 542, + 543 + ] + }, + { + "label": "640 width", + "name": "width", + "type": "INT", + "links": [ + 516 + ] + }, + { + "label": "640 height", + "name": "height", + "type": "INT", + "links": [ + 517 + ] + }, + { + "label": "1 count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3", + "Node name for S&R": "GetImageSizeAndCount" + }, + "widgets_values": [] + }, + { + "id": 125, + "type": "LoadAudio", + "pos": [ + 571.5848999023438, + -1854.312744140625 + ], + "size": [ + 404.9698791503906, + 136 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 257 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "one-does-not-simply-walk-into-mordor-its-black-gates-are-guarded-by-more-than-just-orcs.mp3", + null, + null + ] + }, + { + "id": 159, + "type": "AudioCrop", + "pos": [ + 711.2973022460938, + -1629.3135986328125 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 257 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 301, + 448 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioCrop" + }, + "widgets_values": [ + "0:02", + "2:00" + ] + }, + { + "id": 272, + "type": "GetNode", + "pos": [ + 1234.6712646484375, + -1557.0916748046875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 544 + ] + } + ], + "title": "Get_max_frames", + "properties": {}, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 130, + "type": "WanVideoDecode", + "pos": [ + 3769.32958984375, + -1952.5198974609375 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 438 + }, + { + "name": "samples", + "type": "LATENT", + "link": 532 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 545 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 230, + "type": "ImageResizeKJv2", + "pos": [ + 1336.364990234375, + -852.6764526367188 + ], + "size": [ + 270, + 336 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 416 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 444 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 443 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 417, + 418, + 547 + ] + }, + { + "name": "width", + "type": "INT", + "links": [] + }, + { + "name": "height", + "type": "INT", + "links": [] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 832, + 480, + "lanczos", + "crop", + "0, 0, 0", + "center", + 16, + "cpu", + "Output: 97 x 640 x 640 | 454.69MB" + ] + }, + { + "id": 231, + "type": "GetImageRangeFromBatch", + "pos": [ + 1340.5838623046875, + -1083.07763671875 + ], + "size": [ + 340.3267517089844, + 102 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 418 + }, + { + "name": "masks", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 540 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "GetImageRangeFromBatch" + }, + "widgets_values": [ + 0, + 1 + ] + }, + { + "id": 300, + "type": "GetImageSizeAndCount", + "pos": [ + 4164.92578125, + -1949.7347412109375 + ], + "size": [ + 190.86483764648438, + 86 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 545 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 549 + ] + }, + { + "label": "640 width", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "640 height", + "name": "height", + "type": "INT", + "links": null + }, + { + "label": "225 count", + "name": "count", + "type": "INT", + "links": [ + 551 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3", + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 301, + "type": "GetImageRangeFromBatch", + "pos": [ + 4187.93701171875, + -1824.931396484375 + ], + "size": [ + 340.3267517089844, + 102 + ], + "flags": { + "collapsed": true + }, + "order": 47, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 549 + }, + { + "name": "masks", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 551 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 550 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "GetImageRangeFromBatch" + }, + "widgets_values": [ + 0, + 1 + ] + }, + { + "id": 131, + "type": "VHS_VideoCombine", + "pos": [ + 3826.9462890625, + -1626.7916259765625 + ], + "size": [ + 725.5770263671875, + 700.7885131835938 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 548 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 449 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 25, + "loop_count": 0, + "filename_prefix": "WanVideo2_1_InfiniteTalk", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_1_InfiniteTalk_00007-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 25, + "workflow": "WanVideo2_1_InfiniteTalk_00007.png", + "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_1_InfiniteTalk_00007-audio.mp4" + } + } + } + }, + { + "id": 299, + "type": "ImageConcatMulti", + "pos": [ + 4190.734375, + -1750.7587890625 + ], + "size": [ + 270, + 150 + ], + "flags": { + "collapsed": true + }, + "order": 48, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 550 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 547 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 548 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3" + }, + "widgets_values": [ + 2, + "left", + false, + null + ] + }, + { + "id": 128, + "type": "WanVideoSampler", + "pos": [ + 3379.65771484375, + -1931.739013671875 + ], + "size": [ + 315, + 999 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 464 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 413 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 437 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": 528 + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": null + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": 357 + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 532 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 4, + 1.0000000000000002, + 11.000000000000002, + 2, + "fixed", + true, + "dpm++_sde", + 0, + 1, + false, + "comfy", + 2, + -1, + true + ] + }, + { + "id": 194, + "type": "MultiTalkWav2VecEmbeds", + "pos": [ + 1435.7890625, + -1734.74169921875 + ], + "size": [ + 291.08203125, + 278 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "link": 334 + }, + { + "name": "audio_1", + "type": "AUDIO", + "link": 335 + }, + { + "name": "audio_2", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_3", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_4", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "ref_target_masks", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 544 + } + ], + "outputs": [ + { + "name": "multitalk_embeds", + "type": "MULTITALK_EMBEDS", + "links": [ + 357 + ] + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "num_frames", + "type": "INT", + "links": [ + 519 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3d7801cee4c8e3106078dd9b9f146caee95069ba", + "Node name for S&R": "MultiTalkWav2VecEmbeds" + }, + "widgets_values": [ + true, + 400, + 25, + 1.5, + 1, + "para" + ], + "color": "#323", + "bgcolor": "#535" + } + ], + "links": [ + [ + 257, + 125, + 0, + 159, + 0, + "AUDIO" + ], + [ + 301, + 159, + 0, + 170, + 0, + "AUDIO" + ], + [ + 334, + 137, + 0, + 194, + 0, + "WAV2VECMODEL" + ], + [ + 335, + 170, + 3, + 194, + 1, + "AUDIO" + ], + [ + 349, + 138, + 0, + 122, + 2, + "WANVIDLORA" + ], + [ + 355, + 120, + 0, + 122, + 6, + "MULTITALKMODEL" + ], + [ + 357, + 194, + 0, + 128, + 15, + "MULTITALK_EMBEDS" + ], + [ + 362, + 134, + 0, + 122, + 1, + "BLOCKSWAPARGS" + ], + [ + 413, + 192, + 0, + 128, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 416, + 228, + 0, + 230, + 0, + "IMAGE" + ], + [ + 417, + 230, + 0, + 229, + 1, + "IMAGE" + ], + [ + 418, + 230, + 0, + 231, + 0, + "IMAGE" + ], + [ + 433, + 237, + 0, + 192, + 2, + "WANVIDIMAGE_CLIPEMBEDS" + ], + [ + 436, + 129, + 0, + 240, + 0, + "*" + ], + [ + 437, + 241, + 0, + 128, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 438, + 242, + 0, + 130, + 0, + "WANVAE" + ], + [ + 439, + 243, + 0, + 229, + 0, + "WANVAE" + ], + [ + 440, + 244, + 0, + 192, + 0, + "WANVAE" + ], + [ + 441, + 245, + 0, + 247, + 0, + "*" + ], + [ + 442, + 246, + 0, + 248, + 0, + "*" + ], + [ + 443, + 249, + 0, + 230, + 3, + "INT" + ], + [ + 444, + 250, + 0, + 230, + 2, + "INT" + ], + [ + 448, + 159, + 0, + 253, + 0, + "*" + ], + [ + 449, + 254, + 0, + 131, + 1, + "AUDIO" + ], + [ + 463, + 122, + 0, + 260, + 0, + "*" + ], + [ + 464, + 261, + 0, + 128, + 0, + "WANVIDEOMODEL" + ], + [ + 466, + 238, + 0, + 264, + 0, + "*" + ], + [ + 467, + 265, + 0, + 237, + 0, + "CLIP_VISION" + ], + [ + 472, + 270, + 0, + 271, + 0, + "*" + ], + [ + 510, + 250, + 0, + 228, + 2, + "INT" + ], + [ + 511, + 249, + 0, + 228, + 3, + "INT" + ], + [ + 516, + 291, + 1, + 192, + 3, + "INT" + ], + [ + 517, + 291, + 2, + 192, + 4, + "INT" + ], + [ + 519, + 194, + 2, + 294, + 0, + "*" + ], + [ + 520, + 294, + 0, + 293, + 0, + "*" + ], + [ + 528, + 229, + 0, + 128, + 3, + "LATENT" + ], + [ + 532, + 128, + 0, + 130, + 1, + "LATENT" + ], + [ + 540, + 231, + 0, + 291, + 0, + "IMAGE" + ], + [ + 542, + 291, + 0, + 237, + 1, + "IMAGE" + ], + [ + 543, + 291, + 0, + 192, + 1, + "IMAGE" + ], + [ + 544, + 272, + 0, + 194, + 6, + "INT" + ], + [ + 545, + 130, + 0, + 300, + 0, + "IMAGE" + ], + [ + 547, + 230, + 0, + 299, + 1, + "IMAGE" + ], + [ + 548, + 299, + 0, + 131, + 0, + "IMAGE" + ], + [ + 549, + 300, + 0, + 301, + 0, + "IMAGE" + ], + [ + 550, + 301, + 0, + 299, + 0, + "IMAGE" + ], + [ + 551, + 300, + 3, + 301, + 2, + "INT" + ] + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 480.1058044433594, + -3147.42529296875, + 1615.13037109375, + 1070.3359375 + ], + "color": "#88A", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Video2video", + "bounding": [ + 498.3595886230469, + -1241.007080078125, + 1720.7049560546875, + 926.4686889648438 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Audio", + "bounding": [ + 497.6866149902344, + -2026.6229248046875, + 1656.1551513671875, + 753.9797973632812 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.41772481694160374, + "offset": [ + -263.000991979628, + 2820.474862310073 + ] + }, + "frontendVersion": "1.26.3", + "node_versions": { + "ComfyUI-WanVideoWrapper": "0a11c67a0c0062b534178920a0d6dcaa75e7b5fe", + "comfy-core": "0.3.43", + "audio-separation-nodes-comfyui": "31a4567726e035097cc2d1f767767908a6fda2ea", + "ComfyUI-KJNodes": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "comfyui-videohelpersuite": "a7ce59e381934733bfae03b1be029756d6ce936d" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file From 6d51934ae816e9c87fb9b6183fb644d7fd564943 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:05:25 +0300 Subject: [PATCH 19/24] Update wanvideo_I2V_InfiniteTalk_example_01.json --- example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json b/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json index 1a2ac56..1f215b5 100644 --- a/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json +++ b/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json @@ -1619,7 +1619,7 @@ "widgets_values": [ "umt5-xxl-enc-bf16.safetensors", "bf16", - "a woman is singing a lullaby", + "a man is talking", "bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards", "disabled", false, From 8bd7a6318c2a74dea6f38ece696116d971994ce7 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Thu, 21 Aug 2025 01:39:34 +0300 Subject: [PATCH 20/24] Allow multiple prompts for Multi/InfiniteTalk loop --- nodes.py | 64 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/nodes.py b/nodes.py index d956ec5..dee2891 100644 --- a/nodes.py +++ b/nodes.py @@ -3016,6 +3016,7 @@ class WanVideoSampler: mode = transformer.multitalk_model_type.lower() log.info(f"Multitalk mode: {mode}") original_images = cond_image = image_embeds.get("multitalk_start_image", None) + cond_frame = None offload = image_embeds.get("force_offload", False) tiled_vae = image_embeds.get("tiled_vae", False) frame_num = clip_length = image_embeds.get("num_frames", 81) @@ -3031,19 +3032,13 @@ class WanVideoSampler: if len(multitalk_embeds['audio_features'])==2 and (multitalk_embeds['ref_target_masks'] is None): face_scale = 0.1 x_min, x_max = int(target_h * face_scale), int(target_h * (1 - face_scale)) - background_mask = torch.zeros([target_h, target_w]) - background_mask = torch.zeros([target_h, target_w]) - human_mask1 = torch.zeros([target_h, target_w]) - human_mask2 = torch.zeros([target_h, target_w]) lefty_min, lefty_max = int((target_w//2) * face_scale), int((target_w//2) * (1 - face_scale)) righty_min, righty_max = int((target_w//2) * face_scale + (target_w//2)), int((target_w//2) * (1 - face_scale) + (target_w//2)) + human_mask1, human_mask2 = (torch.zeros([target_h, target_w]) for _ in range(2)) human_mask1[x_min:x_max, lefty_min:lefty_max] = 1 human_mask2[x_min:x_max, righty_min:righty_max] = 1 - background_mask += human_mask1 - background_mask += human_mask2 - human_masks = [human_mask1, human_mask2] - background_mask = torch.where(background_mask > 0, torch.tensor(0), torch.tensor(1)) - human_masks.append(background_mask) + background_mask = torch.where((human_mask1 + human_mask2) > 0, torch.tensor(0), torch.tensor(1)) + human_masks = [human_mask1, human_mask2, background_mask] ref_target_masks = torch.stack(human_masks, dim=0) multitalk_embeds['ref_target_masks'] = ref_target_masks @@ -3084,7 +3079,7 @@ class WanVideoSampler: while True: # start video generation iteratively cur_motion_frames_latent_num = int(1 + (cur_motion_frames_num-1) // 4) if mode == "infinitetalk": - cond_image = original_images[:, :, current_condframe_index:current_condframe_index+1] + cond_image = original_images[:, :, current_condframe_index:current_condframe_index+1] if cond_image is not None else None if multitalk_embeds is not None: audio_embs = [] # split audio with window size @@ -3112,7 +3107,7 @@ class WanVideoSampler: ).to(dtype) pcd_data['render_latent'] = render_latent - h, w = cond_image.shape[-2], cond_image.shape[-1] + h, w = cond_image.shape[-2], cond_image.shape[-1] if cond_image is not None else (target_h, target_w) lat_h, lat_w = h // VAE_STRIDE[1], w // VAE_STRIDE[2] seq_len = ((frame_num - 1) // VAE_STRIDE[0] + 1) * lat_h * lat_w // (PATCH_SIZE[1] * PATCH_SIZE[2]) @@ -3163,25 +3158,28 @@ class WanVideoSampler: mm.soft_empty_cache() # zero padding and vae encode - video_frames = torch.zeros(1, cond_image.shape[1], frame_num-cond_image.shape[2], target_h, target_w, device=device, dtype=vae.dtype) - padding_frames_pixels_values = torch.concat([cond_image.to(device, vae.dtype), video_frames], dim=2) + if cond_image is not None or cond_frame is not None: + video_frames = torch.zeros(1, cond_image.shape[1], frame_num-cond_image.shape[2], target_h, target_w, device=device, dtype=vae.dtype) + padding_frames_pixels_values = torch.concat([cond_image.to(device, vae.dtype), video_frames], dim=2) - # encode - vae.to(device) - y = vae.encode(padding_frames_pixels_values, device=device, tiled=tiled_vae, pbar=False).to(dtype) + # encode + vae.to(device) + y = vae.encode(padding_frames_pixels_values, device=device, tiled=tiled_vae, pbar=False).to(dtype) - if mode == "multitalk": - latent_motion_frames = y[:, :, :cur_motion_frames_latent_num][0] # C T H W - else: - if is_first_clip: - latent_motion_frames = vae.encode(cond_image.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) + if mode == "multitalk": + latent_motion_frames = y[:, :, :cur_motion_frames_latent_num][0] # C T H W else: - latent_motion_frames = vae.encode(cond_frame.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) - latent_motion_frames = latent_motion_frames[0] - vae.to(offload_device) - - y = torch.concat([msk, y], dim=1) # B 4+C T H W - mm.soft_empty_cache() + if is_first_clip: + latent_motion_frames = vae.encode(cond_image.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) + else: + latent_motion_frames = vae.encode(cond_frame.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) + latent_motion_frames = latent_motion_frames[0] + vae.to(offload_device) + y = torch.concat([msk, y], dim=1).squeeze(0) # 4+C T H W + mm.soft_empty_cache() + else: + y = None + latent_motion_frames = noise[:, :1] if scheduler == "multitalk": timesteps = list(np.linspace(1000, 1, steps, dtype=np.float32)) @@ -3261,6 +3259,14 @@ class WanVideoSampler: elif model["manual_offloading"]: transformer.to(device) + # Use the appropriate prompt for this section + if len(text_embeds["prompt_embeds"]) > 1: + prompt_index = min(iteration_count, len(text_embeds["prompt_embeds"]) - 1) + positive = [text_embeds["prompt_embeds"][prompt_index]] + log.info(f"Using prompt index: {prompt_index}") + else: + positive = text_embeds["prompt_embeds"] + sampling_pbar = tqdm(total=len(timesteps)-1, desc=f"Sampling audio indices {audio_start_idx}-{audio_end_idx}", position=0, leave=True) for i in range(len(timesteps)-1): timestep = timesteps[i] @@ -3271,9 +3277,9 @@ class WanVideoSampler: noise_pred, self.cache_state = predict_with_cfg( latent_model_input, cfg[idx], - text_embeds["prompt_embeds"], + positive, text_embeds["negative_prompt_embeds"], - timestep, idx, y.squeeze(0), clip_embeds, control_latents, vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, + timestep, idx, y, clip_embeds, control_latents, vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, cache_state=self.cache_state, multitalk_audio_embeds=audio_embs) sampling_pbar.update(1) From fd6404bb2f1c56b107986c8818dd5086a48871a7 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Thu, 21 Aug 2025 01:45:31 +0300 Subject: [PATCH 21/24] Update nodes.py --- nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index dee2891..9d8a6fe 100644 --- a/nodes.py +++ b/nodes.py @@ -3107,7 +3107,7 @@ class WanVideoSampler: ).to(dtype) pcd_data['render_latent'] = render_latent - h, w = cond_image.shape[-2], cond_image.shape[-1] if cond_image is not None else (target_h, target_w) + h, w = (cond_image.shape[-2], cond_image.shape[-1]) if cond_image is not None else (target_h, target_w) lat_h, lat_w = h // VAE_STRIDE[1], w // VAE_STRIDE[2] seq_len = ((frame_num - 1) // VAE_STRIDE[0] + 1) * lat_h * lat_w // (PATCH_SIZE[1] * PATCH_SIZE[2]) From bcdd6dc664eeecaf921db8f0706a13ee172f5bb9 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Thu, 21 Aug 2025 02:36:52 +0300 Subject: [PATCH 22/24] Support loading VACE GGUF modules --- nodes_model_loading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodes_model_loading.py b/nodes_model_loading.py index 866fa9f..4e1a6e2 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -503,7 +503,7 @@ class WanVideoVACEModelSelect: def INPUT_TYPES(s): return { "required": { - "vace_model": (folder_paths.get_filename_list("diffusion_models"), {"tooltip": "These models are loaded from the 'ComfyUI/models/diffusion_models' VACE model to use when not using model that has it included"}), + "vace_model": (folder_paths.get_filename_list("unet_gguf") + folder_paths.get_filename_list("diffusion_models"), {"tooltip": "These models are loaded from the 'ComfyUI/models/diffusion_models' VACE model to use when not using model that has it included"}), }, } @@ -835,7 +835,7 @@ class WanVideoModelLoader: if gguf: if not vace_model["path"].endswith(".gguf"): raise ValueError("With GGUF main model the VACE module must also be a GGUF quantized, if the main model already has VACE included, you can disconnect the VACE module loader") - vace_sd = load_gguf_checkpoint(model_path) + vace_sd = load_gguf_checkpoint(vace_model["path"]) else: vace_sd = load_torch_file(vace_model["path"], device=transformer_load_device, safe_load=True) sd.update(vace_sd) From bd5f76a3c8c2f19cfc4d63b327098007c5d5b6d0 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Thu, 21 Aug 2025 12:26:29 +0300 Subject: [PATCH 23/24] Fix differential diffusion, VACE tweaks --- nodes.py | 88 ++++++++++++++++++++++++--------------- wanvideo/wan_video_vae.py | 4 +- 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/nodes.py b/nodes.py index 9d8a6fe..88e5abc 100644 --- a/nodes.py +++ b/nodes.py @@ -1232,8 +1232,6 @@ class WanVideoVACEEncode: CATEGORY = "WanVideoWrapper" def process(self, vae, width, height, num_frames, strength, vace_start_percent, vace_end_percent, input_frames=None, ref_images=None, input_masks=None, prev_vace_embeds=None, tiled_vae=False): - vae = vae.to(device) - width = (width // 16) * 16 height = (height // 16) * 16 @@ -1244,8 +1242,8 @@ class WanVideoVACEEncode: if input_frames is None: input_frames = torch.zeros((1, 3, num_frames, height, width), device=device, dtype=vae.dtype) else: - input_frames = input_frames[:num_frames] - input_frames = common_upscale(input_frames.clone().movedim(-1, 1), width, height, "lanczos", "disabled").movedim(1, -1) + input_frames = input_frames.clone()[:num_frames, :, :, :3] + input_frames = common_upscale(input_frames.movedim(-1, 1), width, height, "lanczos", "disabled").movedim(1, -1) input_frames = input_frames.to(vae.dtype).to(device).unsqueeze(0).permute(0, 4, 1, 2, 3) # B, C, T, H, W input_frames = input_frames * 2 - 1 if input_masks is None: @@ -1258,10 +1256,11 @@ class WanVideoVACEEncode: input_masks = input_masks.unsqueeze(-1).unsqueeze(0).permute(0, 4, 1, 2, 3).repeat(1, 3, 1, 1, 1) # B, C, T, H, W if ref_images is not None: + ref_images = ref_images.clone()[..., :3] # Create padded image if ref_images.shape[0] > 1: ref_images = torch.cat([ref_images[i] for i in range(ref_images.shape[0])], dim=1).unsqueeze(0) - + B, H, W, C = ref_images.shape current_aspect = W / H target_aspect = width / height @@ -1283,12 +1282,12 @@ class WanVideoVACEEncode: ref_images = ref_images.to(vae.dtype).to(device).unsqueeze(0).permute(0, 4, 1, 2, 3).unsqueeze(0) ref_images = ref_images * 2 - 1 - + + vae = vae.to(device) z0 = self.vace_encode_frames(vae, input_frames, ref_images, masks=input_masks, tiled_vae=tiled_vae) vae.model.clear_cache() m0 = self.vace_encode_masks(input_masks, ref_images) z = self.vace_latent(z0, m0) - vae.to(offload_device) vace_input = { @@ -1306,9 +1305,14 @@ class WanVideoVACEEncode: if prev_vace_embeds is not None: if "additional_vace_inputs" in prev_vace_embeds and prev_vace_embeds["additional_vace_inputs"]: vace_input["additional_vace_inputs"] = prev_vace_embeds["additional_vace_inputs"].copy() + else: + new_entry = prev_vace_embeds.copy() + new_entry.update(vace_input) + return (new_entry,) vace_input["additional_vace_inputs"].append(prev_vace_embeds) return (vace_input,) + def vace_encode_frames(self, vae, frames, ref_images, masks=None, tiled_vae=False): if ref_images is None: ref_images = [None] * len(frames) @@ -2710,18 +2714,6 @@ class WanVideoSampler: if idx < skip_steps: continue - # diff diff - if masks is not None: - if idx < len(timesteps) - 1: - noise_timestep = timesteps[idx+1] - image_latent = sample_scheduler.scale_noise( - original_image, torch.tensor([noise_timestep]), noise.to(device) - ) - mask = masks[idx] - mask = mask.to(latent) - latent = image_latent * mask + latent * (1-mask) - # end diff diff - if bidirectional_sampling: latent_flipped = torch.flip(latent, dims=[1]) latent_model_input_flipped = latent_flipped.to(device) @@ -3015,7 +3007,6 @@ class WanVideoSampler: if mode == "auto": mode = transformer.multitalk_model_type.lower() log.info(f"Multitalk mode: {mode}") - original_images = cond_image = image_embeds.get("multitalk_start_image", None) cond_frame = None offload = image_embeds.get("force_offload", False) tiled_vae = image_embeds.get("tiled_vae", False) @@ -3028,6 +3019,9 @@ class WanVideoSampler: motion_frame = image_embeds.get("motion_frame", 25) target_w = image_embeds.get("target_w", None) target_h = image_embeds.get("target_h", None) + original_images = cond_image = image_embeds.get("multitalk_start_image", None) + if original_images is None: + original_images = torch.zeros([noise.shape[0], 1, target_h, target_w], device=device) if len(multitalk_embeds['audio_features'])==2 and (multitalk_embeds['ref_target_masks'] is None): face_scale = 0.1 @@ -3115,18 +3109,18 @@ class WanVideoSampler: 16, (frame_num - 1) // 4 + 1, lat_h, lat_w, dtype=torch.float32, device=torch.device("cpu"), generator=seed_g).to(device) + # Calculate the correct latent slice based on current iteration + if is_first_clip: + latent_start_idx = 0 + latent_end_idx = noise.shape[1] + else: + new_frames_per_iteration = frame_num - motion_frame + new_latent_frames_per_iteration = ((new_frames_per_iteration - 1) // 4 + 1) + latent_start_idx = iteration_count * new_latent_frames_per_iteration + latent_end_idx = latent_start_idx + noise.shape[1] + if samples is not None: - input_samples = samples["samples"].squeeze(0).to(noise) - # Calculate the correct slice based on current iteration - if is_first_clip: - latent_start_idx = 0 - latent_end_idx = noise.shape[1] - else: - new_frames_per_iteration = frame_num - motion_frame - new_latent_frames_per_iteration = ((new_frames_per_iteration - 1) // 4 + 1) - latent_start_idx = iteration_count * new_latent_frames_per_iteration - latent_end_idx = latent_start_idx + noise.shape[1] - + input_samples = samples["samples"].squeeze(0).to(noise) # Check if we have enough frames in input_samples if latent_end_idx > input_samples.shape[1]: # We need more frames than available - pad the input_samples at the end @@ -3143,6 +3137,22 @@ class WanVideoSampler: else: noise = input_samples + window_vace_data = None + if vace_data is not None: + window_vace_data = [] + for vace_entry in vace_data: + partial_context = vace_entry["context"][0][:, latent_start_idx:latent_end_idx] + if has_ref: + partial_context[:, 0] = vace_entry["context"][0][:, 0] + + window_vace_data.append({ + "context": [partial_context], + "scale": vace_entry["scale"], + "start": vace_entry["start"], + "end": vace_entry["end"], + "seq_len": vace_entry["seq_len"] + }) + # get mask msk = torch.ones(1, frame_num, lat_h, lat_w, device=device) if mode == "multitalk": @@ -3158,8 +3168,8 @@ class WanVideoSampler: mm.soft_empty_cache() # zero padding and vae encode - if cond_image is not None or cond_frame is not None: - video_frames = torch.zeros(1, cond_image.shape[1], frame_num-cond_image.shape[2], target_h, target_w, device=device, dtype=vae.dtype) + if cond_image is not None: + video_frames = torch.zeros(1, 3, frame_num-cond_image.shape[2], target_h, target_w, device=device, dtype=vae.dtype) padding_frames_pixels_values = torch.concat([cond_image.to(device, vae.dtype), video_frames], dim=2) # encode @@ -3279,7 +3289,7 @@ class WanVideoSampler: cfg[idx], positive, text_embeds["negative_prompt_embeds"], - timestep, idx, y, clip_embeds, control_latents, vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, + timestep, idx, y, clip_embeds, control_latents, window_vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, cache_state=self.cache_state, multitalk_audio_embeds=audio_embs) sampling_pbar.update(1) @@ -3468,6 +3478,16 @@ class WanVideoSampler: **scheduler_step_args)[0].squeeze(0) latent_backwards = torch.flip(latent_backwards, dims=[1]) latent = latent * 0.5 + latent_backwards * 0.5 + + # differential diffusion inpaint + if masks is not None: + if idx < len(timesteps) - 1: + noise_timestep = timesteps[idx+1] + image_latent = sample_scheduler.scale_noise( + original_image, torch.tensor([noise_timestep]), noise.to(device) + ) + mask = masks[idx].to(latent) + latent = image_latent * mask + latent * (1-mask) #InfiniteTalk first frame handling if (extra_latents is not None diff --git a/wanvideo/wan_video_vae.py b/wanvideo/wan_video_vae.py index c26acd3..2f55bc4 100644 --- a/wanvideo/wan_video_vae.py +++ b/wanvideo/wan_video_vae.py @@ -1406,7 +1406,7 @@ class VideoVAE38_(VideoVAE_): attn_scales, self.temperal_upsample, dropout) - def encode(self, x, pbar=False): + def encode(self, x, pbar=True): self.clear_cache() x = patchify(x, patch_size=2) t = x.shape[2] @@ -1434,7 +1434,7 @@ class VideoVAE38_(VideoVAE_): return mu - def decode(self, z, pbar=False): + def decode(self, z, pbar=True): self.clear_cache() z = z / self.inv_std.to(z) + self.mean.to(z) From bd0a634ec0bdf9d55ff80871053bde05ac1b6d90 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Thu, 21 Aug 2025 18:26:06 +0300 Subject: [PATCH 24/24] Update nodes.py --- nodes.py | 70 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/nodes.py b/nodes.py index 88e5abc..79d562e 100644 --- a/nodes.py +++ b/nodes.py @@ -1305,10 +1305,6 @@ class WanVideoVACEEncode: if prev_vace_embeds is not None: if "additional_vace_inputs" in prev_vace_embeds and prev_vace_embeds["additional_vace_inputs"]: vace_input["additional_vace_inputs"] = prev_vace_embeds["additional_vace_inputs"].copy() - else: - new_entry = prev_vace_embeds.copy() - new_entry.update(vace_input) - return (new_entry,) vace_input["additional_vace_inputs"].append(prev_vace_embeds) return (vace_input,) @@ -2051,25 +2047,25 @@ class WanVideoSampler: else: noise = input_samples - mask = samples.get("noise_mask", None) - if mask is not None: - log.info(f"Latent mask shape: {mask.shape}") + noise_mask = samples.get("noise_mask", None) + if noise_mask is not None: + log.info(f"Latent noise_mask shape: {noise_mask.shape}") original_image = input_samples.to(device) - if len(mask.shape) == 4: - mask = mask.squeeze(1) + if len(noise_mask.shape) == 4: + noise_mask = noise_mask.squeeze(1) - mask = torch.nn.functional.interpolate( - mask.unsqueeze(0).unsqueeze(0), # Add batch and channel dims [1,1,T,H,W] + noise_mask = torch.nn.functional.interpolate( + noise_mask.unsqueeze(0).unsqueeze(0), # Add batch and channel dims [1,1,T,H,W] size=(noise.shape[1], noise.shape[2], noise.shape[3]), mode='trilinear', align_corners=False ).squeeze(0) # Remove batch dim, keep channel dim # Add batch & channel dims for final output - mask = mask.unsqueeze(0).repeat(1, noise.shape[0], 1, 1, 1) + noise_mask = noise_mask.unsqueeze(0).repeat(1, noise.shape[0], 1, 1, 1) - if mask.shape[2] != noise.shape[1]: - mask = torch.cat([torch.zeros(1, noise.shape[0], noise.shape[1] - mask.shape[2], noise.shape[2], noise.shape[3]), mask], dim=2) + if noise_mask.shape[2] != noise.shape[1]: + noise_mask = torch.cat([torch.zeros(1, noise.shape[0], noise.shape[1] - noise_mask.shape[2], noise.shape[2], noise.shape[3]), noise_mask], dim=2) # extra latents (Pusa) and 5b latents_to_insert = add_index = None @@ -2628,11 +2624,11 @@ class WanVideoSampler: # diff diff prep masks = None - if not multitalk_sampling and samples is not None and mask is not None: - mask = 1 - mask + if not multitalk_sampling and samples is not None and noise_mask is not None: + noise_mask = 1 - noise_mask thresholds = torch.arange(len(timesteps), dtype=original_image.dtype) / len(timesteps) thresholds = thresholds.unsqueeze(1).unsqueeze(1).unsqueeze(1).unsqueeze(1).to(device) - masks = mask.repeat(len(timesteps), 1, 1, 1, 1).to(device) + masks = noise_mask.repeat(len(timesteps), 1, 1, 1, 1).to(device) masks = masks > thresholds latent_shift_loop = False @@ -3137,6 +3133,15 @@ class WanVideoSampler: else: noise = input_samples + # diff diff prep + masks = None + if noise_mask is not None: + noise_mask = 1 - noise_mask + thresholds = torch.arange(len(timesteps), dtype=original_image.dtype) / len(timesteps) + thresholds = thresholds.unsqueeze(1).unsqueeze(1).unsqueeze(1).unsqueeze(1).to(device) + masks = noise_mask.repeat(len(timesteps), 1, 1, 1, 1).to(device) + masks = masks > thresholds + window_vace_data = None if vace_data is not None: window_vace_data = [] @@ -3153,7 +3158,7 @@ class WanVideoSampler: "seq_len": vace_entry["seq_len"] }) - # get mask + # get image cond mask msk = torch.ones(1, frame_num, lat_h, lat_w, device=device) if mode == "multitalk": msk[:, cur_motion_frames_num:] = 0 @@ -3308,13 +3313,22 @@ class WanVideoSampler: latent = latent + noise_pred * dt[:, None, None, None] else: latent = latent.to(intermediate_device) - temp_x0 = sample_scheduler.step( noise_pred.unsqueeze(0), timestep, latent.unsqueeze(0), **scheduler_step_args)[0] latent = temp_x0.squeeze(0) + + # differential diffusion inpaint + if masks is not None: + if idx < len(timesteps) - 1: + noise_timestep = timesteps[idx+1] + image_latent = sample_scheduler.scale_noise( + original_image, torch.tensor([noise_timestep]), noise.to(device) + ) + mask = masks[idx].to(latent) + latent = image_latent * mask + latent * (1-mask) # injecting motion frames if not is_first_clip and mode == "multitalk": @@ -3479,6 +3493,15 @@ class WanVideoSampler: latent_backwards = torch.flip(latent_backwards, dims=[1]) latent = latent * 0.5 + latent_backwards * 0.5 + #InfiniteTalk first frame handling + if (extra_latents is not None + and not multitalk_sampling + and transformer.multitalk_model_type=="InfiniteTalk"): + for entry in extra_latents: + add_index = entry["index"] + num_extra_frames = entry["samples"].shape[2] + latent[:, add_index:add_index+num_extra_frames] = entry["samples"].to(latent) + # differential diffusion inpaint if masks is not None: if idx < len(timesteps) - 1: @@ -3488,15 +3511,6 @@ class WanVideoSampler: ) mask = masks[idx].to(latent) latent = image_latent * mask + latent * (1-mask) - - #InfiniteTalk first frame handling - if (extra_latents is not None - and not multitalk_sampling - and transformer.multitalk_model_type=="InfiniteTalk"): - for entry in extra_latents: - add_index = entry["index"] - num_extra_frames = entry["samples"].shape[2] - latent[:, add_index:add_index+num_extra_frames] = entry["samples"].to(latent) if freeinit_args is not None: current_latent = latent.clone()