From 38fd791a775dee0f5577add90e1a09cc6a9ecab6 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Sat, 13 Sep 2025 16:55:28 +0300 Subject: [PATCH] Squashed commit of the following: commit fda0fe6e0c21eb10276ae302cd88b6cbcf5b36b5 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 16:55:00 2025 +0300 Create wanvideo_HuMo_example_01.json commit cffe3039c3d2fbacd4803329bf31b5fdc45215ba Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 16:30:49 2025 +0300 Update model.py commit ddce018a5a6ffeb926860342889b12efb7343ec0 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 16:29:27 2025 +0300 cleanup commit 8c021b8b3f66144804e500e74aa6f2be52f9f9fc Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 16:23:27 2025 +0300 avoid compile graph break commit ef9c7732042261581b4bba6d980def78633f56dc Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 16:16:13 2025 +0300 Allow using whisper model without decoder layers commit 8d0ba29ee84d14be6084ecbcee1cbc9414128fb3 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 15:55:26 2025 +0300 start/end percent for HuMo audio commit bfe0d358a8820240f262351e61cfb979cb9a47ff Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 15:37:11 2025 +0300 cleanup commit e563ae317f24a7f5751b43cdef4bffbaeaea5114 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Sat Sep 13 14:02:21 2025 +0300 Make audio work commit 95855196c51b1124a19079b746c5d12ec70d9026 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Fri Sep 12 18:10:04 2025 +0300 cfg commit d5a18b090fe719b7f0b00a0f68e6824685598313 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Fri Sep 12 03:10:15 2025 +0300 wrong way around commit 34c8c4842c14002fe4694dfa23c24a65b7ea39d0 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Fri Sep 12 03:01:45 2025 +0300 Update nodes.py commit 47d1e2ab5f3e1483782d0739f5b51cdd33707c36 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Fri Sep 12 02:47:03 2025 +0300 update image inputs are working but audio still doesn't do anything commit 67890d816a64459944091cb01478c1e0ec4c4a82 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Thu Sep 11 21:13:12 2025 +0300 update commit dbcef53405bb78feae4c5d2c6b310b76e4ef9949 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Thu Sep 11 17:09:37 2025 +0300 Update model.py commit 92c9aac51f4d37988757510a4e57179834cc5de2 Author: kijai <40791699+kijai@users.noreply.github.com> Date: Thu Sep 11 16:15:39 2025 +0300 init untested as no weights released as of yet --- HuMo/audio_proj.py | 87 + HuMo/nodes.py | 249 +++ HuMo/whisper_config.json | 50 + __init__.py | 9 + .../wanvideo_HuMo_example_01.json | 1771 +++++++++++++++++ nodes.py | 175 +- nodes_model_loading.py | 5 +- wanvideo/modules/model.py | 132 +- 8 files changed, 2368 insertions(+), 110 deletions(-) create mode 100644 HuMo/audio_proj.py create mode 100644 HuMo/nodes.py create mode 100644 HuMo/whisper_config.json create mode 100644 example_workflows/wanvideo_HuMo_example_01.json diff --git a/HuMo/audio_proj.py b/HuMo/audio_proj.py new file mode 100644 index 0000000..8483a3f --- /dev/null +++ b/HuMo/audio_proj.py @@ -0,0 +1,87 @@ +import torch +from einops import rearrange +from torch import nn +from einops import rearrange + +class WanRMSNorm(nn.Module): + + def __init__(self, dim, eps=1e-5): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + r""" + Args: + x(Tensor): Shape [B, L, C] + """ + return self._norm(x.float()).type_as(x) * self.weight + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) + + +class DummyAdapterLayer(nn.Module): + def __init__(self, layer): + super().__init__() + self.layer = layer + + def forward(self, *args, **kwargs): + return self.layer(*args, **kwargs) + + +class AudioProjModel(nn.Module): + def __init__( + self, + seq_len=5, + blocks=13, # add a new parameter blocks + channels=768, # add a new parameter channels + intermediate_dim=512, + output_dim=1536, + context_tokens=16, + ): + super().__init__() + + self.seq_len = seq_len + self.blocks = blocks + self.channels = channels + self.input_dim = seq_len * blocks * channels # update input_dim to be the product of blocks and channels. + self.intermediate_dim = intermediate_dim + self.context_tokens = context_tokens + self.output_dim = output_dim + + # define multiple linear layers + self.audio_proj_glob_1 = DummyAdapterLayer(nn.Linear(self.input_dim, intermediate_dim)) + self.audio_proj_glob_2 = DummyAdapterLayer(nn.Linear(intermediate_dim, intermediate_dim)) + self.audio_proj_glob_3 = DummyAdapterLayer(nn.Linear(intermediate_dim, context_tokens * output_dim)) + + self.audio_proj_glob_norm = DummyAdapterLayer(nn.LayerNorm(output_dim)) + + self.initialize_weights() + + def initialize_weights(self): + # Initialize transformer layers: + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + + self.apply(_basic_init) + + def forward(self, audio_embeds): + video_length = audio_embeds.shape[1] + audio_embeds = rearrange(audio_embeds, "bz f w b c -> (bz f) w b c") + batch_size, window_size, blocks, channels = audio_embeds.shape + audio_embeds = audio_embeds.view(batch_size, window_size * blocks * channels) + + audio_embeds = torch.relu(self.audio_proj_glob_1(audio_embeds)) + audio_embeds = torch.relu(self.audio_proj_glob_2(audio_embeds)) + + context_tokens = self.audio_proj_glob_3(audio_embeds).reshape(batch_size, self.context_tokens, self.output_dim) + + context_tokens = self.audio_proj_glob_norm(context_tokens) + context_tokens = rearrange(context_tokens, "(bz f) m c -> bz f m c", f=video_length) + + return context_tokens \ No newline at end of file diff --git a/HuMo/nodes.py b/HuMo/nodes.py new file mode 100644 index 0000000..dfc5fd3 --- /dev/null +++ b/HuMo/nodes.py @@ -0,0 +1,249 @@ +import folder_paths +import torch +import torch.nn.functional as F +import os +import json +import torchaudio + +from comfy.utils import load_torch_file +import comfy.model_management as mm + +from accelerate import init_empty_weights +from ..utils import set_module_tensor_to_device +from ..nodes import WanVideoEncodeLatentBatch + +script_directory = os.path.dirname(os.path.abspath(__file__)) +device = mm.get_torch_device() +offload_device = mm.unet_offload_device() + +def linear_interpolation_fps(features, input_fps, output_fps, output_len=None): + features = features.transpose(1, 2) # [1, C, T] + seq_len = features.shape[2] / float(input_fps) + if output_len is None: + output_len = int(seq_len * output_fps) + output_features = F.interpolate(features, size=output_len, align_corners=True, mode='linear') + return output_features.transpose(1, 2) + +def get_audio_emb_window(audio_emb, frame_num, frame0_idx, audio_shift=2): + zero_audio_embed = torch.zeros((audio_emb.shape[1], audio_emb.shape[2]), dtype=audio_emb.dtype, device=audio_emb.device) + zero_audio_embed_3 = torch.zeros((3, audio_emb.shape[1], audio_emb.shape[2]), dtype=audio_emb.dtype, device=audio_emb.device) + iter_ = 1 + (frame_num - 1) // 4 + audio_emb_wind = [] + for lt_i in range(iter_): + if lt_i == 0: + st = frame0_idx + lt_i - 2 + ed = frame0_idx + lt_i + 3 + wind_feat = torch.stack([ + audio_emb[i] if (0 <= i < audio_emb.shape[0]) else zero_audio_embed + for i in range(st, ed) + ], dim=0) + wind_feat = torch.cat((zero_audio_embed_3, wind_feat), dim=0) + else: + st = frame0_idx + 1 + 4 * (lt_i - 1) - audio_shift + ed = frame0_idx + 1 + 4 * lt_i + audio_shift + wind_feat = torch.stack([ + audio_emb[i] if (0 <= i < audio_emb.shape[0]) else zero_audio_embed + for i in range(st, ed) + ], dim=0) + audio_emb_wind.append(wind_feat) + audio_emb_wind = torch.stack(audio_emb_wind, dim=0) + + return audio_emb_wind, ed - audio_shift + +class WhisperModelLoader: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": (folder_paths.get_filename_list("audio_encoders"), {"tooltip": "These models are loaded from the 'ComfyUI/models/wav2vec2' or 'ComfyUI/models/audio_encoders' folder",}), + "base_precision": (["fp32", "bf16", "fp16"], {"default": "fp16"}), + "load_device": (["main_device", "offload_device"], {"default": "main_device", "tooltip": "Initial device to load the model to, NOT recommended with the larger models unless you have 48GB+ VRAM"}), + }, + } + + RETURN_TYPES = ("WHISPERMODEL",) + RETURN_NAMES = ("whisper_model", ) + FUNCTION = "loadmodel" + CATEGORY = "WanVideoWrapper" + + def loadmodel(self, model, base_precision, load_device): + from transformers import WhisperConfig, WhisperModel, WhisperFeatureExtractor + + base_dtype = {"fp8_e4m3fn": torch.float8_e4m3fn, "fp8_e4m3fn_fast": torch.float8_e4m3fn, "bf16": torch.bfloat16, "fp16": torch.float16, "fp16_fast": torch.float16, "fp32": torch.float32}[base_precision] + + if load_device == "offload_device": + transformer_load_device = offload_device + else: + transformer_load_device = device + + config_path = os.path.join(script_directory, "whisper_config.json") + whisper_config = WhisperConfig(**json.load(open(config_path))) + + with init_empty_weights(): + whisper = WhisperModel(whisper_config).eval() + whisper.decoder = None # we only need the encoder + + feature_extractor_config = { + "chunk_length": 30, + "feature_extractor_type": "WhisperFeatureExtractor", + "feature_size": 128, + "hop_length": 160, + "n_fft": 400, + "n_samples": 480000, + "nb_max_frames": 3000, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "WhisperProcessor", + "return_attention_mask": False, + "sampling_rate": 16000 + } + + feature_extractor = WhisperFeatureExtractor(**feature_extractor_config) + + model_path = folder_paths.get_full_path_or_raise("audio_encoders", model) + sd = load_torch_file(model_path, device=transformer_load_device, safe_load=True) + + for name, param in whisper.named_parameters(): + key = "model." + name + value=sd[key] + set_module_tensor_to_device(whisper, name, device=offload_device, dtype=base_dtype, value=value) + + whisper_model = { + "feature_extractor": feature_extractor, + "model": whisper, + "dtype": base_dtype, + } + + return (whisper_model,) + +class HuMoEmbeds: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "whisper_model": ("WHISPERMODEL",), + "vae": ("WANVAE", ), + "num_frames": ("INT", {"default": 81, "min": -1, "max": 10000, "step": 1, "tooltip": "The total frame count to generate."}), + "reference_images": ("IMAGE", {"tooltip": "reference images for the humo model"}), + "audio_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.01, "tooltip": "Strength of the audio conditioning"}), + "audio_cfg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.01, "tooltip": "When not 1.0, an extra model pass without audio conditioning is done: slower inference but more motion is allowed"}), + "audio_start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "The percent of the video to start applying audio conditioning"}), + "audio_end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "The percent of the video to stop applying audio conditioning"}) + }, + "optional" : { + "audio": ("AUDIO",), + } + } + + RETURN_TYPES = ("WANVIDIMAGE_EMBEDS", ) + RETURN_NAMES = ("image_embeds", ) + FUNCTION = "process" + CATEGORY = "WanVideoWrapper" + + def process(self, whisper_model, vae, reference_images, num_frames, audio_scale, audio_cfg_scale, audio_start_percent, audio_end_percent, audio=None): + model = whisper_model["model"] + feature_extractor = whisper_model["feature_extractor"] + dtype = whisper_model["dtype"] + + sampling_rate = 16000 + + if audio is not None: + audio_input = audio["waveform"][0] + sample_rate = audio["sample_rate"] + + if sample_rate != sampling_rate: + audio_input = torchaudio.functional.resample(audio_input, sample_rate, sampling_rate) + if audio_input.shape[1] == 2: + audio_input = audio_input.mean(dim=0, keepdim=False) + else: + audio_input = audio_input[0] + + model.to(device) + audio_len = len(audio_input) // 640 + + # feature extraction + audio_features = [] + window = 750*640 + for i in range(0, len(audio_input), window): + audio_feature = feature_extractor(audio_input[i:i+window], sampling_rate=sampling_rate, return_tensors="pt").input_features + audio_features.append(audio_feature) + audio_features = torch.cat(audio_features, dim=-1).to(device, dtype) + + # preprocess + window = 3000 + audio_prompts = [] + for i in range(0, audio_features.shape[-1], window): + audio_prompt = model.encoder(audio_features[:,:,i:i+window], output_hidden_states=True).hidden_states + audio_prompt = torch.stack(audio_prompt, dim=2) + audio_prompts.append(audio_prompt) + + model.to(offload_device) + + audio_prompts = torch.cat(audio_prompts, dim=1) + audio_prompts = audio_prompts[:,:audio_len*2] + + feat0 = linear_interpolation_fps(audio_prompts[:, :, 0: 8].mean(dim=2), 50, 25) + feat1 = linear_interpolation_fps(audio_prompts[:, :, 8: 16].mean(dim=2), 50, 25) + feat2 = linear_interpolation_fps(audio_prompts[:, :, 16: 24].mean(dim=2), 50, 25) + feat3 = linear_interpolation_fps(audio_prompts[:, :, 24: 32].mean(dim=2), 50, 25) + feat4 = linear_interpolation_fps(audio_prompts[:, :, 32], 50, 25) + audio_emb = torch.stack([feat0, feat1, feat2, feat3, feat4], dim=2)[0] # [T, 5, 1280] + else: + audio_emb = torch.zeros(num_frames, 5, 1280, device=device) + audio_len = num_frames + + frame_num = num_frames if num_frames != -1 else audio_len + frame_num = 4 * ((frame_num - 1) // 4) + 1 + audio_emb, _ = get_audio_emb_window(audio_emb, frame_num, frame0_idx=0) + + samples, = WanVideoEncodeLatentBatch.encode(self, vae, reference_images, False, 0, 0, 0, 0) + samples = samples["samples"].transpose(0, 2).squeeze(0) + + C, T, H, W = samples.shape + + target_shape = (16, (num_frames - 1) // 4 + 1 + T, + H * 8 // 8, + W * 8 // 8) + + vae.to(device) + zero_frames = torch.zeros(1, 3, num_frames + 4*T, H * 8, W * 8, device=device, dtype=vae.dtype) + zero_latents = vae.encode(zero_frames, device=device)[0].to(samples.device) + vae.model.clear_cache() + vae.to(offload_device) + mm.soft_empty_cache() + + mask = torch.ones(4, target_shape[1], target_shape[2], target_shape[3], device=samples.device, dtype=vae.dtype) + mask[:,:-T] = 0 + image_cond = torch.cat([zero_latents[:, :(target_shape[1]-T)], samples], dim=1) + image_cond = torch.cat([mask, image_cond], dim=0) + image_cond_neg = torch.cat([mask, zero_latents], dim=0) + + zero_audio_pad = torch.zeros(T, *audio_emb.shape[1:]).to(audio_emb.device) + audio_emb = torch.cat([audio_emb, zero_audio_pad], dim=0) + audio_emb_neg = torch.zeros_like(audio_emb, dtype=audio_emb.dtype, device=audio_emb.device) + + embeds = { + "humo_audio_emb": audio_emb, + "humo_audio_emb_neg": audio_emb_neg, + "humo_image_cond": image_cond, + "humo_image_cond_neg": image_cond_neg, + "humo_reference_count": T, + "target_shape": target_shape, + "num_frames": num_frames, + "humo_audio_scale": audio_scale, + "humo_audio_cfg_scale": audio_cfg_scale, + "humo_start_percent": audio_start_percent, + "humo_end_percent": audio_end_percent, + } + + return (embeds, ) + + +NODE_CLASS_MAPPINGS = { + "WhisperModelLoader": WhisperModelLoader, + "HuMoEmbeds": HuMoEmbeds, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "WhisperModelLoader": "Whisper Model Loader", + "HuMoEmbeds": "HuMo Embeds", +} \ No newline at end of file diff --git a/HuMo/whisper_config.json b/HuMo/whisper_config.json new file mode 100644 index 0000000..14c6c8c --- /dev/null +++ b/HuMo/whisper_config.json @@ -0,0 +1,50 @@ +{ + "_name_or_path": "openai/whisper-large-v3", + "activation_dropout": 0.0, + "activation_function": "gelu", + "apply_spec_augment": false, + "architectures": [ + "WhisperForConditionalGeneration" + ], + "attention_dropout": 0.0, + "begin_suppress_tokens": [ + 220, + 50257 + ], + "bos_token_id": 50257, + "classifier_proj_size": 256, + "d_model": 1280, + "decoder_attention_heads": 20, + "decoder_ffn_dim": 5120, + "decoder_layerdrop": 0.0, + "decoder_layers": 32, + "decoder_start_token_id": 50258, + "dropout": 0.0, + "encoder_attention_heads": 20, + "encoder_ffn_dim": 5120, + "encoder_layerdrop": 0.0, + "encoder_layers": 32, + "eos_token_id": 50257, + "init_std": 0.02, + "is_encoder_decoder": true, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "max_length": 448, + "max_source_positions": 1500, + "max_target_positions": 448, + "median_filter_width": 7, + "model_type": "whisper", + "num_hidden_layers": 32, + "num_mel_bins": 128, + "pad_token_id": 50256, + "scale_embedding": false, + "torch_dtype": "float16", + "transformers_version": "4.36.0.dev0", + "use_cache": true, + "use_weighted_layer_sum": false, + "vocab_size": 51866 +} diff --git a/__init__.py b/__init__.py index 8ea20c2..0882734 100644 --- a/__init__.py +++ b/__init__.py @@ -43,6 +43,13 @@ except Exception as e: MTV_NODE_CLASS_MAPPINGS = {} MTV_NODE_DISPLAY_NAME_MAPPINGS = {} +try: + from .HuMo.nodes import NODE_CLASS_MAPPINGS as HUMO_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as HUMO_NODE_DISPLAY_NAME_MAPPINGS +except Exception as e: + print(f"HuMo nodes not available due to error in importing them: {e}") + HUMO_NODE_CLASS_MAPPINGS = {} + HUMO_NODE_DISPLAY_NAME_MAPPINGS = {} + NODE_CLASS_MAPPINGS.update(RECAM_MASTER_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(UNIANIMATE_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(SKYREELS_NODE_CLASS_MAPPINGS) @@ -60,6 +67,7 @@ NODE_CLASS_MAPPINGS.update(DEPRECATED_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(QWEN_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(MTV_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(S2V_NODE_CLASS_MAPPINGS) +NODE_CLASS_MAPPINGS.update(HUMO_NODE_CLASS_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(RECAM_MASTER_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(UNIANIMATE_NODE_DISPLAY_NAME_MAPPINGS) @@ -78,5 +86,6 @@ NODE_DISPLAY_NAME_MAPPINGS.update(DEPRECATED_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(QWEN_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(MTV_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(S2V_NODE_DISPLAY_NAME_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(HUMO_NODE_DISPLAY_NAME_MAPPINGS) __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] \ No newline at end of file diff --git a/example_workflows/wanvideo_HuMo_example_01.json b/example_workflows/wanvideo_HuMo_example_01.json new file mode 100644 index 0000000..8fdfd8d --- /dev/null +++ b/example_workflows/wanvideo_HuMo_example_01.json @@ -0,0 +1,1771 @@ +{ + "id": "c6e410bc-5e2c-460b-ae81-c91b6094fbb1", + "revision": 0, + "last_node_id": 121, + "last_link_id": 221, + "nodes": [ + { + "id": 36, + "type": "Note", + "pos": [ + 714.1553955078125, + -685.7478637695312 + ], + "size": [ + 374.3061828613281, + 171.9547576904297 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "fp8_fast seems to cause huge quality degradation\n\nfp_16_fast enables \"Full FP16 Accmumulation in FP16 GEMMs\" feature available in the very latest pytorch nightly, this is around 20% speed boost. \n\nSageattn if you have it installed can be used for almost double inference speed" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 84, + "type": "WanVideoSetBlockSwap", + "pos": [ + 1203.40625, + -451.20733642578125 + ], + "size": [ + 202.5749969482422, + 46 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 149 + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 151 + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 150 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoSetBlockSwap", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "cc3b79ab180e8c81e6125d539662935cf171c87b" + }, + "widgets_values": [], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 81, + "type": "ImageResizeKJv2", + "pos": [ + -261.44146728515625, + 296.5487976074219 + ], + "size": [ + 270, + 336 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 202 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 139 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 140 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 145, + 205, + 220 + ] + }, + { + "name": "width", + "type": "INT", + "links": [] + }, + { + "name": "height", + "type": "INT", + "links": [] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "ImageResizeKJv2", + "cnr_id": "comfyui-kjnodes", + "ver": "0adab07d1ad3d0780afc97319eaf37c3681af37d" + }, + "widgets_values": [ + 512, + 512, + "lanczos", + "pad", + "255,255,255", + "center", + 16, + "cpu", + "Output: 1 x 1280 x 720 | 10.55MB" + ] + }, + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 2345.96484375, + -400.7390441894531 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 43 + }, + { + "name": "samples", + "type": "LATENT", + "link": 33 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 126 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoDecode", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "6099ad393b071728032fd481e96d77d2900eee2c" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 79, + "type": "ImageConcatMulti", + "pos": [ + 2370.963134765625, + -85.9253158569336 + ], + "size": [ + 315, + 170 + ], + "flags": { + "collapsed": false + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 126 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 205 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 127 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "3e3a1a8aac61dc4515f6a7da74e026f05a80299f" + }, + "widgets_values": [ + 2, + "left", + true, + null + ] + }, + { + "id": 83, + "type": "WanVideoSetLoRAs", + "pos": [ + 1471.3389892578125, + -454.92852783203125 + ], + "size": [ + 175.1999969482422, + 46 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 150 + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 147 + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 148 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoSetLoRAs", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "cc3b79ab180e8c81e6125d539662935cf171c87b" + }, + "widgets_values": [], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 22, + "type": "WanVideoModelLoader", + "pos": [ + 617.1249389648438, + -441.7767028808594 + ], + "size": [ + 530.5924072265625, + 314 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": 208 + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": null + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "extra_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": null + }, + { + "name": "fantasyportrait_model", + "shape": 7, + "type": "FANTASYPORTRAITMODEL", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "slot_index": 0, + "links": [ + 149 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoModelLoader", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "6099ad393b071728032fd481e96d77d2900eee2c" + }, + "widgets_values": [ + "WanVideo/HuMo/Wan2_1-HuMo-14B_fp8_e4m3fn_scaled_KJ.safetensors", + "fp16_fast", + "disabled", + "offload_device", + "sageattn" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 39, + "type": "WanVideoBlockSwap", + "pos": [ + 1175.088134765625, + -351.6418151855469 + ], + "size": [ + 315, + 202 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "slot_index": 0, + "links": [ + 151 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoBlockSwap", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "6099ad393b071728032fd481e96d77d2900eee2c" + }, + "widgets_values": [ + 20, + false, + false, + true, + 0, + 1, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 38, + "type": "WanVideoVAELoader", + "pos": [ + 2340.400390625, + -578.1526489257812 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 43, + 176 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoVAELoader", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "6099ad393b071728032fd481e96d77d2900eee2c" + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 112, + "type": "WanVideoTextEncodeCached", + "pos": [ + 672.4511108398438, + -51.898902893066406 + ], + "size": [ + 477.0235900878906, + 471.79931640625 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "extender_args", + "shape": 7, + "type": "WANVIDEOPROMPTEXTENDER_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 209 + ] + }, + { + "name": "negative_text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + }, + { + "name": "positive_prompt", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "WanVideoTextEncodeCached" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "old man is issuing a dire warning", + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + "disabled", + true, + "gpu" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "WanVideoLoraSelect", + "pos": [ + 1191.41259765625, + -703.387939453125 + ], + "size": [ + 596.4673461914062, + 150 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 147 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoLoraSelect", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "cc3b79ab180e8c81e6125d539662935cf171c87b" + }, + "widgets_values": [ + "WanVid/Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 1, + false, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 111, + "type": "CreateCFGScheduleFloatList", + "pos": [ + 1608.3153076171875, + -152.25692749023438 + ], + "size": [ + 298.3199157714844, + 228 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "float_list", + "type": "FLOAT", + "links": [ + 207 + ] + } + ], + "properties": { + "Node name for S&R": "CreateCFGScheduleFloatList", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "95855196c51b1124a19079b746c5d12ec70d9026" + }, + "widgets_values": [ + 8, + 2, + 2, + "linear", + 0, + 0.01, + "[2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]" + ] + }, + { + "id": 27, + "type": "WanVideoSampler", + "pos": [ + 1968.231201171875, + -397.3578796386719 + ], + "size": [ + 315, + 869.9375 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 148 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 183 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 209 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": null + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": null + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + }, + { + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 207 + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 33 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "WanVideoSampler", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "6099ad393b071728032fd481e96d77d2900eee2c" + }, + "widgets_values": [ + 8, + 1, + 5, + 42, + "fixed", + true, + "lcm", + 0, + 1, + false, + "comfy", + 0, + -1, + false + ] + }, + { + "id": 115, + "type": "MelBandRoFormerSampler", + "pos": [ + 1072.6689453125, + 835.3041381835938 + ], + "size": [ + 223.64999389648438, + 46 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MELROFORMERMODEL", + "link": 213 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 217 + } + ], + "outputs": [ + { + "name": "vocals", + "type": "AUDIO", + "links": [ + 214 + ] + }, + { + "name": "instruments", + "type": "AUDIO", + "links": null + } + ], + "properties": { + "Node name for S&R": "MelBandRoFormerSampler" + } + }, + { + "id": 116, + "type": "NormalizeAudioLoudness", + "pos": [ + 1079.6055908203125, + 952.2372436523438 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 214 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 215 + ] + } + ], + "properties": { + "Node name for S&R": "NormalizeAudioLoudness" + }, + "widgets_values": [ + -23 + ] + }, + { + "id": 88, + "type": "WhisperModelLoader", + "pos": [ + 709.6544189453125, + 661.9351806640625 + ], + "size": [ + 484.66668701171875, + 106 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "whisper_model", + "type": "WHISPERMODEL", + "links": [ + 175 + ] + } + ], + "properties": { + "Node name for S&R": "WhisperModelLoader", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "9cefe309e3d8eb9ad3afda576cf7098c72e9efad" + }, + "widgets_values": [ + "whisper_large_v3_encoder_fp16.safetensors", + "fp16", + "main_device" + ] + }, + { + "id": 67, + "type": "LoadImage", + "pos": [ + -268.9026184082031, + -156.98599243164062 + ], + "size": [ + 340.7649230957031, + 379.4031982421875 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 202 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "cnr_id": "comfy-core", + "ver": "0.3.29" + }, + "widgets_values": [ + "oldman_upscaled.png", + "image" + ] + }, + { + "id": 87, + "type": "LoadAudio", + "pos": [ + 741.7787475585938, + 954.3341674804688 + ], + "size": [ + 274.080078125, + 136 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 217, + 218 + ] + } + ], + "properties": { + "Node name for S&R": "LoadAudio", + "cnr_id": "comfy-core", + "ver": "0.3.57" + }, + "widgets_values": [ + "one-does-not-simply-walk-into-mordor-its-black-gates-are-guarded-by-more-than-just-orcs.mp3", + null, + null + ] + }, + { + "id": 117, + "type": "SetNode", + "pos": [ + 1078.615234375, + 1067.1883544921875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "link": 218 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_input_audio", + "properties": { + "previousName": "input_audio" + }, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 30, + "type": "VHS_VideoCombine", + "pos": [ + 2769.601806640625, + -589.9664916992188 + ], + "size": [ + 1915.138671875, + 881.0077514648438 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 127 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 219 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine", + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8" + }, + "widgets_values": { + "frame_rate": 25, + "loop_count": 0, + "filename_prefix": "WanVideo21_HuMo", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo21_HuMo_00005-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 25, + "workflow": "WanVideo21_HuMo_00005.png", + "fullpath": "/home/kijai/AI/ComfyUI/temp/WanVideo21_HuMo_00005-audio.mp4" + } + } + } + }, + { + "id": 118, + "type": "GetNode", + "pos": [ + 2771.7490234375, + -643.5535888671875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 219 + ] + } + ], + "title": "Get_input_audio", + "properties": {}, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 61, + "type": "INTConstant", + "pos": [ + -543.3141479492188, + 214.99432373046875 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 139 + ] + } + ], + "title": "Width", + "properties": { + "Node name for S&R": "INTConstant", + "cnr_id": "comfyui-kjnodes", + "ver": "3e3a1a8aac61dc4515f6a7da74e026f05a80299f" + }, + "widgets_values": [ + 1280 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 62, + "type": "INTConstant", + "pos": [ + -537.188232421875, + 340.2867431640625 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 140 + ] + } + ], + "title": "Height", + "properties": { + "Node name for S&R": "INTConstant", + "cnr_id": "comfyui-kjnodes", + "ver": "3e3a1a8aac61dc4515f6a7da74e026f05a80299f" + }, + "widgets_values": [ + 720 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 75, + "type": "MarkdownNote", + "pos": [ + 713.1290283203125, + -1113.56005859375 + ], + "size": [ + 590.143798828125, + 328.435791015625 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Markdown Note: Model Links", + "properties": {}, + "widgets_values": [ + "Model links:\n\n[https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled/tree/main/HuMo](https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled/tree/main/HuMo)\n\nWhisper:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/blob/main/HuMo/whisper_large_v3_encoder_fp16.safetensors](https://huggingface.co/Kijai/WanVideo_comfy/blob/main/HuMo/whisper_large_v3_encoder_fp16.safetensors)\n\nVocal separator:\n\n[https://huggingface.co/Kijai/MelBandRoFormer_comfy/blob/main/MelBandRoformer_fp16.safetensors](https://huggingface.co/Kijai/MelBandRoFormer_comfy/blob/main/MelBandRoformer_fp16.safetensors)\n\nLoRA:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/tree/main/Lightx2v](https://huggingface.co/Kijai/WanVideo_comfy/tree/main/Lightx2v)\n\nText encoder:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/blob/main/umt5-xxl-enc-bf16.safetensors](https://huggingface.co/Kijai/WanVideo_comfy/blob/main/umt5-xxl-enc-bf16.safetensors)\n\nVAE:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan2_1_VAE_bf16.safetensors](https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan2_1_VAE_bf16.safetensors)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 119, + "type": "Note", + "pos": [ + -544.3449096679688, + 26.11699867248535 + ], + "size": [ + 210, + 127.15582275390625 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Seems to work best with original defaults of 1280x720 and 832x480" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "HuMoEmbeds", + "pos": [ + 1597.50732421875, + 183.74229431152344 + ], + "size": [ + 283.1015625, + 216 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "whisper_model", + "type": "WHISPERMODEL", + "link": 175 + }, + { + "name": "vae", + "type": "WANVAE", + "link": 176 + }, + { + "name": "reference_images", + "type": "IMAGE", + "link": 221 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 215 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 183 + ] + } + ], + "properties": { + "Node name for S&R": "HuMoEmbeds", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "dbcef53405bb78feae4c5d2c6b310b76e4ef9949" + }, + "widgets_values": [ + 65, + 1, + 2.5, + 0, + 1 + ] + }, + { + "id": 121, + "type": "ImageBatchMulti", + "pos": [ + 1265.909423828125, + 149.07740783691406 + ], + "size": [ + 270, + 102 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 220 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 221 + ] + } + ], + "properties": {}, + "widgets_values": [ + 2, + null + ] + }, + { + "id": 120, + "type": "Note", + "pos": [ + 1273.8370361328125, + 308.62152099609375 + ], + "size": [ + 230.1770782470703, + 88 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "You can add more reference images here" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 42, + "type": "Note", + "pos": [ + 1511.07373046875, + -344.02752685546875 + ], + "size": [ + 298.7467956542969, + 88 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Adjust the blocks to swap based on your VRAM, this is a tradeoff between speed and memory usage." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 114, + "type": "MelBandRoFormerModelLoader", + "pos": [ + 716.1973876953125, + 834.9273071289062 + ], + "size": [ + 317.2749938964844, + 58 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "MELROFORMERMODEL", + "links": [ + 213 + ] + } + ], + "properties": { + "Node name for S&R": "MelBandRoFormerModelLoader" + }, + "widgets_values": [ + "MelBandRoformer_fp32.safetensors" + ] + }, + { + "id": 35, + "type": "WanVideoTorchCompileSettings", + "pos": [ + 166.3423309326172, + -440.3662414550781 + ], + "size": [ + 421.6000061035156, + 202 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "slot_index": 0, + "links": [ + 208 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoTorchCompileSettings", + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "6099ad393b071728032fd481e96d77d2900eee2c" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 82, + "type": "PreviewImage", + "pos": [ + 61.709815979003906, + 346.294677734375 + ], + "size": [ + 441.4006042480469, + 281.9812316894531 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 145 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.3.29" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 33, + 27, + 0, + 28, + 1, + "LATENT" + ], + [ + 43, + 38, + 0, + 28, + 0, + "VAE" + ], + [ + 126, + 28, + 0, + 79, + 0, + "IMAGE" + ], + [ + 127, + 79, + 0, + 30, + 0, + "IMAGE" + ], + [ + 139, + 61, + 0, + 81, + 2, + "INT" + ], + [ + 140, + 62, + 0, + 81, + 3, + "INT" + ], + [ + 145, + 81, + 0, + 82, + 0, + "IMAGE" + ], + [ + 147, + 78, + 0, + 83, + 1, + "WANVIDLORA" + ], + [ + 148, + 83, + 0, + 27, + 0, + "WANVIDEOMODEL" + ], + [ + 149, + 22, + 0, + 84, + 0, + "WANVIDEOMODEL" + ], + [ + 150, + 84, + 0, + 83, + 0, + "WANVIDEOMODEL" + ], + [ + 151, + 39, + 0, + 84, + 1, + "BLOCKSWAPARGS" + ], + [ + 175, + 88, + 0, + 95, + 0, + "WHISPERMODEL" + ], + [ + 176, + 38, + 0, + 95, + 1, + "WANVAE" + ], + [ + 183, + 95, + 0, + 27, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 202, + 67, + 0, + 81, + 0, + "IMAGE" + ], + [ + 205, + 81, + 0, + 79, + 1, + "IMAGE" + ], + [ + 207, + 111, + 0, + 27, + 17, + "FLOAT" + ], + [ + 208, + 35, + 0, + 22, + 0, + "WANCOMPILEARGS" + ], + [ + 209, + 112, + 0, + 27, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 213, + 114, + 0, + 115, + 0, + "MELROFORMERMODEL" + ], + [ + 214, + 115, + 0, + 116, + 0, + "AUDIO" + ], + [ + 215, + 116, + 0, + 95, + 3, + "AUDIO" + ], + [ + 217, + 87, + 0, + 115, + 1, + "AUDIO" + ], + [ + 218, + 87, + 0, + 117, + 0, + "*" + ], + [ + 219, + 118, + 0, + 30, + 1, + "AUDIO" + ], + [ + 220, + 81, + 0, + 121, + 0, + "IMAGE" + ], + [ + 221, + 121, + 0, + 95, + 2, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Audio input", + "bounding": [ + 666.8719482421875, + 520.6751708984375, + 1083.3914794921875, + 625.0265502929688 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.555991731349262, + "offset": [ + -380.9118726255019, + 1233.774132735748 + ] + }, + "frontendVersion": "1.27.3", + "node_versions": { + "ComfyUI-WanVideoWrapper": "8d0ba29ee84d14be6084ecbcee1cbc9414128fb3", + "ComfyUI-KJNodes": "d8b8c637fbd1fe97b7a8c7ece843150df38f8b21", + "ComfyUI-MelBandRoFormer": "b68d9077815387b64d596f8c39607052b95b6eba", + "comfy-core": "0.3.59", + "comfyui-videohelpersuite": "8e4d79471bf1952154768e8435a9300077b534fa" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index 5023be7..ea9056e 100644 --- a/nodes.py +++ b/nodes.py @@ -1917,6 +1917,7 @@ class WanVideoSampler: fun_or_fl2v_model = has_ref = drop_last = False phantom_latents = fun_ref_image = ATI_tracks = None add_cond = attn_cond = attn_cond_neg = noise_pred_flipped = None + humo_audio = humo_audio_neg = None #I2V image_cond = image_embeds.get("image_embeds", None) @@ -2103,6 +2104,24 @@ class WanVideoSampler: phantom_start_percent = image_embeds.get("phantom_start_percent", 0.0) phantom_end_percent = image_embeds.get("phantom_end_percent", 1.0) + + #HuMo inputs + humo_audio = image_embeds.get("humo_audio_emb", None) + if humo_audio is not None: + humo_audio = humo_audio.to(device, dtype) + humo_audio_neg = image_embeds.get("humo_audio_emb_neg", None) + if humo_audio_neg is not None: + humo_audio_neg = humo_audio_neg.to(device, dtype) + humo_audio_scale = image_embeds.get("humo_audio_scale", 1.0) + humo_image_cond = image_embeds.get("humo_image_cond", None) + humo_image_cond_neg = image_embeds.get("humo_image_cond_neg", None) + humo_reference_count = image_embeds.get("humo_reference_count", 0) + humo_audio_cfg_scale = image_embeds.get("humo_audio_cfg_scale", 1.0) + humo_start_percent = image_embeds.get("humo_start_percent", 0.0) + humo_end_percent = image_embeds.get("humo_end_percent", 1.0) + if not isinstance(humo_audio_cfg_scale, list): + humo_audio_cfg_scale = [humo_audio_cfg_scale] * (steps + 1) + latent_video_length = noise.shape[1] # Initialize FreeInit filter if enabled @@ -2654,6 +2673,12 @@ class WanVideoSampler: elif ATI_tracks is not None and ((ati_start_percent <= current_step_percentage <= ati_end_percent) or (ati_end_percent > 0 and idx == 0 and current_step_percentage >= ati_start_percent)): image_cond_input = image_cond_ati.to(z) + elif humo_image_cond is not None: + if context_window is not None: + image_cond_input = humo_image_cond[:, context_window].to(z) + image_cond_input[:, -humo_reference_count:] = humo_image_cond[:, -humo_reference_count:] + else: + image_cond_input = humo_image_cond.to(z) elif image_cond is not None: if reverse_time: # Flip the image condition image_cond_input = torch.cat([ @@ -2759,12 +2784,25 @@ class WanVideoSampler: (s2v_pose_end_percent > 0 and idx == 0 and current_step_percentage >= s2v_pose_start_percent)): s2v_pose = None + + if humo_audio is not None and ((humo_start_percent <= current_step_percentage <= humo_end_percent) or \ + (humo_end_percent > 0 and idx == 0 and current_step_percentage >= humo_start_percent)): + humo_audio_input = humo_audio + humo_audio_input_neg = humo_audio_neg if humo_audio_neg is not None else None + else: + humo_audio_input = humo_audio_input_neg = None + base_params = { + 'x': [z], # latent + 'y': [image_cond_input] if image_cond_input is not None else None, # image cond + 'clip_fea': clip_fea, # clip features 'seq_len': seq_len, # sequence length 'device': device, # main device 'freqs': freqs, # rope freqs 't': timestep, # current timestep + 'is_uncond': False, # is unconditional 'current_step': idx, # current step + 'current_step_percentage': current_step_percentage, # current step percentage 'last_step': len(timesteps) - 1 == idx, # is last step 'control_lora_enabled': control_lora_enabled, # control lora toggle for patch embed selection 'enhance_enabled': enhance_enabled, # enhance-a-video toggle @@ -2796,7 +2834,9 @@ class WanVideoSampler: "s2v_ref_motion": s2v_ref_motion, # speech-to-video reference motion latent "s2v_audio_scale": s2v_audio_scale if s2v_audio_input is not None else 1.0, # speech-to-video audio scale "s2v_pose": s2v_pose if s2v_pose is not None else None, # speech-to-video pose control - "s2v_motion_frames": s2v_motion_frames, # speech-to-video motion frames + "s2v_motion_frames": s2v_motion_frames, # speech-to-video motion frames, + "humo_audio": humo_audio_input, # humo audio input + "humo_audio_scale": humo_audio_scale if humo_audio is not None else 1.0, # humo audio scale } batch_size = 1 @@ -2809,98 +2849,88 @@ class WanVideoSampler: try: if not batched_cfg: - #cond + #conditional (positive) pass noise_pred_cond, cache_state_cond = transformer( - [z], context=positive_embeds, y=[image_cond_input] if image_cond_input is not None else None, - clip_fea=clip_fea, is_uncond=False, current_step_percentage=current_step_percentage, + context=positive_embeds, pred_id=cache_state[0] if cache_state else None, vace_data=vace_data, attn_cond=attn_cond, **base_params ) - noise_pred_cond = noise_pred_cond[0].to(intermediate_device) + noise_pred_cond = noise_pred_cond[0] if math.isclose(cfg_scale, 1.0): if use_fresca: - noise_pred_cond = fourier_filter( - noise_pred_cond, - scale_low=fresca_scale_low, - scale_high=fresca_scale_high, - freq_cutoff=fresca_freq_cutoff, - ) + noise_pred_cond = fourier_filter(noise_pred_cond, fresca_scale_low, fresca_scale_high, fresca_freq_cutoff) return noise_pred_cond, [cache_state_cond] - #uncond - if fantasytalking_embeds is not None: - if not math.isclose(audio_cfg_scale[idx], 1.0): - base_params['audio_proj'] = None + + #unconditional (negative) pass + base_params['is_uncond'] = True + base_params['clip_fea'] = clip_fea_neg if clip_fea_neg is not None else clip_fea + if humo_audio_input_neg is not None: + base_params['humo_audio'] = humo_audio_input_neg + noise_pred_uncond, cache_state_uncond = transformer( - [z], context=negative_embeds, clip_fea=clip_fea_neg if clip_fea_neg is not None else clip_fea, - y=[image_cond_input] if image_cond_input is not None else None, - is_uncond=True, current_step_percentage=current_step_percentage, + context=negative_embeds if humo_audio_input_neg is None else positive_embeds, #ti pred_id=cache_state[1] if cache_state else None, vace_data=vace_data, attn_cond=attn_cond_neg, - **base_params - ) - noise_pred_uncond = noise_pred_uncond[0].to(intermediate_device) + **base_params) + noise_pred_uncond = noise_pred_uncond[0] + + # HuMo + if humo_audio_input_neg is not None and not math.isclose(humo_audio_cfg_scale[idx], 1.0): + if len(cache_state) !=3: + cache_state.append(None) + if t > 980 and humo_image_cond_neg is not None: # use image cond for first timesteps + base_params['y'] = [humo_image_cond_neg.to(z)] + + noise_pred_humo_audio_uncond, cache_state_humo = transformer( + context=negative_embeds, pred_id=cache_state[2] if cache_state else None, vace_data=None, + **base_params) + + noise_pred = (noise_pred_uncond + humo_audio_cfg_scale[idx] * (noise_pred_cond - noise_pred_humo_audio_uncond[0]) + + (cfg_scale - 2.0) * (noise_pred_humo_audio_uncond[0] - noise_pred_uncond)) + return noise_pred, [cache_state_cond, cache_state_uncond, cache_state_humo] #phantom if use_phantom and not math.isclose(phantom_cfg_scale[idx], 1.0): noise_pred_phantom, cache_state_phantom = transformer( - [z], context=negative_embeds, clip_fea=clip_fea_neg if clip_fea_neg is not None else clip_fea, - y=[image_cond_input] if image_cond_input is not None else None, - is_uncond=True, current_step_percentage=current_step_percentage, - pred_id=cache_state[2] if cache_state else None, - vace_data=None, - **base_params - ) - noise_pred_phantom = noise_pred_phantom[0].to(intermediate_device) - - noise_pred = noise_pred_uncond + phantom_cfg_scale[idx] * (noise_pred_phantom - noise_pred_uncond) + cfg_scale * (noise_pred_cond - noise_pred_phantom) + context=negative_embeds, pred_id=cache_state[2] if cache_state else None, vace_data=None, + **base_params) + + noise_pred = (noise_pred_uncond + phantom_cfg_scale[idx] * (noise_pred_phantom[0] - noise_pred_uncond) + + cfg_scale * (noise_pred_cond - noise_pred_phantom[0])) return noise_pred, [cache_state_cond, cache_state_uncond, cache_state_phantom] - #fantasytalking - if fantasytalking_embeds is not None: + #audio cfg (fantasytalking and multitalk) + if (fantasytalking_embeds is not None or multitalk_audio_embedding is not None): if not math.isclose(audio_cfg_scale[idx], 1.0): if cache_state is not None and len(cache_state) != 3: cache_state.append(None) - base_params['audio_proj'] = None + + # Set audio parameters to None/zeros based on type + if fantasytalking_embeds is not None: + base_params['audio_proj'] = None + audio_context = positive_embeds + else: # multitalk + base_params['multitalk_audio'] = torch.zeros_like(multitalk_audio_input)[-1:] + audio_context = negative_embeds + noise_pred_no_audio, cache_state_audio = transformer( - [z], context=positive_embeds, y=[image_cond_input] if image_cond_input is not None else None, - clip_fea=clip_fea, is_uncond=False, current_step_percentage=current_step_percentage, + context=audio_context, is_uncond=False, pred_id=cache_state[2] if cache_state else None, vace_data=vace_data, - **base_params - ) - noise_pred_no_audio = noise_pred_no_audio[0].to(intermediate_device) - noise_pred = ( - noise_pred_uncond - + cfg_scale * (noise_pred_no_audio - noise_pred_uncond) - + audio_cfg_scale[idx] * (noise_pred_cond - noise_pred_no_audio) - ) - return noise_pred, [cache_state_cond, cache_state_uncond, cache_state_audio] - elif multitalk_audio_embedding is not None: - if not math.isclose(audio_cfg_scale[idx], 1.0): - if cache_state is not None and len(cache_state) != 3: - cache_state.append(None) - base_params['multitalk_audio'] = torch.zeros_like(multitalk_audio_input)[-1:] - noise_pred_no_audio, cache_state_audio = transformer( - [z], context=negative_embeds, y=[image_cond_input] if image_cond_input is not None else None, - clip_fea=clip_fea, is_uncond=False, current_step_percentage=current_step_percentage, - pred_id=cache_state[2] if cache_state else None, - vace_data=vace_data, - **base_params - ) - noise_pred_no_audio = noise_pred_no_audio[0].to(intermediate_device) - noise_pred = ( - noise_pred_no_audio - + cfg_scale * (noise_pred_cond - noise_pred_uncond) - + audio_cfg_scale[idx] * (noise_pred_uncond - noise_pred_no_audio) - ) + **base_params) + + noise_pred = (noise_pred_uncond + + cfg_scale * (noise_pred_no_audio[0] - noise_pred_uncond) + + audio_cfg_scale[idx] * (noise_pred_cond - noise_pred_no_audio[0])) return noise_pred, [cache_state_cond, cache_state_uncond, cache_state_audio] #batched else: + base_params['z'] = [z] * 2 + base_params['y'] = [image_cond_input] * 2 if image_cond_input is not None else None + base_params['clip_fea'] = torch.cat([clip_fea, clip_fea], dim=0) cache_state_uncond = None [noise_pred_cond, noise_pred_uncond], cache_state_cond = transformer( - [z] + [z], context=positive_embeds + negative_embeds, - y=[image_cond_input] + [image_cond_input] if image_cond_input is not None else None, - clip_fea=clip_fea.repeat(2,1,1), is_uncond=False, current_step_percentage=current_step_percentage, + context=positive_embeds + negative_embeds, is_uncond=False, pred_id=cache_state[0] if cache_state else None, **base_params ) @@ -2919,7 +2949,6 @@ class WanVideoSampler: noise_pred_uncond.view(batch_size, -1) ).view(batch_size, 1, 1, 1) - noise_pred_uncond_scaled = noise_pred_uncond * alpha if use_tangential: @@ -2932,18 +2961,12 @@ class WanVideoSampler: #https://github.com/WikiChao/FreSca if use_fresca: - filtered_cond = fourier_filter( - noise_pred_cond - noise_pred_uncond, - scale_low=fresca_scale_low, - scale_high=fresca_scale_high, - freq_cutoff=fresca_freq_cutoff, - ) + filtered_cond = fourier_filter(noise_pred_cond - noise_pred_uncond, fresca_scale_low, fresca_scale_high, fresca_freq_cutoff) noise_pred = noise_pred_uncond_scaled + cfg_scale * filtered_cond * alpha else: noise_pred = noise_pred_uncond_scaled + cfg_scale * (noise_pred_cond - noise_pred_uncond_scaled) del noise_pred_uncond_scaled, noise_pred_cond, noise_pred_uncond - return noise_pred, [cache_state_cond, cache_state_uncond] if args.preview_method in [LatentPreviewMethod.Auto, LatentPreviewMethod.Latent2RGB]: #default for latent2rgb @@ -4055,6 +4078,8 @@ class WanVideoSampler: callback_latent = (latent_model_input[:, :orig_noise_len].to(device) - noise_pred[:, :orig_noise_len].to(device) * t.to(device) / 1000).detach() #elif phantom_latents is not None: # callback_latent = (latent_model_input[:,:-phantom_latents.shape[1]].to(device) - noise_pred[:,:-phantom_latents.shape[1]].to(device) * t.to(device) / 1000).detach() + elif humo_image_cond is not None: + callback_latent = (latent_model_input[:,:-humo_reference_count].to(device) - noise_pred[:,:-humo_reference_count].to(device) * t.to(device) / 1000).detach() else: callback_latent = (latent_model_input.to(device) - noise_pred.to(device) * t.to(device) / 1000).detach() callback(idx, callback_latent.permute(1,0,2,3), None, len(timesteps)) @@ -4075,6 +4100,8 @@ class WanVideoSampler: if phantom_latents is not None: latent = latent[:,:-phantom_latents.shape[1]] + if humo_image_cond is not None: + latent = latent[:,:-humo_reference_count] cache_states = None if cache_args is not None: diff --git a/nodes_model_loading.py b/nodes_model_loading.py index 174bebb..4fe04f7 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -761,7 +761,7 @@ class WanVideoSetLoRAs: def load_weights(transformer, sd=None, weight_dtype=None, base_dtype=None, transformer_load_device=None, block_swap_args=None, gguf=False, reader=None, patcher=None): params_to_keep = {"time_in", "patch_embedding", "time_", "modulation", "text_embedding", - "adapter", "add", "ref_conv", "casual_audio_encoder", "cond_encoder", "frame_packer"} + "adapter", "add", "ref_conv", "casual_audio_encoder", "cond_encoder", "frame_packer", "audio_proj_glob"} param_count = sum(1 for _ in transformer.named_parameters()) pbar = ProgressBar(param_count) cnt = 0 @@ -1112,6 +1112,8 @@ class WanVideoModelLoader: ffn_dim = sd["blocks.0.ffn.0.bias"].shape[0] ffn2_dim = sd["blocks.0.ffn.2.weight"].shape[1] + is_humo = "audio_proj.audio_proj_glob_1.layer.weight" in sd + model_type = "t2v" if "audio_injector.injector.0.k.weight" in sd: model_type = "s2v" @@ -1226,6 +1228,7 @@ class WanVideoModelLoader: "enable_adain": True if "audio_injector.injector_adain_layers.0.linear.weight" in sd else False, "cond_dim": sd["cond_encoder.weight"].shape[1] if "cond_encoder.weight" in sd else 0, "zero_timestep": model_type == "s2v", + "humo_audio": is_humo, } diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index 6769153..68d1c3e 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -14,7 +14,6 @@ except: from .attention import attention import numpy as np -from copy import deepcopy from tqdm import tqdm import gc @@ -25,10 +24,7 @@ from ...echoshot.echoshot import rope_apply_z, rope_apply_c, rope_apply_echoshot from ...MTV.mtv import apply_rotary_emb -#from .s2v.motioner import MotionerTransformers, FramePackMotioner, rope_precompute - -#from comfy.ldm.wan.model import FramePackMotioner -class FramePackMotioner(nn.Module): +class FramePackMotioner(nn.Module):#from comfy.ldm.wan.model def __init__( self, inner_dim=1024, @@ -362,7 +358,8 @@ class WanSelfAttention(nn.Module): num_heads, qk_norm=True, eps=1e-6, - attention_mode='sdpa'): + attention_mode='sdpa', + kv_dim=None): assert out_features % num_heads == 0 super().__init__() self.dim = out_features @@ -379,8 +376,12 @@ class WanSelfAttention(nn.Module): # layers self.q = nn.Linear(in_features, out_features) - self.k = nn.Linear(in_features, out_features) - self.v = nn.Linear(in_features, out_features) + if kv_dim is not None: + self.k = nn.Linear(kv_dim, out_features) + self.v = nn.Linear(kv_dim, out_features) + else: + self.k = nn.Linear(in_features, out_features) + self.v = nn.Linear(in_features, out_features) self.o = nn.Linear(in_features, out_features) self.norm_q = WanRMSNorm(out_features, eps=eps) if qk_norm else nn.Identity() self.norm_k = WanRMSNorm(out_features, eps=eps) if qk_norm else nn.Identity() @@ -587,8 +588,8 @@ class LoRALinearLayer(nn.Module): #region crossattn class WanT2VCrossAttention(WanSelfAttention): - def __init__(self, in_features, out_features, num_heads, qk_norm=True, eps=1e-6, attention_mode='sdpa'): - super().__init__(in_features, out_features, num_heads, qk_norm, eps) + def __init__(self, in_features, out_features, num_heads, kv_dim=None, qk_norm=True, eps=1e-6, attention_mode='sdpa'): + super().__init__(in_features, out_features, num_heads, qk_norm, eps, kv_dim=kv_dim) self.attention_mode = attention_mode def forward(self, x, context, grid_sizes=None, clip_embed=None, audio_proj=None, audio_scale=1.0, @@ -721,6 +722,45 @@ class WanI2VCrossAttention(WanSelfAttention): x = x + adapter_x * ip_scale return self.o(x) + +class WanHuMoCrossAttention(WanSelfAttention): + + def __init__(self, in_features, out_features, num_heads, kv_dim=None, qk_norm=True, eps=1e-6, attention_mode='sdpa'): + super().__init__(in_features, out_features, num_heads, qk_norm, eps, kv_dim=kv_dim) + self.attention_mode = attention_mode + + def forward(self, x, context, grid_sizes, **kwargs): + + b, n, d = x.size(0), self.num_heads, self.head_dim + q = self.norm_q(self.q(x)).view(b, -1, n, d) + k = self.norm_k(self.k(context)).view(b, -1, n, d) + v = self.v(context).view(b, -1, n, d) + + # Handle video spatial structure + hlen_wlen = grid_sizes[0][1] * grid_sizes[0][2] + q = q.reshape(-1, hlen_wlen, n, d) + + # Handle audio temporal structure (16 tokens per frame) + k = k.reshape(-1, 16, n, d) + v = v.reshape(-1, 16, n, d) + + x_text = attention(q, k, v, attention_mode=self.attention_mode) + x_text = x_text.view(b, -1, n, d).flatten(2) + + x = x_text + + return self.o(x) + +class AudioCrossAttentionWrapper(nn.Module): + def __init__(self, in_features, out_features, num_heads, qk_norm=True, eps=1e-6, kv_dim=None): + super().__init__() + + self.audio_cross_attn = WanHuMoCrossAttention(in_features, out_features, num_heads, kv_dim=kv_dim) + self.norm1_audio = WanLayerNorm(out_features, eps, elementwise_affine=True) + + def forward(self, x, audio, grid_sizes, humo_audio_scale=1.0): + x = x + self.audio_cross_attn(self.norm1_audio(x), audio, grid_sizes) * humo_audio_scale + return x class MTVCrafterMotionAttention(WanSelfAttention): @@ -768,7 +808,8 @@ class WanAttentionBlock(nn.Module): eps=1e-6, attention_mode='sdpa', rope_func="comfy", - use_motion_attn=False + use_motion_attn=False, + use_humo_audio_attn=False, ): super().__init__() self.dim = out_features @@ -813,6 +854,10 @@ class WanAttentionBlock(nn.Module): self.modulation = nn.Parameter(torch.randn(1, 6, out_features) / in_features**0.5) self.seg_idx = None + # HuMo audio cross-attn + if use_humo_audio_attn: + self.audio_cross_attn_wrapper = AudioCrossAttentionWrapper(in_features, out_features, num_heads, qk_norm, eps, kv_dim=1536) + #@torch.compiler.disable() def get_mod(self, e): if e.dim() == 3: @@ -877,26 +922,21 @@ class WanAttentionBlock(nn.Module): num_latent_frames=21, original_seq_len=None, enhance_enabled=False, - block_mask=None, nag_params={}, nag_context=None, is_uncond=False, multitalk_audio_embedding=None, ref_target_masks=None, human_num=0, - inner_t=None, - inner_c=None, + inner_t=None, inner_c=None, cross_freqs=None, - x_ip=None, - e_ip=None, + x_ip=None, e_ip=None, freqs_ip=None, adapter_proj=None, ip_scale=1.0, reverse_time=False, - mtv_motion_tokens=None, - mtv_motion_rotary_emb=None, - mtv_strength=1.0, - mtv_freqs=None + mtv_motion_tokens=None, mtv_motion_rotary_emb=None, mtv_strength=1.0, mtv_freqs=None, + humo_audio_input=None, humo_audio_scale=1.0, ): r""" Args: @@ -1054,7 +1094,9 @@ class WanAttentionBlock(nn.Module): audio_proj, audio_scale, num_latent_frames, nag_params, nag_context, is_uncond, multitalk_audio_embedding, x_ref_attn_map, human_num, inner_t, inner_c, cross_freqs, adapter_proj=adapter_proj, ip_scale=ip_scale, - mtv_freqs=mtv_freqs, mtv_motion_tokens=mtv_motion_tokens, mtv_motion_rotary_emb=mtv_motion_rotary_emb, mtv_strength=mtv_strength) + mtv_freqs=mtv_freqs, mtv_motion_tokens=mtv_motion_tokens, mtv_motion_rotary_emb=mtv_motion_rotary_emb, mtv_strength=mtv_strength, + humo_audio_input=humo_audio_input, humo_audio_scale=humo_audio_scale + ) else: if self.rope_func == "comfy_chunked": y = self.ffn_chunked(x, shift_mlp, scale_mlp) @@ -1074,7 +1116,8 @@ class WanAttentionBlock(nn.Module): def cross_attn_ffn(self, x, context, grid_sizes, shift_mlp, scale_mlp, gate_mlp, clip_embed, audio_proj, audio_scale, num_latent_frames, nag_params, nag_context, is_uncond, multitalk_audio_embedding, x_ref_attn_map, human_num, - inner_t, inner_c, cross_freqs, adapter_proj, ip_scale, mtv_freqs, mtv_motion_tokens, mtv_motion_rotary_emb, mtv_strength): + inner_t, inner_c, cross_freqs, adapter_proj, ip_scale, mtv_freqs, mtv_motion_tokens, mtv_motion_rotary_emb, mtv_strength, + humo_audio_input, humo_audio_scale): x = x + self.cross_attn(self.norm3(x), context, grid_sizes, clip_embed=clip_embed, audio_proj=audio_proj, audio_scale=audio_scale, @@ -1092,6 +1135,10 @@ class WanAttentionBlock(nn.Module): x_motion = self.motion_attn(self.norm4(x), mtv_motion_tokens, mtv_motion_rotary_emb, grid_sizes, mtv_freqs) x = x + x_motion * mtv_strength + # HuMo Audio Cross-Attention + if humo_audio_input is not None: + x = self.audio_cross_attn_wrapper(x, humo_audio_input, grid_sizes, humo_audio_scale) + if self.rope_func == "comfy_chunked" and not self.zero_timestep: y = self.ffn_chunked(x, shift_mlp, scale_mlp) else: @@ -1415,7 +1462,8 @@ class WanModel(torch.nn.Module): enable_adain=False, adain_mode="attn_norm", audio_inject_layers=[0, 4, 8, 12, 16, 20, 24, 27, 30, 33, 36, 39], - zero_timestep=False + zero_timestep=False, + humo_audio=False, ): r""" Initialize the diffusion model backbone. @@ -1525,6 +1573,8 @@ class WanModel(torch.nn.Module): self.multitalk_model_type = "none" + self.humo_audio = humo_audio + # embeddings self.patch_embedding = nn.Conv3d( in_dim, dim, kernel_size=patch_size, stride=patch_size) @@ -1577,7 +1627,7 @@ class WanModel(torch.nn.Module): self.blocks = nn.ModuleList([ WanAttentionBlock(cross_attn_type, self.in_features, self.out_features, ffn_dim, ffn2_dim, num_heads, qk_norm, cross_attn_norm, eps, - attention_mode=self.attention_mode, rope_func=self.rope_func, use_motion_attn=(i % 4 == 0 and use_motion_attn)) + attention_mode=self.attention_mode, rope_func=self.rope_func, use_motion_attn=(i % 4 == 0 and use_motion_attn), use_humo_audio_attn=self.humo_audio) for i in range(num_layers) ]) #MTV Crafter @@ -1620,8 +1670,6 @@ class WanModel(torch.nn.Module): else: self.control_adapter = None - self.block_mask=None - #S2V self.zero_timestep = self.audio_injector = self.trainable_cond_mask =None if cond_dim > 0: @@ -1661,6 +1709,12 @@ class WanModel(torch.nn.Module): self.adain_mode = adain_mode self.zero_timestep = zero_timestep + # HuMo Audio + if self.humo_audio: + from ...HuMo.audio_proj import AudioProjModel + self.audio_proj = AudioProjModel(seq_len=8, blocks=5, channels=1280, + intermediate_dim=512, output_dim=1536, context_tokens=16) + def block_swap(self, blocks_to_swap, offload_txt_emb=False, offload_img_emb=False, vace_blocks_to_swap=None, prefetch_blocks=0, block_swap_debug=False): # Clamp blocks_to_swap to valid range @@ -1893,6 +1947,8 @@ class WanModel(torch.nn.Module): s2v_ref_motion=None, s2v_pose=None, s2v_motion_frames=[1, 0], + humo_audio=None, + humo_audio_scale=1.0, ): r""" @@ -2251,6 +2307,16 @@ class WanModel(torch.nn.Module): token_ref_target_masks = token_ref_target_masks.view(token_ref_target_masks.shape[0], -1) token_ref_target_masks = token_ref_target_masks.to(x.dtype).to(device) + humo_audio_input = None + if humo_audio is not None: + humo_audio_input = self.audio_proj(humo_audio.unsqueeze(0)).permute(0, 3, 1, 2) + + humo_audio_seq_len = torch.tensor(humo_audio.shape[2] * humo_audio_input.shape[3], device=device) + humo_audio_input = humo_audio_input.flatten(2).transpose(1, 2) # 1, t*32, 1536 + pad_len = int(humo_audio_seq_len - humo_audio_input.size(1)) + if pad_len > 0: + humo_audio_input = torch.nn.functional.pad(humo_audio_input, (0, 0, 0, pad_len)) + should_calc = True #TeaCache if self.enable_teacache and self.teacache_start_step <= current_step <= self.teacache_end_step: @@ -2392,25 +2458,21 @@ class WanModel(torch.nn.Module): original_seq_len=self.original_seq_len, enhance_enabled=enhance_enabled, audio_scale=audio_scale, - block_mask=self.block_mask, - nag_params=nag_params, - nag_context=nag_context, + nag_params=nag_params, nag_context=nag_context, is_uncond = is_uncond, multitalk_audio_embedding=multitalk_audio_embedding if multitalk_audio is not None else None, ref_target_masks=token_ref_target_masks if multitalk_audio is not None else None, human_num=human_num if multitalk_audio is not None else 0, - inner_t=inner_t, - inner_c=inner_c, + inner_t=inner_t, inner_c=inner_c, cross_freqs=self.cross_freqs if inner_t is not None else None, freqs_ip=freqs_ip if x_ip is not None else None, e_ip=e0_ip if x_ip is not None else None, adapter_proj=adapter_proj, ip_scale=ip_scale, reverse_time=reverse_time, - mtv_motion_tokens=mtv_motion_tokens, - mtv_motion_rotary_emb=mtv_motion_rotary_emb, - mtv_strength=mtv_strength, - mtv_freqs=mtv_freqs + mtv_motion_tokens=mtv_motion_tokens, mtv_motion_rotary_emb=mtv_motion_rotary_emb, mtv_strength=mtv_strength, mtv_freqs=mtv_freqs, + humo_audio_input=humo_audio_input, + humo_audio_scale=humo_audio_scale, ) if vace_data is not None: