diff --git a/nodes.py b/nodes.py index 50a570e..b7ad45b 100644 --- a/nodes.py +++ b/nodes.py @@ -1823,6 +1823,8 @@ class WanVideoSampler: log.info(f"sigmas: {sample_scheduler.sigmas}") else: timesteps = torch.tensor([1000, 750, 500, 250], device=device) + + log.info(f"timesteps: {timesteps}") total_steps = steps steps = len(timesteps) @@ -2223,14 +2225,19 @@ class WanVideoSampler: mtv_freqs = mtv_freqs.to(device, dtype) #region S2V - s2v_audio_input = s2v_ref_latent = None + s2v_audio_input = s2v_ref_latent = s2v_pose = s2v_ref_motion = None framepack = False s2v_audio_embeds = image_embeds.get("audio_embeds", None) if s2v_audio_embeds is not None: log.info(f"Using S2V audio embeddings") + framepack = s2v_audio_embeds.get("enable_framepack", False) + if framepack and context_options is not None: + raise ValueError("S2V framepack and context windows cannot be used at the same time") + s2v_audio_input = s2v_audio_embeds.get("audio_embed_bucket", None) if s2v_audio_input is not None: - s2v_audio_input = s2v_audio_input[..., 0:image_embeds["num_frames"]].to(device, dtype) + #s2v_audio_input = s2v_audio_input[..., 0:image_embeds["num_frames"]] + s2v_audio_input = s2v_audio_input.to(device, dtype) s2v_audio_scale = s2v_audio_embeds["audio_scale"] s2v_ref_latent = s2v_audio_embeds.get("ref_latent", None) if s2v_ref_latent is not None: @@ -2241,10 +2248,10 @@ class WanVideoSampler: s2v_pose = s2v_audio_embeds.get("pose_latent", None) if s2v_pose is not None: s2v_pose = s2v_pose.to(device, dtype) - + s2v_pose_start_percent = s2v_audio_embeds.get("pose_start_percent", 0.0) + s2v_pose_end_percent = s2v_audio_embeds.get("pose_end_percent", 1.0) s2v_num_repeat = s2v_audio_embeds.get("num_repeat", 1) - vae = image_embeds.get("vae", None) - framepack = False + vae = s2v_audio_embeds.get("vae", None) # vid2vid noise_mask=original_image=None @@ -2525,7 +2532,7 @@ class WanVideoSampler: def predict_with_cfg(z, cfg_scale, positive_embeds, negative_embeds, timestep, idx, image_cond=None, clip_fea=None, control_latents=None, vace_data=None, unianim_data=None, audio_proj=None, control_camera_latents=None, add_cond=None, cache_state=None, context_window=None, multitalk_audio_embeds=None, fantasy_portrait_input=None, reverse_time=False, - mtv_motion_tokens=None, s2v_audio_input=None, s2v_ref_motion=None): + mtv_motion_tokens=None, s2v_audio_input=None, s2v_ref_motion=None, s2v_motion_frames=[1, 0], s2v_pose=None): nonlocal transformer z = z.to(dtype) autocast_enabled = ("fp8" in model["quantization"] and not transformer.patched_linear) @@ -2670,7 +2677,11 @@ class WanVideoSampler: else: pcd_data_input = pcd_data - + if s2v_pose is not None: + if not ((s2v_pose_start_percent <= current_step_percentage <= s2v_pose_end_percent) or \ + (s2v_pose_end_percent > 0 and idx == 0 and current_step_percentage >= s2v_pose_start_percent)): + s2v_pose = None + base_params = { 'seq_len': seq_len, # sequence length 'device': device, # main device @@ -2707,7 +2718,8 @@ class WanVideoSampler: "s2v_ref_latent": s2v_ref_latent, # speech-to-video reference latent "s2v_ref_motion": s2v_ref_motion, # speech-to-video reference motion latent "s2v_audio_scale": s2v_audio_scale if s2v_audio_input is not None else 1.0, # speech-to-video audio scale - "s2v_pose": s2v_pose if s2v_pose is not None else None # speech-to-video pose control + "s2v_pose": s2v_pose if s2v_pose is not None else None, # speech-to-video pose control + "s2v_motion_frames": s2v_motion_frames, # speech-to-video motion frames } batch_size = 1 @@ -2862,7 +2874,7 @@ class WanVideoSampler: from .latent_preview import prepare_callback #custom for tiny VAE previews callback = prepare_callback(patcher, len(timesteps)) - if not multitalk_sampling: + if not multitalk_sampling and not framepack: log.info(f"Input sequence length: {seq_len}") log.info(f"Sampling {(latent_video_length-1) * 4 + 1} frames at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") @@ -3229,6 +3241,10 @@ class WanVideoSampler: center_indices = torch.clamp(center_indices, min=0, max=s2v_audio_input.shape[-1] - 1) partial_s2v_audio_input = s2v_audio_input[..., center_indices] + partial_s2v_pose = None + if s2v_pose is not None: + partial_s2v_pose = s2v_pose[:, :, c].to(device, dtype) + partial_add_cond = None if add_cond is not None: partial_add_cond = add_cond[:, :, c].to(device, dtype) @@ -3246,7 +3262,7 @@ class WanVideoSampler: text_embeds["negative_prompt_embeds"], partial_timestep, idx, partial_img_emb, clip_fea, partial_control_latents, partial_vace_context, partial_unianim_data,partial_audio_proj, partial_control_camera_latents, partial_add_cond, current_teacache, context_window=c, fantasy_portrait_input=partial_fantasy_portrait_input, - mtv_motion_tokens=partial_mtv_motion_tokens, s2v_audio_input=partial_s2v_audio_input) + mtv_motion_tokens=partial_mtv_motion_tokens, s2v_audio_input=partial_s2v_audio_input, s2v_motion_frames=[1, 0], s2v_pose=partial_s2v_pose) if cache_args is not None: self.window_tracker.cache_states[window_id] = new_teacache @@ -3671,67 +3687,124 @@ class WanVideoSampler: except: pass return {"video": gen_video_samples.permute(1, 2, 3, 0)}, + # region framepack loop elif framepack: framepack_out = [] ref_motion_image = None - motion_frames = 5 - infer_frames = image_embeds["num_frames"] + #infer_frames = image_embeds["num_frames"] + infer_frames = s2v_audio_embeds.get("frame_window_size", 80) + motion_frames = infer_frames - 7 #73 default + lat_motion_frames = (motion_frames + 3) // 4 + lat_target_frames = (infer_frames + 3 + motion_frames) // 4 - lat_motion_frames + + step_iteration_count = 0 + total_frames = s2v_audio_input.shape[-1] + s2v_motion_frames = [motion_frames, lat_motion_frames] + + noise = torch.randn( #C, T, H, W + 48 if is_5b else 16, + lat_target_frames, + target_shape[2], + target_shape[3], + dtype=torch.float32, + generator=seed_g, + device=torch.device("cpu")) + + seq_len = math.ceil((noise.shape[2] * noise.shape[3]) / 4 * noise.shape[1]) + + if ref_motion_image is None: + ref_motion_image = torch.zeros( + [1, 3, motion_frames, latent.shape[2]*vae_upscale_factor, latent.shape[3]*vae_upscale_factor], + dtype=vae.dtype, + device=device) + videos_last_frames = ref_motion_image + + pose_cond_list = [] + for r in range(s2v_num_repeat): + pose_start = r * (infer_frames // 4) + pose_end = pose_start + (infer_frames // 4) + + cond_lat = s2v_pose[:, :, pose_start:pose_end] + + pad_len = (infer_frames // 4) - cond_lat.shape[2] + if pad_len > 0: + pad = -torch.ones(cond_lat.shape[0], cond_lat.shape[1], pad_len, cond_lat.shape[3], cond_lat.shape[4], device=cond_lat.device, dtype=cond_lat.dtype) + cond_lat = torch.cat([cond_lat, pad], dim=2) + pose_cond_list.append(cond_lat.cpu()) + + log.info(f"Sampling {total_frames} frames in {s2v_num_repeat} windows, at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") + # sample for r in range(s2v_num_repeat): if ref_motion_image is not None: - if ref_motion_image.shape[0] > 73: - ref_motion_image = ref_motion_image[-73:] - - if ref_motion_image.shape[0] < 73: - ref = torch.ones([73, ref_motion_image.shape[1], ref_motion_image.shape[2], 3]) * 0.5 - ref[-ref_motion_image.shape[0]:] = ref_motion_image - ref_motion_image = ref - vae.to(device) - ref_motion = vae.encode(ref_motion_image[:, :, :, :3], device=device, pbar=False)[0].to(dtype) + ref_motion = vae.encode(ref_motion_image.to(vae.dtype), device=device, pbar=False).to(dtype)[0] vae.to(offload_device) left_idx = r * infer_frames right_idx = r * infer_frames + infer_frames - #cond_latents = COND[r] if pose_video else COND[0] * 0 - #cond_latents = cond_latents.to(dtype=self.param_dtype, device=self.device) - s2v_audio_input = s2v_audio_embeds[..., left_idx:right_idx] - input_motion_latents = ref_motion.clone() - - noise_pred, self.cache_state = predict_with_cfg( - latent_model_input, - cfg[idx], - text_embeds["prompt_embeds"], - text_embeds["negative_prompt_embeds"], - timestep, idx, image_cond, clip_fea, control_latents, vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, - cache_state=self.cache_state, fantasy_portrait_input=fantasy_portrait_input, mtv_motion_tokens=mtv_motion_tokens, - s2v_audio_input=s2v_audio_input, s2v_ref_motion=input_motion_latents) - latent = sample_scheduler.step( - noise_pred.unsqueeze(0), timestep, latent.unsqueeze(0), - **scheduler_step_args)[0].squeeze(0) - - latents = torch.stack(latent) - #if not (drop_first_motion and r == 0): - # decode_latents = torch.cat([motion_latents, latents], dim=2) - #else: - decode_latents = torch.cat([s2v_ref_latent, latents], dim=2) - image = torch.stack(vae.decode(decode_latents), device=device) - image = image[:, :, -(infer_frames):] - #if (drop_first_motion and r == 0): - # image = image[:, :, 3:] + s2v_audio_input_slice = s2v_audio_input[..., left_idx:right_idx] + if s2v_audio_input_slice.shape[-1] < (right_idx - left_idx): + pad_len = (right_idx - left_idx) - s2v_audio_input_slice.shape[-1] + pad_shape = list(s2v_audio_input_slice.shape) + pad_shape[-1] = pad_len + pad = torch.zeros(pad_shape, device=s2v_audio_input_slice.device, dtype=s2v_audio_input_slice.dtype) + log.info(f"Padding s2v_audio_input_slice from {s2v_audio_input_slice.shape[-1]} to {right_idx - left_idx}") + s2v_audio_input_slice = torch.cat([s2v_audio_input_slice, pad], dim=-1) - overlap_frames_num = min(motion_frames, image.shape[2]) - videos_last_frames = torch.cat([ - videos_last_frames[:, :, overlap_frames_num:], - image[:, :, -overlap_frames_num:]], dim=2).to(vae.device, vae.dtype) - + if ref_motion_image is not None: + input_motion_latents = ref_motion.clone().unsqueeze(0) + else: + input_motion_latents = None + + if s2v_pose is not None: + s2v_pose_slice = pose_cond_list[r].to(device) + + sample_scheduler, timesteps,_,_ = get_scheduler(scheduler, total_steps, start_step, end_step, shift, device, transformer.dim, flowedit_args, denoise_strength, sigmas=sigmas) + + latent = noise.to(device) + for i, t in enumerate(tqdm(timesteps, desc=f"Sampling audio indices {left_idx}-{right_idx}", position=0)): + latent_model_input = latent.to(device) + timestep = torch.tensor([t]).to(device) + noise_pred, self.cache_state = predict_with_cfg( + latent_model_input, + cfg[idx], + text_embeds["prompt_embeds"], + text_embeds["negative_prompt_embeds"], + timestep, idx, image_cond, clip_fea, control_latents, vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, + cache_state=self.cache_state, fantasy_portrait_input=fantasy_portrait_input, mtv_motion_tokens=mtv_motion_tokens, + s2v_audio_input=s2v_audio_input_slice, s2v_ref_motion=input_motion_latents, s2v_motion_frames=s2v_motion_frames, s2v_pose=s2v_pose_slice) + + latent = sample_scheduler.step( + noise_pred.unsqueeze(0), timestep, latent.unsqueeze(0), + **scheduler_step_args)[0].squeeze(0) + if callback is not None: + callback_latent = (latent_model_input.to(device) - noise_pred.to(device) * t.to(device) / 1000).detach().permute(1,0,2,3) + callback(step_iteration_count, callback_latent, None, s2v_num_repeat*(len(timesteps))) + del callback_latent + step_iteration_count += 1 + + vae.to(device) - ref_motion_image = torch.stack(vae.encode(videos_last_frames, device=device, pbar=False)[0]) - vae.to(device) + decode_latents = torch.cat([ref_motion.unsqueeze(0), latent.unsqueeze(0)], dim=2) + image = vae.decode(decode_latents.to(device, vae.dtype), device=device, pbar=False)[0] + image = image.unsqueeze(0)[:, :, -infer_frames:] + if r == 0: + image = image[:, :, 3:] + framepack_out.append(image.cpu()) - gen_video_samples = torch.cat(framepack_out, dim=1) + overlap_frames_num = min(motion_frames, image.shape[2]) + + videos_last_frames = torch.cat([ + videos_last_frames[:, :, overlap_frames_num:], + image[:, :, -overlap_frames_num:]], dim=2).to(device, vae.dtype) + + ref_motion_image = videos_last_frames + + vae.to(offload_device) + gen_video_samples = torch.cat(framepack_out, dim=2).squeeze(0).permute(1, 2, 3, 0) if force_offload: if not model["auto_cpu_offload"]: @@ -3741,7 +3814,7 @@ class WanVideoSampler: torch.cuda.reset_peak_memory_stats(device) except: pass - return {"video": gen_video_samples.permute(1, 2, 3, 0)}, + return {"video": gen_video_samples}, #region normal inference else: diff --git a/nodes_model_loading.py b/nodes_model_loading.py index c5f9002..4612e95 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -731,7 +731,8 @@ class WanVideoSetLoRAs: def load_weights(transformer, sd=None, weight_dtype=None, base_dtype=None, transformer_load_device=None, block_swap_args=None, gguf=False, reader=None, patcher=None): - params_to_keep = {"time_in", "patch_embedding", "time_", "modulation", "text_embedding", "adapter", "add", "ref_conv", "audio", "cond_encoder"} + params_to_keep = {"time_in", "patch_embedding", "time_", "modulation", "text_embedding", + "adapter", "add", "ref_conv", "casual_audio_encoder", "cond_encoder", "frame_packer"} param_count = sum(1 for _ in transformer.named_parameters()) pbar = ProgressBar(param_count) cnt = 0 @@ -878,6 +879,7 @@ def patch_stand_in_lora(transformer, lora_sd, transformer_load_device, base_dtyp def add_lora_weights(patcher, lora, base_dtype, merge_loras=False): unianimate_sd = None + control_lora=False #spacepxl's control LoRA patch for l in lora: log.info(f"Loading LoRA: {l['name']} with strength: {l['strength']}") @@ -904,7 +906,6 @@ def add_lora_weights(patcher, lora, base_dtype, merge_loras=False): # Filter out any LoRA keys containing 'img' if the base model state_dict has no 'img' keys #if not any('img' in k for k in sd.keys()): # lora_sd = {k: v for k, v in lora_sd.items() if 'img' not in k} - control_lora=False if "diffusion_model.patch_embedding.lora_A.weight" in lora_sd: control_lora = True #stand-in LoRA patch diff --git a/s2v/nodes.py b/s2v/nodes.py index f854e16..5c18cc6 100644 --- a/s2v/nodes.py +++ b/s2v/nodes.py @@ -46,47 +46,55 @@ def linear_interpolation(features, input_fps, output_fps, output_len=None): mode='linear') # [1, 512, output_len] return output_features.transpose(1, 2) # [1, output_len, 512] -class WanVideoAddAudioEmbeds: +class WanVideoAddS2VEmbeds: @classmethod def INPUT_TYPES(s): return {"required": { "embeds": ("WANVIDIMAGE_EMBEDS",), - "frames": ("INT", {"default": 81, "min": 1, "max": 100000, "step": 1, "tooltip": "Number of frames to process"}), - "audio_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1, "tooltip": "Scale factor for audio embeddings"}) + "frame_window_size": ("INT", {"default": 80, "min": 1, "max": 100000, "step": 1, "tooltip": "Number of frames in a single window"}), + "audio_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1, "tooltip": "Scale factor for audio embeddings"}), + "pose_start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Start percentage for pose embeddings"}), + "pose_end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "End percentage for pose embeddings"}) }, "optional": { "audio_encoder_output": ("AUDIO_ENCODER_OUTPUT",), "ref_latent": ("LATENT",), - "pose_latent": ("LATENT",) + "pose_latent": ("LATENT",), + "vae": ("WANVAE",), + "enable_framepack": ("BOOLEAN", {"default": False, "tooltip": "Enable Framepack sampling loop, not compatible with context windows"}) } - } - RETURN_TYPES = ("WANVIDIMAGE_EMBEDS",) - RETURN_NAMES = ("image_embeds",) + RETURN_TYPES = ("WANVIDIMAGE_EMBEDS", "INT",) + RETURN_NAMES = ("image_embeds", "audio_frame_count") FUNCTION = "add" CATEGORY = "WanVideoWrapper" - def add(self, embeds, frames, audio_encoder_output=None, audio_scale=1.0, ref_latent=None, pose_latent=None): + def add(self, embeds, frame_window_size, audio_encoder_output=None, audio_scale=1.0, ref_latent=None, pose_latent=None, vae=None, pose_start_percent=0.0, pose_end_percent=1.0, enable_framepack=False): if audio_encoder_output is not None: all_layers = audio_encoder_output["encoded_audio_all_layers"] audio_feat = torch.stack(all_layers, dim=0).squeeze(1) # shape: [num_layers, T, 512] - print("audio_feat", audio_feat.shape) - input_fps = 50 - output_fps = 30 - bucket_fps = 16 + print("audio_feat in", audio_feat.shape) + input_fps = 50 # determined by the model itself + output_fps = 30 # determined by the model itself + bucket_fps = 16 # target fps for the generation if input_fps != output_fps: audio_feat = linear_interpolation(audio_feat, input_fps=input_fps, output_fps=output_fps) + print("audio_feat after interpolation", audio_feat.shape) + + audio_feat = audio_feat[:, :embeds["num_frames"] * output_fps // bucket_fps, :] + print("audio_feat after trim", audio_feat.shape) self.video_rate = output_fps audio_embed_bucket, num_repeat = self.get_audio_embed_bucket_fps( audio_feat, fps=bucket_fps, - batch_frames=frames-1 + batch_frames=frame_window_size ) + print("audio_embed_bucket", audio_embed_bucket.shape) audio_embed_bucket = audio_embed_bucket.unsqueeze(0) if len(audio_embed_bucket.shape) == 3: @@ -94,6 +102,8 @@ class WanVideoAddAudioEmbeds: elif len(audio_embed_bucket.shape) == 4: audio_embed_bucket = audio_embed_bucket.permute(0, 2, 3, 1) + audio_frame_count = audio_embed_bucket.shape[-1] + print("audio_embed_bucket", audio_embed_bucket.shape) new_entry = { @@ -101,11 +111,16 @@ class WanVideoAddAudioEmbeds: "num_repeat": num_repeat if audio_encoder_output is not None else None, "ref_latent": ref_latent["samples"] if ref_latent is not None else None, "pose_latent": pose_latent["samples"] if pose_latent is not None else None, - "audio_scale": audio_scale + "audio_scale": audio_scale, + "vae": vae, + "pose_start_percent": pose_start_percent, + "pose_end_percent": pose_end_percent, + "enable_framepack": enable_framepack, + "frame_window_size": frame_window_size } updated = dict(embeds) updated["audio_embeds"] = new_entry - return (updated,) + return (updated, audio_frame_count) def get_audio_embed_bucket_fps(self, audio_embed, fps=16, batch_frames=81, m=0): num_layers, audio_frame_num, audio_dim = audio_embed.shape @@ -120,8 +135,7 @@ class WanVideoAddAudioEmbeds: min_batch_num = int(audio_frame_num / (batch_frames * scale)) + 1 bucket_num = min_batch_num * batch_frames - padd_audio_num = math.ceil(min_batch_num * batch_frames / fps * - self.video_rate) - audio_frame_num + padd_audio_num = math.ceil(min_batch_num * batch_frames / fps * self.video_rate) - audio_frame_num batch_idx = get_sample_indices( original_fps=self.video_rate, total_frames=audio_frame_num + padd_audio_num, @@ -161,9 +175,9 @@ class WanVideoAddAudioEmbeds: NODE_CLASS_MAPPINGS = { - "WanVideoAddAudioEmbeds": WanVideoAddAudioEmbeds, + "WanVideoAddS2VEmbeds": WanVideoAddS2VEmbeds, } NODE_DISPLAY_NAME_MAPPINGS = { - "WanVideoAddAudioEmbeds": "WanVideo Add Audio Embeds", + "WanVideoAddS2VEmbeds": "WanVideo Add S2V Embeds", } \ No newline at end of file diff --git a/s2v/wanvideo2_2_S2V_context_window_testing.json b/s2v/wanvideo2_2_S2V_context_window_testing.json index 16e0a8e..d15e17a 100644 --- a/s2v/wanvideo2_2_S2V_context_window_testing.json +++ b/s2v/wanvideo2_2_S2V_context_window_testing.json @@ -1,8 +1,8 @@ { "id": "c6e410bc-5e2c-460b-ae81-c91b6094fbb1", "revision": 0, - "last_node_id": 100, - "last_link_id": 155, + "last_node_id": 102, + "last_link_id": 164, "nodes": [ { "id": 44, @@ -402,7 +402,7 @@ "type": "LATENT", "links": [ 92, - 128 + 157 ] } ], @@ -602,52 +602,6 @@ "color": "#223", "bgcolor": "#335" }, - { - "id": 83, - "type": "WanVideoContextOptions", - "pos": [ - 1290.159423828125, - -496.63787841796875 - ], - "size": [ - 275.783203125, - 202 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "reference_latent", - "shape": 7, - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "context_options", - "type": "WANVIDCONTEXT", - "links": [ - 121 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WanVideoWrapper", - "ver": "3c79851230c9ab042f52c8e176f349ecc3e51a64", - "Node name for S&R": "WanVideoContextOptions" - }, - "widgets_values": [ - "uniform_standard", - 73, - 4, - 16, - true, - false, - "linear" - ] - }, { "id": 77, "type": "InsertLatentToIndexed", @@ -758,7 +712,8 @@ "0, 0, 0", "center", 2, - "cpu" + "cpu", + "Output: 1 x 960 x 640 | 7.03MB" ] }, { @@ -773,7 +728,7 @@ 202 ], "flags": {}, - "order": 9, + "order": 8, "mode": 0, "inputs": [], "outputs": [ @@ -845,7 +800,7 @@ "links": null }, { - "label": "601 count", + "label": "201 count", "name": "count", "type": "INT", "links": null @@ -870,7 +825,7 @@ 342 ], "flags": {}, - "order": 10, + "order": 9, "mode": 0, "inputs": [ { @@ -917,6 +872,503 @@ "color": "#223", "bgcolor": "#335" }, + { + "id": 94, + "type": "VHS_LoadAudio", + "pos": [ + -59.62910079956055, + 496.306396484375 + ], + "size": [ + 415.6340026855469, + 126 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 148, + 153, + 154 + ] + }, + { + "name": "duration", + "type": "FLOAT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "8e4d79471bf1952154768e8435a9300077b534fa", + "Node name for S&R": "VHS_LoadAudio" + }, + "widgets_values": { + "audio_file": "input/weightoftheworld2.mp4", + "seek_seconds": 0, + "duration": 0 + } + }, + { + "id": 66, + "type": "LoadAudio", + "pos": [ + -40.06507873535156, + 719.501953125 + ], + "size": [ + 361.2844543457031, + 155.81912231445312 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "NieR_ Automata - _Weight of the World_ ENG VER. by Lizz Robinett [CyOSTbel3AM].mp3", + null, + null + ] + }, + { + "id": 81, + "type": "MelBandRoFormerModelLoader", + "pos": [ + 563.452392578125, + 647.8037109375 + ], + "size": [ + 316.2164001464844, + 58 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "MELROFORMERMODEL", + "links": [ + 106 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-MelBandRoFormer", + "ver": "b40e263224778ec417114d91d8b3b39934e30de5", + "Node name for S&R": "MelBandRoFormerModelLoader" + }, + "widgets_values": [ + "MelBandRoFormer\\MelBandRoformer_fp16.safetensors" + ] + }, + { + "id": 82, + "type": "MelBandRoFormerSampler", + "pos": [ + 555.0003662109375, + 770.088623046875 + ], + "size": [ + 222.73397827148438, + 46 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MELROFORMERMODEL", + "link": 106 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 154 + } + ], + "outputs": [ + { + "name": "vocals", + "type": "AUDIO", + "links": [ + 149 + ] + }, + { + "name": "instruments", + "type": "AUDIO", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-MelBandRoFormer", + "ver": "b40e263224778ec417114d91d8b3b39934e30de5", + "Node name for S&R": "MelBandRoFormerSampler" + }, + "widgets_values": [] + }, + { + "id": 98, + "type": "NormalizeAudioLoudness", + "pos": [ + 834.3135986328125, + 760.9041137695312 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 149 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 150 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "90c3bbb6c2e4ff5e05305e765d007d5e58428ce4", + "Node name for S&R": "NormalizeAudioLoudness" + }, + "widgets_values": [ + -23 + ] + }, + { + "id": 95, + "type": "DownloadAndLoadGIMMVFIModel", + "pos": [ + 3262.200927734375, + -713.7930908203125 + ], + "size": [ + 339.4301452636719, + 106 + ], + "flags": {}, + "order": 13, + "mode": 2, + "inputs": [], + "outputs": [ + { + "name": "gimmvfi_model", + "type": "GIMMVIF_MODEL", + "links": [ + 144 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-GIMM-VFI", + "ver": "4c9a3123762af85e7c796e41737da0b70c75d72d", + "Node name for S&R": "DownloadAndLoadGIMMVFIModel" + }, + "widgets_values": [ + "gimmvfi_r_arb_lpips_fp32.safetensors", + "fp16", + false + ] + }, + { + "id": 80, + "type": "VHS_SplitImages", + "pos": [ + 2397.607666015625, + -214.64483642578125 + ], + "size": [ + 210, + 118 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 100 + } + ], + "outputs": [ + { + "name": "IMAGE_A", + "type": "IMAGE", + "links": null + }, + { + "name": "A_count", + "type": "INT", + "links": null + }, + { + "name": "IMAGE_B", + "type": "IMAGE", + "links": [ + 145, + 155 + ] + }, + { + "name": "B_count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "8e4d79471bf1952154768e8435a9300077b534fa", + "Node name for S&R": "VHS_SplitImages" + }, + "widgets_values": { + "split_index": 3 + } + }, + { + "id": 64, + "type": "AudioEncoderEncode", + "pos": [ + 1198.93212890625, + 778.826904296875 + ], + "size": [ + 285.087890625, + 46 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "audio_encoder", + "type": "AUDIO_ENCODER", + "link": 69 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 150 + } + ], + "outputs": [ + { + "name": "AUDIO_ENCODER_OUTPUT", + "type": "AUDIO_ENCODER_OUTPUT", + "links": [ + 156 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "AudioEncoderEncode" + }, + "widgets_values": [] + }, + { + "id": 65, + "type": "AudioEncoderLoader", + "pos": [ + 1136.3577880859375, + 641.1036376953125 + ], + "size": [ + 346.756103515625, + 58 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO_ENCODER", + "type": "AUDIO_ENCODER", + "links": [ + 69 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "AudioEncoderLoader" + }, + "widgets_values": [ + "wav2vec_xlsr_53_english_fp32.safetensors" + ] + }, + { + "id": 69, + "type": "PreviewAny", + "pos": [ + 2732.885009765625, + -1020.7718505859375 + ], + "size": [ + 490.69390869140625, + 355.7907409667969 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 159 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [] + }, + { + "id": 37, + "type": "WanVideoEmptyEmbeds", + "pos": [ + 1442.7791748046875, + -1070.2607421875 + ], + "size": [ + 315, + 126 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "control_embeds", + "shape": 7, + "type": "WANVIDIMAGE_EMBEDS", + "link": null + }, + { + "name": "extra_latents", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 85 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 86 + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 79 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 158 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoEmptyEmbeds" + }, + "widgets_values": [ + 832, + 480, + 201 + ] + }, + { + "id": 71, + "type": "PrimitiveNode", + "pos": [ + 1388.9095458984375, + -791.9364013671875 + ], + "size": [ + 210, + 82 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "links": [ + 79, + 161 + ] + } + ], + "title": "num_frames", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 201, + "fixed" + ] + }, { "id": 27, "type": "WanVideoSampler", @@ -929,7 +1381,7 @@ 900.6666870117188 ], "flags": {}, - "order": 29, + "order": 28, "mode": 0, "inputs": [ { @@ -940,7 +1392,7 @@ { "name": "image_embeds", "type": "WANVIDIMAGE_EMBEDS", - "link": 91 + "link": 162 }, { "name": "text_embeds", @@ -1055,13 +1507,13 @@ "Node name for S&R": "WanVideoSampler" }, "widgets_values": [ - 6, + 4, 1, 4, 45, "fixed", true, - "lcm", + "dpm++_sde", 0, 1, false, @@ -1072,205 +1524,314 @@ ] }, { - "id": 94, - "type": "VHS_LoadAudio", + "id": 101, + "type": "WanVideoAddS2VEmbeds", "pos": [ - -59.62910079956055, - 496.306396484375 + 2000.4896240234375, + -871.2449340820312 ], "size": [ - 415.6340026855469, - 126 + 327.17578125, + 234 ], "flags": {}, - "order": 11, + "order": 27, "mode": 0, - "inputs": [], + "inputs": [ + { + "name": "embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 158 + }, + { + "name": "audio_encoder_output", + "shape": 7, + "type": "AUDIO_ENCODER_OUTPUT", + "link": 156 + }, + { + "name": "ref_latent", + "shape": 7, + "type": "LATENT", + "link": 157 + }, + { + "name": "pose_latent", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "WANVAE", + "link": null + }, + { + "name": "frame_window_size", + "type": "INT", + "widget": { + "name": "frame_window_size" + }, + "link": 161 + } + ], "outputs": [ { - "name": "audio", - "type": "AUDIO", + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", "links": [ - 148, - 153, - 154 + 162 ] }, { - "name": "duration", - "type": "FLOAT", + "name": "audio_frame_count", + "type": "INT", + "links": [ + 159 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "a5621b87391013155b4f688fbe01dba10a8104aa", + "Node name for S&R": "WanVideoAddS2VEmbeds" + }, + "widgets_values": [ + 201, + 1, + 0, + 1, + false + ] + }, + { + "id": 83, + "type": "WanVideoContextOptions", + "pos": [ + 1290.159423828125, + -496.63787841796875 + ], + "size": [ + 275.783203125, + 202 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "reference_latent", + "shape": 7, + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "context_options", + "type": "WANVIDCONTEXT", + "links": [ + 121 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3c79851230c9ab042f52c8e176f349ecc3e51a64", + "Node name for S&R": "WanVideoContextOptions" + }, + "widgets_values": [ + "uniform_standard", + 81, + 4, + 16, + true, + false, + "linear" + ] + }, + { + "id": 96, + "type": "GIMMVFI_interpolate", + "pos": [ + 3209.548583984375, + -509.1043701171875 + ], + "size": [ + 270, + 174 + ], + "flags": {}, + "order": 34, + "mode": 2, + "inputs": [ + { + "name": "gimmvfi_model", + "type": "GIMMVIF_MODEL", + "link": 144 + }, + { + "name": "images", + "type": "IMAGE", + "link": 145 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 163 + ] + }, + { + "name": "flow_tensors", + "type": "IMAGE", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-GIMM-VFI", + "ver": "4c9a3123762af85e7c796e41737da0b70c75d72d", + "Node name for S&R": "GIMMVFI_interpolate" + }, + "widgets_values": [ + 1, + 3, + 0, + "fixed", + false + ] + }, + { + "id": 30, + "type": "VHS_VideoCombine", + "pos": [ + 3815.402099609375, + -864.041748046875 + ], + "size": [ + 940.8292846679688, + 334 + ], + "flags": {}, + "order": 37, + "mode": 2, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 164 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 153 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "330bce6c3c0d47ebdedcc0348d9ab355707b7523", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "WanVideo2_2_S2V", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_2_S2V_00013-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 32, + "workflow": "WanVideo2_2_S2V_00013.png", + "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_2_S2V_00013-audio.mp4" + } + } + } + }, + { + "id": 102, + "type": "VHS_SelectEveryNthImage", + "pos": [ + 3513.563720703125, + -508.5723876953125 + ], + "size": [ + 266.349609375, + 102 + ], + "flags": {}, + "order": 36, + "mode": 2, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 163 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 164 + ] + }, + { + "name": "count", + "type": "INT", "links": null } ], "properties": { "cnr_id": "comfyui-videohelpersuite", "ver": "8e4d79471bf1952154768e8435a9300077b534fa", - "Node name for S&R": "VHS_LoadAudio" + "Node name for S&R": "VHS_SelectEveryNthImage" }, "widgets_values": { - "audio_file": "input/weightoftheworld2.mp4", - "seek_seconds": 0, - "duration": 0 + "select_every_nth": 2, + "skip_first_images": 0 } }, - { - "id": 66, - "type": "LoadAudio", - "pos": [ - -40.06507873535156, - 719.501953125 - ], - "size": [ - 361.2844543457031, - 155.81912231445312 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "AUDIO", - "type": "AUDIO", - "links": [] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.52", - "Node name for S&R": "LoadAudio" - }, - "widgets_values": [ - "NieR_ Automata - _Weight of the World_ ENG VER. by Lizz Robinett [CyOSTbel3AM].mp3", - null, - null - ] - }, - { - "id": 81, - "type": "MelBandRoFormerModelLoader", - "pos": [ - 563.452392578125, - 647.8037109375 - ], - "size": [ - 316.2164001464844, - 58 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "model", - "type": "MELROFORMERMODEL", - "links": [ - 106 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-MelBandRoFormer", - "ver": "b40e263224778ec417114d91d8b3b39934e30de5", - "Node name for S&R": "MelBandRoFormerModelLoader" - }, - "widgets_values": [ - "MelBandRoFormer\\MelBandRoformer_fp16.safetensors" - ] - }, - { - "id": 82, - "type": "MelBandRoFormerSampler", - "pos": [ - 555.0003662109375, - 770.088623046875 - ], - "size": [ - 222.73397827148438, - 46 - ], - "flags": {}, - "order": 19, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MELROFORMERMODEL", - "link": 106 - }, - { - "name": "audio", - "type": "AUDIO", - "link": 154 - } - ], - "outputs": [ - { - "name": "vocals", - "type": "AUDIO", - "links": [ - 149 - ] - }, - { - "name": "instruments", - "type": "AUDIO", - "links": null - } - ], - "properties": { - "cnr_id": "ComfyUI-MelBandRoFormer", - "ver": "b40e263224778ec417114d91d8b3b39934e30de5", - "Node name for S&R": "MelBandRoFormerSampler" - }, - "widgets_values": [] - }, - { - "id": 98, - "type": "NormalizeAudioLoudness", - "pos": [ - 834.3135986328125, - 760.9041137695312 - ], - "size": [ - 270, - 58 - ], - "flags": {}, - "order": 24, - "mode": 0, - "inputs": [ - { - "name": "audio", - "type": "AUDIO", - "link": 149 - } - ], - "outputs": [ - { - "name": "audio", - "type": "AUDIO", - "links": [ - 150 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WanVideoWrapper", - "ver": "90c3bbb6c2e4ff5e05305e765d007d5e58428ce4", - "Node name for S&R": "NormalizeAudioLoudness" - }, - "widgets_values": [ - -23 - ] - }, { "id": 97, "type": "VHS_VideoCombine", "pos": [ - 3319.513671875, - 193.33419799804688 + 2914.45703125, + -176.39088439941406 ], "size": [ 940.8292846679688, @@ -1331,509 +1892,16 @@ "hidden": false, "paused": false, "params": { - "filename": "WanVideo2_2_S2V_00012-audio.mp4", + "filename": "WanVideo2_2_S2V_00015-audio.mp4", "subfolder": "", "type": "temp", "format": "video/h264-mp4", "frame_rate": 16, - "workflow": "WanVideo2_2_S2V_00012.png", - "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_2_S2V_00012-audio.mp4" + "workflow": "WanVideo2_2_S2V_00015.png", + "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_2_S2V_00015-audio.mp4" } } } - }, - { - "id": 30, - "type": "VHS_VideoCombine", - "pos": [ - 3649.713623046875, - -868.4019165039062 - ], - "size": [ - 940.8292846679688, - 961.8861694335938 - ], - "flags": {}, - "order": 36, - "mode": 2, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 146 - }, - { - "name": "audio", - "shape": 7, - "type": "AUDIO", - "link": 153 - }, - { - "name": "meta_batch", - "shape": 7, - "type": "VHS_BatchManager", - "link": null - }, - { - "name": "vae", - "shape": 7, - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null - } - ], - "properties": { - "cnr_id": "comfyui-videohelpersuite", - "ver": "330bce6c3c0d47ebdedcc0348d9ab355707b7523", - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 32, - "loop_count": 0, - "filename_prefix": "WanVideo2_2_S2V", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 19, - "save_metadata": true, - "trim_to_audio": false, - "pingpong": false, - "save_output": false, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "WanVideo2_2_S2V_00013-audio.mp4", - "subfolder": "", - "type": "temp", - "format": "video/h264-mp4", - "frame_rate": 32, - "workflow": "WanVideo2_2_S2V_00013.png", - "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_2_S2V_00013-audio.mp4" - } - } - } - }, - { - "id": 96, - "type": "GIMMVFI_interpolate", - "pos": [ - 3256.453369140625, - -512.8270263671875 - ], - "size": [ - 270, - 174 - ], - "flags": {}, - "order": 34, - "mode": 2, - "inputs": [ - { - "name": "gimmvfi_model", - "type": "GIMMVIF_MODEL", - "link": 144 - }, - { - "name": "images", - "type": "IMAGE", - "link": 145 - } - ], - "outputs": [ - { - "name": "images", - "type": "IMAGE", - "links": [ - 146 - ] - }, - { - "name": "flow_tensors", - "type": "IMAGE", - "links": null - } - ], - "properties": { - "cnr_id": "ComfyUI-GIMM-VFI", - "ver": "4c9a3123762af85e7c796e41737da0b70c75d72d", - "Node name for S&R": "GIMMVFI_interpolate" - }, - "widgets_values": [ - 1, - 2, - 0, - "fixed", - false - ] - }, - { - "id": 95, - "type": "DownloadAndLoadGIMMVFIModel", - "pos": [ - 3262.200927734375, - -713.7930908203125 - ], - "size": [ - 339.4301452636719, - 106 - ], - "flags": {}, - "order": 14, - "mode": 2, - "inputs": [], - "outputs": [ - { - "name": "gimmvfi_model", - "type": "GIMMVIF_MODEL", - "links": [ - 144 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GIMM-VFI", - "ver": "4c9a3123762af85e7c796e41737da0b70c75d72d", - "Node name for S&R": "DownloadAndLoadGIMMVFIModel" - }, - "widgets_values": [ - "gimmvfi_r_arb_lpips_fp32.safetensors", - "fp16", - false - ] - }, - { - "id": 80, - "type": "VHS_SplitImages", - "pos": [ - 2397.607666015625, - -214.64483642578125 - ], - "size": [ - 210, - 118 - ], - "flags": {}, - "order": 33, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 100 - } - ], - "outputs": [ - { - "name": "IMAGE_A", - "type": "IMAGE", - "links": null - }, - { - "name": "A_count", - "type": "INT", - "links": null - }, - { - "name": "IMAGE_B", - "type": "IMAGE", - "links": [ - 145, - 155 - ] - }, - { - "name": "B_count", - "type": "INT", - "links": null - } - ], - "properties": { - "cnr_id": "comfyui-videohelpersuite", - "ver": "8e4d79471bf1952154768e8435a9300077b534fa", - "Node name for S&R": "VHS_SplitImages" - }, - "widgets_values": { - "split_index": 3 - } - }, - { - "id": 63, - "type": "WanVideoAddAudioEmbeds", - "pos": [ - 2055.220703125, - -847.4767456054688 - ], - "size": [ - 254.5941619873047, - 122 - ], - "flags": {}, - "order": 27, - "mode": 0, - "inputs": [ - { - "name": "embeds", - "type": "WANVIDIMAGE_EMBEDS", - "link": 66 - }, - { - "name": "audio_encoder_output", - "type": "AUDIO_ENCODER_OUTPUT", - "link": 68 - }, - { - "name": "ref_latent", - "shape": 7, - "type": "LATENT", - "link": 128 - }, - { - "name": "frames", - "type": "INT", - "widget": { - "name": "frames" - }, - "link": 96 - } - ], - "outputs": [ - { - "name": "image_embeds", - "type": "WANVIDIMAGE_EMBEDS", - "links": [ - 75, - 91 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WanVideoWrapper", - "ver": "a1ca0985ec120ff97e34676a64de19c99767bbd4", - "Node name for S&R": "WanVideoAddAudioEmbeds" - }, - "widgets_values": [ - 601, - 1 - ] - }, - { - "id": 69, - "type": "PreviewAny", - "pos": [ - 2441.904052734375, - -1017.1683349609375 - ], - "size": [ - 490.69390869140625, - 355.7907409667969 - ], - "flags": {}, - "order": 28, - "mode": 0, - "inputs": [ - { - "name": "source", - "type": "*", - "link": 75 - } - ], - "outputs": [], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.52", - "Node name for S&R": "PreviewAny" - }, - "widgets_values": [] - }, - { - "id": 37, - "type": "WanVideoEmptyEmbeds", - "pos": [ - 1686.0142822265625, - -1071.16162109375 - ], - "size": [ - 315, - 126 - ], - "flags": {}, - "order": 21, - "mode": 0, - "inputs": [ - { - "name": "control_embeds", - "shape": 7, - "type": "WANVIDIMAGE_EMBEDS", - "link": null - }, - { - "name": "extra_latents", - "shape": 7, - "type": "LATENT", - "link": null - }, - { - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": 85 - }, - { - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": 86 - }, - { - "name": "num_frames", - "type": "INT", - "widget": { - "name": "num_frames" - }, - "link": 79 - } - ], - "outputs": [ - { - "name": "image_embeds", - "type": "WANVIDIMAGE_EMBEDS", - "links": [ - 66 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WanVideoWrapper", - "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", - "Node name for S&R": "WanVideoEmptyEmbeds" - }, - "widgets_values": [ - 832, - 480, - 601 - ] - }, - { - "id": 71, - "type": "PrimitiveNode", - "pos": [ - 1682.593505859375, - -863.1051635742188 - ], - "size": [ - 210, - 82 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "INT", - "type": "INT", - "widget": { - "name": "num_frames" - }, - "links": [ - 79, - 96 - ] - } - ], - "title": "num_frames", - "properties": { - "Run widget replace on values": false - }, - "widgets_values": [ - 601, - "fixed" - ] - }, - { - "id": 64, - "type": "AudioEncoderEncode", - "pos": [ - 1198.93212890625, - 778.826904296875 - ], - "size": [ - 285.087890625, - 46 - ], - "flags": {}, - "order": 26, - "mode": 0, - "inputs": [ - { - "name": "audio_encoder", - "type": "AUDIO_ENCODER", - "link": 69 - }, - { - "name": "audio", - "type": "AUDIO", - "link": 150 - } - ], - "outputs": [ - { - "name": "AUDIO_ENCODER_OUTPUT", - "type": "AUDIO_ENCODER_OUTPUT", - "links": [ - 68 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.52", - "Node name for S&R": "AudioEncoderEncode" - }, - "widgets_values": [] - }, - { - "id": 65, - "type": "AudioEncoderLoader", - "pos": [ - 1136.3577880859375, - 641.1036376953125 - ], - "size": [ - 346.756103515625, - 58 - ], - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "AUDIO_ENCODER", - "type": "AUDIO_ENCODER", - "links": [ - 69 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.52", - "Node name for S&R": "AudioEncoderLoader" - }, - "widgets_values": [ - "wav2vec_xlsr_53_english_fp32.safetensors" - ] } ], "links": [ @@ -1893,22 +1961,6 @@ 0, "*" ], - [ - 66, - 37, - 0, - 63, - 0, - "WANVIDIMAGE_EMBEDS" - ], - [ - 68, - 64, - 0, - 63, - 1, - "AUDIO_ENCODER_OUTPUT" - ], [ 69, 65, @@ -1925,14 +1977,6 @@ 2, "WANVIDEOTEXTEMBEDS" ], - [ - 75, - 63, - 0, - 69, - 0, - "*" - ], [ 77, 28, @@ -1989,14 +2033,6 @@ 3, "INT" ], - [ - 91, - 63, - 0, - 27, - 1, - "WANVIDIMAGE_EMBEDS" - ], [ 92, 72, @@ -2013,14 +2049,6 @@ 1, "LATENT" ], - [ - 96, - 71, - 0, - 63, - 3, - "INT" - ], [ 100, 70, @@ -2053,14 +2081,6 @@ 5, "WANVIDCONTEXT" ], - [ - 128, - 72, - 0, - 63, - 2, - "LATENT" - ], [ 129, 35, @@ -2085,14 +2105,6 @@ 1, "IMAGE" ], - [ - 146, - 96, - 0, - 30, - 0, - "IMAGE" - ], [ 148, 94, @@ -2140,16 +2152,80 @@ 97, 0, "IMAGE" + ], + [ + 156, + 64, + 0, + 101, + 1, + "AUDIO_ENCODER_OUTPUT" + ], + [ + 157, + 72, + 0, + 101, + 2, + "LATENT" + ], + [ + 158, + 37, + 0, + 101, + 0, + "WANVIDIMAGE_EMBEDS" + ], + [ + 159, + 101, + 1, + 69, + 0, + "*" + ], + [ + 161, + 71, + 0, + 101, + 5, + "INT" + ], + [ + 162, + 101, + 0, + 27, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 163, + 96, + 0, + 102, + 0, + "IMAGE" + ], + [ + 164, + 102, + 0, + 30, + 0, + "IMAGE" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.505447028499326, + "scale": 0.5116450399201969, "offset": [ - 740.042928663884, - 1057.007052457307 + 572.0674176111402, + 1097.812069895393 ] }, "frontendVersion": "1.26.6", diff --git a/s2v/wanvideo2_2_S2V_framepack_pose_testing.json b/s2v/wanvideo2_2_S2V_framepack_pose_testing.json new file mode 100644 index 0000000..4008de1 --- /dev/null +++ b/s2v/wanvideo2_2_S2V_framepack_pose_testing.json @@ -0,0 +1,3448 @@ +{ + "id": "c6e410bc-5e2c-460b-ae81-c91b6094fbb1", + "revision": 0, + "last_node_id": 146, + "last_link_id": 244, + "nodes": [ + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 1994.2247314453125, + -394.9518737792969 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 197 + }, + { + "name": "samples", + "type": "LATENT", + "link": 105 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 35, + "type": "WanVideoTorchCompileSettings", + "pos": [ + -1117.06494140625, + -1156.921875 + ], + "size": [ + 390.5999755859375, + 202 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "slot_index": 0, + "links": [ + 129 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ] + }, + { + "id": 44, + "type": "Note", + "pos": [ + -1106.779052734375, + -1309.5294189453125 + ], + "size": [ + 303.0501403808594, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "If you have Triton installed, connect this for ~20-30% speed increase and reduced peak VRAM use" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 36, + "type": "Note", + "pos": [ + -559.8602294921875, + -1394.335205078125 + ], + "size": [ + 374.3061828613281, + 171.9547576904297 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "fp_16_fast enables \"Full FP16 Accmumulation in FP16 GEMMs\" feature available in the very latest pytorch nightly, this is around 20% speed boost. \n\nSageattn if you have it installed can be used for almost double inference speed at higher resolutions\n\nRadial attention is even faster but has worst quality, it should be used along with Set Radial Attention node to control which steps/blocks it's applied on to balance quality and speed." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 22, + "type": "WanVideoModelLoader", + "pos": [ + -593.1958618164062, + -1146.4970703125 + ], + "size": [ + 477.4410095214844, + 314 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": 129 + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": null + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "extra_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": null + }, + { + "name": "fantasyportrait_model", + "shape": 7, + "type": "FANTASYPORTRAITMODEL", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "slot_index": 0, + "links": [ + 61 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoModelLoader" + }, + "widgets_values": [ + "WanVideo\\S2V\\Wan2_2-S2V-14B_fp8_e4m3fn_scaled_KJ.safetensors", + "fp16_fast", + "fp8_e4m3fn_scaled", + "offload_device", + "sageattn" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 61, + "type": "MarkdownNote", + "pos": [ + -83.98224639892578, + -1481.6065673828125 + ], + "size": [ + 688.705078125, + 156.6822052001953 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Models:\n\n[https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled](https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled)\n\nIf you want to use torch compile on GPUs prior to 4000 series:\n\nLoRA:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors](https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 60, + "type": "WanVideoLoraSelectMulti", + "pos": [ + 17.099323272705078, + -1224.06787109375 + ], + "size": [ + 617.9208374023438, + 342 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 64 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoLoraSelectMulti" + }, + "widgets_values": [ + "WanVideo\\Lightx2v\\lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16_.safetensors", + 1.2, + "none", + 1, + "none", + 1, + "none", + 1, + "none", + 1, + false, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 58, + "type": "WanVideoSetLoRAs", + "pos": [ + 36.42268371582031, + -805.3877563476562 + ], + "size": [ + 174.53378295898438, + 46 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 61 + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 64 + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 62 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoSetLoRAs" + }, + "widgets_values": [], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 39, + "type": "WanVideoBlockSwap", + "pos": [ + -434.31292724609375, + -777.4451293945312 + ], + "size": [ + 315, + 202 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "slot_index": 0, + "links": [ + 58 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 32, + false, + false, + true, + 0, + 1, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 42, + "type": "Note", + "pos": [ + -765.3607788085938, + -775.6983032226562 + ], + "size": [ + 312.98052978515625, + 92.32489013671875 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Adjust the blocks to swap based on your VRAM, this is a tradeoff between speed and memory usage." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 56, + "type": "WanVideoSetBlockSwap", + "pos": [ + 250.55487060546875, + -803.0498657226562 + ], + "size": [ + 201.76815795898438, + 46 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 62 + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 58 + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 60 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoSetBlockSwap" + }, + "widgets_values": [], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 38, + "type": "WanVideoVAELoader", + "pos": [ + 20.023881912231445, + -646.4891357421875 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 196 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoVAELoader" + }, + "widgets_values": [ + "wanvideo\\Wan2_1_VAE_bf16.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 119, + "type": "SetNode", + "pos": [ + 391.3757019042969, + -615.3194580078125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "link": 196 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_VAE", + "properties": { + "previousName": "VAE" + }, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 120, + "type": "GetNode", + "pos": [ + 2002.47412109375, + -472.52252197265625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 197 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 109, + "type": "WanVideoEncode", + "pos": [ + 226.22377014160156, + 1413.26171875 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 199 + }, + { + "name": "image", + "type": "IMAGE", + "link": 174 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 192 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "63d4b6aadaae543f96d101788122563f3e2ba0c8", + "Node name for S&R": "WanVideoEncode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + 0, + 0.5 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 123, + "type": "Note", + "pos": [ + 2024.4696044921875, + -677.442626953125 + ], + "size": [ + 239.09585571289062, + 94.3968734741211 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "The sampling windows are constant size, rounded up and padded with empty if there isn't enough audio" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 37, + "type": "WanVideoEmptyEmbeds", + "pos": [ + 403.56707763671875, + -283.66357421875 + ], + "size": [ + 315, + 126 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "control_embeds", + "shape": 7, + "type": "WANVIDIMAGE_EMBEDS", + "link": null + }, + { + "name": "extra_latents", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 85 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 86 + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 79 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 190 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoEmptyEmbeds" + }, + "widgets_values": [ + 832, + 480, + 501 + ] + }, + { + "id": 72, + "type": "WanVideoEncode", + "pos": [ + 432.53973388671875, + -18.44605827331543 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 198 + }, + { + "name": "image", + "type": "IMAGE", + "link": 202 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 191 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "63d4b6aadaae543f96d101788122563f3e2ba0c8", + "Node name for S&R": "WanVideoEncode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + 0, + 1 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 121, + "type": "GetNode", + "pos": [ + 422.7891845703125, + -93.28303527832031 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 198, + 200 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 94, + "type": "VHS_LoadAudio", + "pos": [ + -1128.97119140625, + 360.84234619140625 + ], + "size": [ + 415.6340026855469, + 126 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [] + }, + { + "name": "duration", + "type": "FLOAT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "8e4d79471bf1952154768e8435a9300077b534fa", + "Node name for S&R": "VHS_LoadAudio" + }, + "widgets_values": { + "audio_file": "input/weightoftheworld2.mp4", + "seek_seconds": 0, + "duration": 0 + } + }, + { + "id": 66, + "type": "LoadAudio", + "pos": [ + -1127.343505859375, + 149.00762939453125 + ], + "size": [ + 361.2844543457031, + 155.81912231445312 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "0321. Alphaville - Big In Japan.mp3", + null, + null + ] + }, + { + "id": 82, + "type": "MelBandRoFormerSampler", + "pos": [ + -442.4711608886719, + 323.1734619140625 + ], + "size": [ + 222.73397827148438, + 46 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MELROFORMERMODEL", + "link": 106 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 225 + } + ], + "outputs": [ + { + "name": "vocals", + "type": "AUDIO", + "links": [ + 149 + ] + }, + { + "name": "instruments", + "type": "AUDIO", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-MelBandRoFormer", + "ver": "b40e263224778ec417114d91d8b3b39934e30de5", + "Node name for S&R": "MelBandRoFormerSampler" + }, + "widgets_values": [] + }, + { + "id": 98, + "type": "NormalizeAudioLoudness", + "pos": [ + -434.1120300292969, + 442.410400390625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 149 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 150 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "90c3bbb6c2e4ff5e05305e765d007d5e58428ce4", + "Node name for S&R": "NormalizeAudioLoudness" + }, + "widgets_values": [ + -23 + ] + }, + { + "id": 64, + "type": "AudioEncoderEncode", + "pos": [ + -73.06085205078125, + 415.04669189453125 + ], + "size": [ + 285.087890625, + 46 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "name": "audio_encoder", + "type": "AUDIO_ENCODER", + "link": 69 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 150 + } + ], + "outputs": [ + { + "name": "AUDIO_ENCODER_OUTPUT", + "type": "AUDIO_ENCODER_OUTPUT", + "links": [ + 189 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "AudioEncoderEncode" + }, + "widgets_values": [] + }, + { + "id": 65, + "type": "AudioEncoderLoader", + "pos": [ + -97.48931884765625, + 258.31158447265625 + ], + "size": [ + 346.756103515625, + 58 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO_ENCODER", + "type": "AUDIO_ENCODER", + "links": [ + 69 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "AudioEncoderLoader" + }, + "widgets_values": [ + "wav2vec_xlsr_53_english_fp32.safetensors" + ] + }, + { + "id": 124, + "type": "MarkdownNote", + "pos": [ + -97.37991333007812, + 112.38502502441406 + ], + "size": [ + 399.6143798828125, + 90.30195617675781 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "[https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/blob/main/split_files/audio_encoders/wav2vec2_large_english_fp16.safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/blob/main/split_files/audio_encoders/wav2vec2_large_english_fp16.safetensors)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 81, + "type": "MelBandRoFormerModelLoader", + "pos": [ + -483.87762451171875, + 185.78466796875 + ], + "size": [ + 316.2164001464844, + 58 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "MELROFORMERMODEL", + "links": [ + 106 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-MelBandRoFormer", + "ver": "b40e263224778ec417114d91d8b3b39934e30de5", + "Node name for S&R": "MelBandRoFormerModelLoader" + }, + "widgets_values": [ + "MelBandRoFormer\\MelBandRoformer_fp16.safetensors" + ] + }, + { + "id": 74, + "type": "ImageResizeKJv2", + "pos": [ + -209.49288940429688, + -372.07373046875 + ], + "size": [ + 270, + 336 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 83 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 219 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 220 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 201 + ] + }, + { + "name": "width", + "type": "INT", + "links": [ + 85 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 86 + ] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 640, + 640, + "lanczos", + "crop", + "0, 0, 0", + "center", + 16, + "cpu", + "Output: 1 x 960 x 640 | 7.03MB" + ] + }, + { + "id": 125, + "type": "SetNode", + "pos": [ + 128.457763671875, + -356.9394836425781 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "link": 201 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 202 + ] + } + ], + "title": "Set_reference_image", + "properties": { + "previousName": "reference_image" + }, + "widgets_values": [ + "reference_image" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 112, + "type": "ImageConcatMulti", + "pos": [ + 2722.126220703125, + 26.577224731445312 + ], + "size": [ + 270, + 150 + ], + "flags": {}, + "order": 57, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 243 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 244 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 213 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0" + }, + "widgets_values": [ + 2, + "right", + false, + null + ] + }, + { + "id": 97, + "type": "VHS_VideoCombine", + "pos": [ + 3336.458740234375, + -465.7515869140625 + ], + "size": [ + 940.8292846679688, + 334 + ], + "flags": {}, + "order": 59, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 208 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 212 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "330bce6c3c0d47ebdedcc0348d9ab355707b7523", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "WanVideo2_2_S2V", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_2_S2V_00014-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 16, + "workflow": "WanVideo2_2_S2V_00014.png", + "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_2_S2V_00014-audio.mp4" + } + } + } + }, + { + "id": 111, + "type": "ImageResizeKJv2", + "pos": [ + -158.63368225097656, + 1323.3763427734375 + ], + "size": [ + 270, + 336.00006103515625 + ], + "flags": {}, + "order": 47, + "mode": 4, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 173 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 174, + 214 + ] + }, + { + "name": "width", + "type": "INT", + "links": null + }, + { + "name": "height", + "type": "INT", + "links": null + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 640, + 640, + "bilinear", + "stretch", + "0, 0, 0", + "center", + 16, + "gpu" + ] + }, + { + "id": 130, + "type": "Reroute", + "pos": [ + 1995.2110595703125, + 1312.697265625 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 214 + } + ], + "outputs": [ + { + "name": "", + "type": "IMAGE", + "links": [ + 243 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 122, + "type": "GetNode", + "pos": [ + 230.59597778320312, + 1699.09423828125 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 199 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 133, + "type": "SetNode", + "pos": [ + -1378.2918701171875, + -320.5899658203125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 216 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_width", + "properties": { + "previousName": "width" + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 132, + "type": "INTConstant", + "pos": [ + -1611.5635986328125, + -232.2957763671875 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 217 + ] + } + ], + "title": "Height", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 134, + "type": "SetNode", + "pos": [ + -1378.2921142578125, + -205.04428100585938 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 217 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_height", + "properties": { + "previousName": "height" + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 73, + "type": "LoadImage", + "pos": [ + -794.7612915039062, + -369.87554931640625 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 83 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2b.jpg", + "image" + ] + }, + { + "id": 137, + "type": "GetNode", + "pos": [ + -369.8827819824219, + -271.25408935546875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 19, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 219 + ] + } + ], + "title": "Get_width", + "properties": {}, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 138, + "type": "GetNode", + "pos": [ + -372.0632019042969, + -212.39117431640625 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 220 + ] + } + ], + "title": "Get_height", + "properties": {}, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 107, + "type": "DWPreprocessor", + "pos": [ + -456.9013977050781, + 1322.96728515625 + ], + "size": [ + 270, + 198 + ], + "flags": {}, + "order": 45, + "mode": 4, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 169 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 173 + ] + }, + { + "name": "POSE_KEYPOINT", + "type": "POSE_KEYPOINT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui_controlnet_aux", + "ver": "1d7cdce8cb771fbc39a432a6338168c12a338ef4", + "Node name for S&R": "DWPreprocessor" + }, + "widgets_values": [ + "disable", + "disable", + "enable", + 640, + "yolox_l.torchscript.pt", + "dw-ll_ucoco_384_bs5.torchscript.pt" + ] + }, + { + "id": 141, + "type": "GetNode", + "pos": [ + -1261.4315185546875, + 737.1270751953125 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 223 + ] + } + ], + "title": "Get_width", + "properties": {}, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 142, + "type": "GetNode", + "pos": [ + -1263.6119384765625, + 795.989990234375 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 22, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 224 + ] + } + ], + "title": "Get_height", + "properties": {}, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 117, + "type": "WanVideoAddS2VEmbeds", + "pos": [ + 842.3013305664062, + -125.87545776367188 + ], + "size": [ + 327.17578125, + 234 + ], + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "name": "embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 190 + }, + { + "name": "audio_encoder_output", + "shape": 7, + "type": "AUDIO_ENCODER_OUTPUT", + "link": 189 + }, + { + "name": "ref_latent", + "shape": 7, + "type": "LATENT", + "link": 191 + }, + { + "name": "pose_latent", + "shape": 7, + "type": "LATENT", + "link": 192 + }, + { + "name": "vae", + "shape": 7, + "type": "WANVAE", + "link": 200 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 194 + ] + }, + { + "name": "audio_frame_count", + "type": "INT", + "links": [ + 195 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "a5621b87391013155b4f688fbe01dba10a8104aa", + "Node name for S&R": "WanVideoAddS2VEmbeds" + }, + "widgets_values": [ + 80, + 1, + 0, + 1, + true + ] + }, + { + "id": 118, + "type": "PreviewAny", + "pos": [ + 1220.6553955078125, + -59.443397521972656 + ], + "size": [ + 226.4835662841797, + 88 + ], + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 195 + } + ], + "outputs": [], + "title": "Actual total frame count", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.52", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [] + }, + { + "id": 139, + "type": "GetNode", + "pos": [ + -1181.0894775390625, + 1225.9921875 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 221, + 229 + ] + } + ], + "title": "Get_width", + "properties": {}, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 140, + "type": "GetNode", + "pos": [ + -1180.52197265625, + 1278.0623779296875 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 24, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 222, + 230 + ] + } + ], + "title": "Get_height", + "properties": {}, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 116, + "type": "VHS_LoadVideo", + "pos": [ + -1224.3709716796875, + 1321.8804931640625 + ], + "size": [ + 247.455078125, + 452.2566223144531 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "custom_width", + "type": "INT", + "widget": { + "name": "custom_width" + }, + "link": 229 + }, + { + "name": "custom_height", + "type": "INT", + "widget": { + "name": "custom_height" + }, + "link": 230 + }, + { + "name": "frame_load_cap", + "type": "INT", + "widget": { + "name": "frame_load_cap" + }, + "link": 228 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 188 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "8e4d79471bf1952154768e8435a9300077b534fa", + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "weight-world-bones_00003-audio.mp4", + "force_rate": 16, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 501, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "Wan", + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "weight-world-bones_00003-audio.mp4", + "type": "input", + "format": "video/mp4", + "force_rate": 16, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 501, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 106, + "type": "VHS_LoadVideo", + "pos": [ + -1112.86865234375, + 593.5896606445312 + ], + "size": [ + 247.455078125, + 451.9747314453125 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "custom_width", + "type": "INT", + "widget": { + "name": "custom_width" + }, + "link": 223 + }, + { + "name": "custom_height", + "type": "INT", + "widget": { + "name": "custom_height" + }, + "link": 224 + }, + { + "name": "frame_load_cap", + "type": "INT", + "widget": { + "name": "frame_load_cap" + }, + "link": 227 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 225, + 226 + ] + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "8e4d79471bf1952154768e8435a9300077b534fa", + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "weightoftheworld2.mp4", + "force_rate": 16, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 501, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "Wan", + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "weightoftheworld2.mp4", + "type": "input", + "format": "video/mp4", + "force_rate": 16, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 501, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 110, + "type": "ImageResizeKJv2", + "pos": [ + -759.1649169921875, + 1320.775146484375 + ], + "size": [ + 270, + 336.00006103515625 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 188 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 221 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 222 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 169 + ] + }, + { + "name": "width", + "type": "INT", + "links": null + }, + { + "name": "height", + "type": "INT", + "links": null + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 640, + 640, + "bilinear", + "crop", + "0, 0, 0", + "center", + 16, + "cpu" + ] + }, + { + "id": 131, + "type": "INTConstant", + "pos": [ + -1611.564208984375, + -356.56207275390625 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 216 + ] + } + ], + "title": "Width", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 67, + "type": "WanVideoTextEncodeCached", + "pos": [ + 1491.1046142578125, + -938.5902709960938 + ], + "size": [ + 459.45745849609375, + 393.8887939453125 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "extender_args", + "shape": 7, + "type": "WANVIDEOPROMPTEXTENDER_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 71 + ] + }, + { + "name": "negative_text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + }, + { + "name": "positive_prompt", + "type": "STRING", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "a1ca0985ec120ff97e34676a64de19c99767bbd4", + "Node name for S&R": "WanVideoTextEncodeCached" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "3D animated scene of a young woman singing melancholically", + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + "disabled", + true, + "gpu" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 70, + "type": "GetImageSizeAndCount", + "pos": [ + 2367.16748046875, + -395.861572265625 + ], + "size": [ + 190.86483764648438, + 86 + ], + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 77 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 237 + ] + }, + { + "label": "640 width", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "640 height", + "name": "height", + "type": "INT", + "links": null + }, + { + "label": "557 count", + "name": "count", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "GetImageSizeAndCount" + }, + "widgets_values": [] + }, + { + "id": 127, + "type": "LazySwitchKJ", + "pos": [ + 3008.5068359375, + -267.5511474609375 + ], + "size": [ + 270, + 78 + ], + "flags": {}, + "order": 58, + "mode": 0, + "inputs": [ + { + "name": "on_false", + "type": "*", + "link": 240 + }, + { + "name": "on_true", + "type": "*", + "link": 213 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": [ + 208 + ] + } + ], + "title": "Switch: Add pose view", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "LazySwitchKJ" + }, + "widgets_values": [ + true + ] + }, + { + "id": 143, + "type": "GetImageRangeFromBatch", + "pos": [ + 2622.79833984375, + -397.2756652832031 + ], + "size": [ + 340.3267517089844, + 102 + ], + "flags": { + "collapsed": false + }, + "order": 55, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 237 + }, + { + "name": "masks", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 242 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 239 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "GetImageRangeFromBatch" + }, + "widgets_values": [ + 0, + 501 + ] + }, + { + "id": 129, + "type": "Reroute", + "pos": [ + 2847.140380859375, + 632.76953125 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 226 + } + ], + "outputs": [ + { + "name": "", + "type": "AUDIO", + "links": [ + 212 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 71, + "type": "PrimitiveNode", + "pos": [ + -1613.3883056640625, + -71.24759674072266 + ], + "size": [ + 210, + 82 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "links": [ + 79, + 227, + 228, + 242 + ] + } + ], + "title": "num_frames", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 501, + "fixed" + ] + }, + { + "id": 27, + "type": "WanVideoSampler", + "pos": [ + 1616.490966796875, + -391.5707092285156 + ], + "size": [ + 315, + 999 + ], + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 60 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 194 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 71 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": null + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": null + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 105 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5406a72f62adf4a31a8a0a0e4923cc5288399652", + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 4, + 1, + 4, + 45, + "fixed", + true, + "dpm++_sde", + 0, + 1, + false, + "comfy", + 0, + -1, + false + ] + }, + { + "id": 105, + "type": "ColorMatch", + "pos": [ + 2680.017333984375, + -202.95472717285156 + ], + "size": [ + 270, + 126 + ], + "flags": {}, + "order": 56, + "mode": 0, + "inputs": [ + { + "name": "image_ref", + "type": "IMAGE", + "link": 203 + }, + { + "name": "image_target", + "type": "IMAGE", + "link": 239 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 240, + 244 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ba9153cb06fc77bfd86c36835f1817482e8328a0", + "Node name for S&R": "ColorMatch" + }, + "widgets_values": [ + "mkl", + 1, + true + ] + }, + { + "id": 126, + "type": "GetNode", + "pos": [ + 2435.40625, + -167.38819885253906 + ], + "size": [ + 210.99107360839844, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 28, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 203 + ] + } + ], + "title": "Get_reference_image", + "properties": {}, + "widgets_values": [ + "reference_image" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + } + ], + "links": [ + [ + 58, + 39, + 0, + 56, + 1, + "BLOCKSWAPARGS" + ], + [ + 60, + 56, + 0, + 27, + 0, + "WANVIDEOMODEL" + ], + [ + 61, + 22, + 0, + 58, + 0, + "WANVIDEOMODEL" + ], + [ + 62, + 58, + 0, + 56, + 0, + "WANVIDEOMODEL" + ], + [ + 64, + 60, + 0, + 58, + 1, + "WANVIDLORA" + ], + [ + 69, + 65, + 0, + 64, + 0, + "AUDIO_ENCODER" + ], + [ + 71, + 67, + 0, + 27, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 77, + 28, + 0, + 70, + 0, + "IMAGE" + ], + [ + 79, + 71, + 0, + 37, + 4, + "INT" + ], + [ + 83, + 73, + 0, + 74, + 0, + "IMAGE" + ], + [ + 85, + 74, + 1, + 37, + 2, + "INT" + ], + [ + 86, + 74, + 2, + 37, + 3, + "INT" + ], + [ + 105, + 27, + 0, + 28, + 1, + "LATENT" + ], + [ + 106, + 81, + 0, + 82, + 0, + "MELROFORMERMODEL" + ], + [ + 129, + 35, + 0, + 22, + 0, + "WANCOMPILEARGS" + ], + [ + 149, + 82, + 0, + 98, + 0, + "AUDIO" + ], + [ + 150, + 98, + 0, + 64, + 1, + "AUDIO" + ], + [ + 169, + 110, + 0, + 107, + 0, + "IMAGE" + ], + [ + 173, + 107, + 0, + 111, + 0, + "IMAGE" + ], + [ + 174, + 111, + 0, + 109, + 1, + "IMAGE" + ], + [ + 188, + 116, + 0, + 110, + 0, + "IMAGE" + ], + [ + 189, + 64, + 0, + 117, + 1, + "AUDIO_ENCODER_OUTPUT" + ], + [ + 190, + 37, + 0, + 117, + 0, + "WANVIDIMAGE_EMBEDS" + ], + [ + 191, + 72, + 0, + 117, + 2, + "LATENT" + ], + [ + 192, + 109, + 0, + 117, + 3, + "LATENT" + ], + [ + 194, + 117, + 0, + 27, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 195, + 117, + 1, + 118, + 0, + "*" + ], + [ + 196, + 38, + 0, + 119, + 0, + "*" + ], + [ + 197, + 120, + 0, + 28, + 0, + "WANVAE" + ], + [ + 198, + 121, + 0, + 72, + 0, + "WANVAE" + ], + [ + 199, + 122, + 0, + 109, + 0, + "WANVAE" + ], + [ + 200, + 121, + 0, + 117, + 4, + "WANVAE" + ], + [ + 201, + 74, + 0, + 125, + 0, + "*" + ], + [ + 202, + 125, + 0, + 72, + 1, + "IMAGE" + ], + [ + 203, + 126, + 0, + 105, + 0, + "IMAGE" + ], + [ + 208, + 127, + 0, + 97, + 0, + "IMAGE" + ], + [ + 212, + 129, + 0, + 97, + 1, + "AUDIO" + ], + [ + 213, + 112, + 0, + 127, + 1, + "*" + ], + [ + 214, + 111, + 0, + 130, + 0, + "*" + ], + [ + 216, + 131, + 0, + 133, + 0, + "*" + ], + [ + 217, + 132, + 0, + 134, + 0, + "*" + ], + [ + 219, + 137, + 0, + 74, + 2, + "INT" + ], + [ + 220, + 138, + 0, + 74, + 3, + "INT" + ], + [ + 221, + 139, + 0, + 110, + 2, + "INT" + ], + [ + 222, + 140, + 0, + 110, + 3, + "INT" + ], + [ + 223, + 141, + 0, + 106, + 2, + "INT" + ], + [ + 224, + 142, + 0, + 106, + 3, + "INT" + ], + [ + 225, + 106, + 2, + 82, + 1, + "AUDIO" + ], + [ + 226, + 106, + 2, + 129, + 0, + "*" + ], + [ + 227, + 71, + 0, + 106, + 4, + "INT" + ], + [ + 228, + 71, + 0, + 116, + 4, + "INT" + ], + [ + 229, + 139, + 0, + 116, + 2, + "INT" + ], + [ + 230, + 140, + 0, + 116, + 3, + "INT" + ], + [ + 237, + 70, + 0, + 143, + 0, + "IMAGE" + ], + [ + 239, + 143, + 0, + 105, + 1, + "IMAGE" + ], + [ + 240, + 105, + 0, + 127, + 0, + "*" + ], + [ + 242, + 71, + 0, + 143, + 2, + "INT" + ], + [ + 243, + 130, + 0, + 112, + 0, + "IMAGE" + ], + [ + 244, + 105, + 0, + 112, + 1, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + -1188.1114501953125, + -1609.4483642578125, + 1896.0758056640625, + 1103.56005859375 + ], + "color": "#88A", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Pose input (optional)", + "bounding": [ + -1292.3486328125, + 1142.282958984375, + 1939.6778564453125, + 689.3397827148438 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Audio input", + "bounding": [ + -1295.53857421875, + 13.78874397277832, + 1630.400146484375, + 1090.776611328125 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Separate vocals from music", + "bounding": [ + -617.63525390625, + 72.55604553222656, + 478.7571105957031, + 458.2166748046875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.5559917313492586, + "offset": [ + 513.4927365364142, + 697.7427692064874 + ] + }, + "frontendVersion": "1.26.6", + "node_versions": { + "ComfyUI-WanVideoWrapper": "5a2383621a05825d0d0437781afcb8552d9590fd", + "comfy-core": "0.3.26", + "ComfyUI-VideoHelperSuite": "0a75c7958fe320efcb052f1d9f8451fd20c730a8" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index ce9cc2f..22f39df 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -40,11 +40,11 @@ class FramePackMotioner(nn.Module): num_heads=16, # Used to indicate the number of heads in the backbone network; unrelated to this module's design zip_frame_buckets=[1, 2, 16], # Three numbers representing the number of frames sampled for patch operations from the nearest to the farthest frames drop_mode="drop", # If not "drop", it will use "padd", meaning padding instead of deletion - dtype=None, device=None): + ): super().__init__() - self.proj = nn.Conv3d(16, inner_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2), dtype=dtype, device=device) - self.proj_2x = nn.Conv3d(16, inner_dim, kernel_size=(2, 4, 4), stride=(2, 4, 4), dtype=dtype, device=device) - self.proj_4x = nn.Conv3d(16, inner_dim, kernel_size=(4, 8, 8), stride=(4, 8, 8), dtype=dtype, device=device) + self.proj = nn.Conv3d(16, inner_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2)) + self.proj_2x = nn.Conv3d(16, inner_dim, kernel_size=(2, 4, 4), stride=(2, 4, 4)) + self.proj_4x = nn.Conv3d(16, inner_dim, kernel_size=(4, 8, 8), stride=(4, 8, 8)) self.zip_frame_buckets = zip_frame_buckets self.inner_dim = inner_dim @@ -79,9 +79,9 @@ class FramePackMotioner(nn.Module): motion_lat = torch.cat([clean_latents_post, clean_latents_2x, clean_latents_4x], dim=1) - rope_post = rope_embedder.rope_encode(1, lat_height, lat_width, t_start=-1, device=motion_latents.device, dtype=motion_latents.dtype) - rope_2x = rope_embedder.rope_encode(1, lat_height, lat_width, t_start=-3, steps_h=l_2x_shape[-2], steps_w=l_2x_shape[-1], device=motion_latents.device, dtype=motion_latents.dtype) - rope_4x = rope_embedder.rope_encode(4, lat_height, lat_width, t_start=-19, steps_h=l_4x_shape[-2], steps_w=l_4x_shape[-1], device=motion_latents.device, dtype=motion_latents.dtype) + rope_post = rope_embedder.rope_encode_comfy(1, lat_height, lat_width, t_start=-1, device=motion_latents.device, dtype=motion_latents.dtype) + rope_2x = rope_embedder.rope_encode_comfy(1, lat_height, lat_width, t_start=-3, steps_h=l_2x_shape[-2], steps_w=l_2x_shape[-1], device=motion_latents.device, dtype=motion_latents.dtype) + rope_4x = rope_embedder.rope_encode_comfy(4, lat_height, lat_width, t_start=-19, steps_h=l_4x_shape[-2], steps_w=l_4x_shape[-1], device=motion_latents.device, dtype=motion_latents.dtype) rope = torch.cat([rope_post, rope_2x, rope_4x], dim=1) return motion_lat, rope @@ -1714,15 +1714,13 @@ class WanModel(torch.nn.Module): # WanLayerNorm(motioner_dim), # zero_module(nn.Linear(motioner_dim, self.dim))) - self.enable_framepack = enable_framepack + enable_framepack = True if enable_framepack: self.frame_packer = FramePackMotioner( inner_dim=self.dim, num_heads=self.num_heads, zip_frame_buckets=[1, 2, 16], - drop_mode='padd', - device=self.main_device, - dtype=self.dtype) + drop_mode='padd') @staticmethod def _prepare_blockwise_causal_attn_mask( @@ -2139,7 +2137,8 @@ class WanModel(torch.nn.Module): s2v_ref_latent=None, s2v_audio_scale=1.0, s2v_ref_motion=None, - s2v_pose=None + s2v_pose=None, + s2v_motion_frames=[1, 0], ): r""" @@ -2189,17 +2188,15 @@ class WanModel(torch.nn.Module): if self.model_type == 's2v' and s2v_audio_input is not None: if is_uncond: s2v_audio_input = s2v_audio_input * 0 # to match original code - #motion_frames=[17, 5] - motion_frames=[1, 0] - s2v_audio_input = torch.cat([s2v_audio_input[..., 0:1].repeat(1, 1, 1, motion_frames[0]), s2v_audio_input], dim=-1) + s2v_audio_input = torch.cat([s2v_audio_input[..., 0:1].repeat(1, 1, 1, s2v_motion_frames[0]), s2v_audio_input], dim=-1) audio_emb_res = self.casual_audio_encoder(s2v_audio_input) if self.enable_adain: audio_emb_global, audio_emb = audio_emb_res - self.audio_emb_global = audio_emb_global[:, motion_frames[1]:].clone() + self.audio_emb_global = audio_emb_global[:, s2v_motion_frames[1]:].clone() else: audio_emb = audio_emb_res - merged_audio_emb = audio_emb[:, motion_frames[1]:, :] + merged_audio_emb = audio_emb[:, s2v_motion_frames[1]:, :] # params device = self.patch_embedding.weight.device @@ -2245,16 +2242,14 @@ class WanModel(torch.nn.Module): ] if s2v_pose is not None: - print("s2v_pose.shape:", s2v_pose.shape) - print("x[0].shape:", x[0].shape) x[0] = x[0] + self.cond_encoder(s2v_pose.to(self.cond_encoder.weight.dtype)).to(x[0].dtype) - if self.control_adapter is not None and fun_camera is not None: fun_camera = self.control_adapter(fun_camera) x = [u + v for u, v in zip(x, fun_camera)] grid_sizes = torch.stack([torch.tensor(u.shape[2:], device=device, dtype=torch.long) for u in x]) + original_grid_sizes = grid_sizes.clone() x = [u.flatten(2).transpose(1, 2) for u in x] seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.float32) @@ -2302,7 +2297,7 @@ class WanModel(torch.nn.Module): seq_len += end_ref_latent_seq_len x = [torch.cat([u, end_ref_latent.unsqueeze(0)], dim=1) for end_ref_latent, u in zip(end_ref_latent, x)] - grid_sizes = grid_sizes + x = torch.cat([ torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], dim=1) for u in x @@ -2344,7 +2339,7 @@ class WanModel(torch.nn.Module): s2v_ref_latent.shape[2], s2v_ref_latent.shape[3], s2v_ref_latent.shape[4], - t_start=30, device=x.device, dtype=x.dtype) + t_start=max(30, F + 9), device=x.device, dtype=x.dtype) freqs = torch.cat([freqs, freqs_ref], dim=1) self.cached_freqs = freqs @@ -2746,10 +2741,10 @@ class WanModel(torch.nn.Module): #uni3c controlnet if pdc_controlnet_states is not None and b < len(pdc_controlnet_states): - x[:, :x_len] += pdc_controlnet_states[b].to(x) * pcd_data["controlnet_weight"] + x[:, :self.original_seq_len] += pdc_controlnet_states[b].to(x) * pcd_data["controlnet_weight"] #controlnet if (controlnet is not None) and (b % controlnet["controlnet_stride"] == 0) and (b // controlnet["controlnet_stride"] < len(controlnet["controlnet_states"])): - x[:, :x_len] += controlnet["controlnet_states"][b // controlnet["controlnet_stride"]].to(x) * controlnet["controlnet_weight"] + x[:, :self.original_seq_len] += controlnet["controlnet_states"][b // controlnet["controlnet_stride"]].to(x) * controlnet["controlnet_weight"] if self.enable_teacache and (self.teacache_start_step <= current_step <= self.teacache_end_step) and pred_id is not None: self.teacache_state.update( @@ -2796,9 +2791,11 @@ class WanModel(torch.nn.Module): x = x[:, :self.original_seq_len] grid_sizes = torch.stack([torch.tensor([u[0] - 1, u[1], u[2]]) for u in grid_sizes]).to(grid_sizes.device) - #x = x[:, :self.original_seq_len] + + x = x[:, :self.original_seq_len] + x = self.head(x, e.to(x.device)) - x = self.unpatchify(x, grid_sizes) # type: ignore[arg-type] + x = self.unpatchify(x, original_grid_sizes) # type: ignore[arg-type] x = [u.float() for u in x] return (x, pred_id) if pred_id is not None else (x, None) diff --git a/wanvideo/schedulers/__init__.py b/wanvideo/schedulers/__init__.py index c778fe1..e878a65 100644 --- a/wanvideo/schedulers/__init__.py +++ b/wanvideo/schedulers/__init__.py @@ -133,9 +133,6 @@ def get_scheduler(scheduler, steps, start_step, end_step, shift, device, transfo sample_scheduler.full_sigmas = sample_scheduler.sigmas.clone() sample_scheduler.sigmas = sample_scheduler.sigmas[start_idx:start_idx+len(timesteps)+1] # always one longer - - log.info(f"timesteps: {timesteps}") - if hasattr(sample_scheduler, 'timesteps'): sample_scheduler.timesteps = timesteps