diff --git a/example_workflows/wanvideo_vid2vid_example_01.json b/example_workflows/wanvideo_vid2vid_example_01.json new file mode 100644 index 0000000..1bd5046 --- /dev/null +++ b/example_workflows/wanvideo_vid2vid_example_01.json @@ -0,0 +1,881 @@ +{ + "last_node_id": 45, + "last_link_id": 55, + "nodes": [ + { + "id": 33, + "type": "Note", + "pos": [ + 227.3764190673828, + -205.28524780273438 + ], + "size": [ + 351.70458984375, + 60 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Models:\nhttps://huggingface.co/Kijai/WanVideo_comfy/tree/main" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 36, + "type": "Note", + "pos": [ + 796.0189208984375, + -521.5020751953125 + ], + "size": [ + 298.2554016113281, + 108.62744140625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "sdpa should work too, haven't tested flaash\n\nfp8_fast seems to cause huge quality degradation" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 35, + "type": "WanVideoTorchCompileSettings", + "pos": [ + 124.67726135253906, + -627.7935180664062 + ], + "size": [ + 390.5999755859375, + 178 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true + ] + }, + { + "id": 11, + "type": "LoadWanVideoT5TextEncoder", + "pos": [ + 224.15325927734375, + -34.481563568115234 + ], + "size": [ + 377.1661376953125, + 115.05413055419922 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wan_t5_model", + "type": "WANTEXTENCODER", + "links": [ + 15 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanVideoT5TextEncoder" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "offload_device" + ] + }, + { + "id": 39, + "type": "WanVideoBlockSwap", + "pos": [ + 190.92202758789062, + -334.3497314453125 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 20 + ] + }, + { + "id": 38, + "type": "WanVideoVAELoader", + "pos": [ + 1687.4093017578125, + -582.2750854492188 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "links": [ + 43, + 48 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoVAELoader" + }, + "widgets_values": [ + "wanvideo\\Wan2_1_VAE_fp32.safetensors", + "fp32" + ] + }, + { + "id": 43, + "type": "VHS_LoadVideo", + "pos": [ + 570.141357421875, + 292.6640625 + ], + "size": [ + 247.455078125, + 551.455078125 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "shape": 7, + "link": null + }, + { + "name": "vae", + "type": "VAE", + "shape": 7, + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 51 + ], + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "wolf_interpolated.mp4", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "AnimateDiff", + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "wolf_interpolated.mp4", + "type": "input", + "format": "video/mp4", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 44, + "type": "ImageResizeKJ", + "pos": [ + 871.0767211914062, + 291.94927978515625 + ], + "size": [ + 315, + 266 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 51 + }, + { + "name": "get_image_size", + "type": "IMAGE", + "shape": 7, + "link": null + }, + { + "name": "width_input", + "type": "INT", + "shape": 7, + "widget": { + "name": "width_input" + }, + "link": null + }, + { + "name": "height_input", + "type": "INT", + "shape": 7, + "widget": { + "name": "height_input" + }, + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 52, + 53 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": null + }, + { + "name": "height", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "ImageResizeKJ" + }, + "widgets_values": [ + 512, + 512, + "nearest-exact", + false, + 2, + 0, + 0, + "disabled" + ] + }, + { + "id": 37, + "type": "WanVideoEmptyEmbeds", + "pos": [ + 1305.26708984375, + -571.7843627929688 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 42 + ] + } + ], + "properties": { + "Node name for S&R": "WanVideoEmptyEmbeds" + }, + "widgets_values": [ + 512, + 512, + 53 + ] + }, + { + "id": 42, + "type": "WanVideoEncode", + "pos": [ + 1298.557861328125, + 306.5940246582031 + ], + "size": [ + 315, + 222 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 48 + }, + { + "name": "image", + "type": "IMAGE", + "link": 52 + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 50 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoEncode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + 0, + 1 + ] + }, + { + "id": 16, + "type": "WanVideoTextEncode", + "pos": [ + 675.8850708007812, + -36.032100677490234 + ], + "size": [ + 420.30511474609375, + 261.5306701660156 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "t5", + "type": "WANTEXTENCODER", + "link": 15 + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 30 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoTextEncode" + }, + "widgets_values": [ + "high quality nature video featuring a red panda", + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + true + ] + }, + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 1692.973876953125, + -404.8614501953125 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 43 + }, + { + "name": "samples", + "type": "LATENT", + "link": 33 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 54 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + true, + 272, + 272, + 144, + 128 + ] + }, + { + "id": 45, + "type": "ImageConcatMulti", + "pos": [ + 1817.8916015625, + 102.76653289794922 + ], + "size": [ + 210, + 150 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 53 + }, + { + "name": "image_2", + "type": "IMAGE", + "link": 54 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 55 + ], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + 2, + "right", + false, + "" + ] + }, + { + "id": 30, + "type": "VHS_VideoCombine", + "pos": [ + 2074.630126953125, + -390.24658203125 + ], + "size": [ + 912.0325317382812, + 794.0162353515625 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 55 + }, + { + "name": "audio", + "type": "AUDIO", + "shape": 7, + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "shape": 7, + "link": null + }, + { + "name": "vae", + "type": "VAE", + "shape": 7, + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "WanVideo2_1_T2V", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_1_T2V_00012.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16, + "workflow": "WanVideo2_1_T2V_00012.png", + "fullpath": "N:\\AI\\ComfyUI\\output\\WanVideo2_1_T2V_00012.mp4" + } + } + } + }, + { + "id": 27, + "type": "WanVideoSampler", + "pos": [ + 1315.2401123046875, + -401.48028564453125 + ], + "size": [ + 315, + 619 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 29 + }, + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "link": 30 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 42 + }, + { + "name": "samples", + "type": "LATENT", + "shape": 7, + "link": 50 + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 33 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 15, + 6, + 5, + 1057359483639288, + "fixed", + true, + "dpm++", + 0, + 0.5, + "" + ] + }, + { + "id": 22, + "type": "WanVideoModelLoader", + "pos": [ + 620.3950805664062, + -357.8426818847656 + ], + "size": [ + 477.4410095214844, + 226.43276977539062 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "type": "WANCOMPILEARGS", + "shape": 7, + "link": null + }, + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "shape": 7, + "link": null + }, + { + "name": "lora", + "type": "HYVIDLORA", + "shape": 7, + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 29 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanVideoModelLoader" + }, + "widgets_values": [ + "WanVideo\\Wan2_1-T2V-1_3B_fp8_e4m3fn.safetensors", + "bf16", + "fp8_e4m3fn", + "offload_device", + "sdpa" + ] + } + ], + "links": [ + [ + 15, + 11, + 0, + 16, + 0, + "WANTEXTENCODER" + ], + [ + 29, + 22, + 0, + 27, + 0, + "WANVIDEOMODEL" + ], + [ + 30, + 16, + 0, + 27, + 1, + "WANVIDEOTEXTEMBEDS" + ], + [ + 33, + 27, + 0, + 28, + 1, + "LATENT" + ], + [ + 42, + 37, + 0, + 27, + 2, + "WANVIDIMAGE_EMBEDS" + ], + [ + 43, + 38, + 0, + 28, + 0, + "VAE" + ], + [ + 48, + 38, + 0, + 42, + 0, + "WANVAE" + ], + [ + 50, + 42, + 0, + 27, + 3, + "LATENT" + ], + [ + 51, + 43, + 0, + 44, + 0, + "IMAGE" + ], + [ + 52, + 44, + 0, + 42, + 1, + "IMAGE" + ], + [ + 53, + 44, + 0, + 45, + 0, + "IMAGE" + ], + [ + 54, + 28, + 0, + 45, + 1, + "IMAGE" + ], + [ + 55, + 45, + 0, + 30, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.7400249944258602, + "offset": [ + 258.6149287858856, + 720.1786467276197 + ] + }, + "node_versions": { + "ComfyUI-WanVideoWrapper": "44c5944f7031949440315038e94ca3f46e80adb2", + "ComfyUI-VideoHelperSuite": "2c25b8b53835aaeb63f831b3137c705cf9f85dce", + "ComfyUI-KJNodes": "4b3009e4bf264b4ee6d777946bba3a6e54997bc9" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index 6bcf790..cae9aff 100644 --- a/nodes.py +++ b/nodes.py @@ -697,13 +697,13 @@ class WanVideoSampler: }, - # "optional": { - # #"samples": ("LATENT", {"tooltip": "init Latents to use for video2video process"} ), - # #"image_cond_latents": ("LATENT", {"tooltip": "init Latents to use for image2video process"} ), - # #"denoise_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "optional": { + "samples": ("LATENT", {"tooltip": "init Latents to use for video2video process"} ), + #"image_cond_latents": ("LATENT", {"tooltip": "init Latents to use for image2video process"} ), + "denoise_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), - # #"riflex_freq_index": ("INT", {"default": 0, "min": 0, "max": 1000, "step": 1, "tooltip": "Frequency index for RIFLEX, disabled when 0, default 4. Allows for new frames to be generated after 129 without looping"}), - # } + #"riflex_freq_index": ("INT", {"default": 0, "min": 0, "max": 1000, "step": 1, "tooltip": "Frequency index for RIFLEX, disabled when 0, default 4. Allows for new frames to be generated after 129 without looping"}), + } } RETURN_TYPES = ("LATENT",) @@ -711,7 +711,7 @@ class WanVideoSampler: FUNCTION = "process" CATEGORY = "WanVideoWrapper" - def process(self, model, text_embeds, image_embeds, shift, steps, cfg, seed, scheduler, riflex_freq_index, force_offload=True): + def process(self, model, text_embeds, image_embeds, shift, steps, cfg, seed, scheduler, riflex_freq_index, force_offload=True, samples=None, denoise_strength=1.0): patcher = model model = model.model transformer = model.diffusion_model @@ -762,6 +762,8 @@ class WanVideoSampler: #for name, param in transformer.named_parameters(): # print(name, param.data.device) + + steps = int(steps/denoise_strength) if scheduler == 'unipc': sample_scheduler = FlowUniPCMultistepScheduler( @@ -789,6 +791,9 @@ class WanVideoSampler: else: raise NotImplementedError("Unsupported solver.") + if denoise_strength < 1.0: + steps = int(steps * denoise_strength) + timesteps = timesteps[-(steps + 1):] seed_g = torch.Generator(device=torch.device("cpu")) seed_g.manual_seed(seed) @@ -814,6 +819,10 @@ class WanVideoSampler: device=torch.device("cpu"), generator=seed_g) + if samples is not None: + latent_timestep = timesteps[:1].to(noise) + noise = noise * latent_timestep / 1000 + (1 - latent_timestep / 1000) * samples["samples"].squeeze(0).to(noise) + latent = noise.to(device) d = transformer.dim // transformer.num_heads @@ -824,6 +833,10 @@ class WanVideoSampler: ], dim=1) + if not isinstance(cfg, list): + cfg = [cfg] * (steps +1) + print(cfg) + base_args = { 'clip_fea': image_embeds.get('clip_context', None), 'seq_len': seq_len, @@ -855,12 +868,14 @@ class WanVideoSampler: noise_pred_cond = transformer( latent_model_input, t=timestep, **arg_c)[0].to(offload_device) - - noise_pred_uncond = transformer( - latent_model_input, t=timestep, **arg_null)[0].to(offload_device) - - noise_pred = noise_pred_uncond + cfg * ( - noise_pred_cond - noise_pred_uncond) + if cfg[i] >= 1.0: + noise_pred_uncond = transformer( + latent_model_input, t=timestep, **arg_null)[0].to(offload_device) + + noise_pred = noise_pred_uncond + cfg[i] * ( + noise_pred_cond - noise_pred_uncond) + else: + noise_pred = noise_pred_cond latent = latent.to(offload_device) @@ -875,9 +890,6 @@ class WanVideoSampler: x0 = [latent.to(device)] if callback is not None: - print(t) - print("latent_model_input", latent_model_input[0].shape) - print("noise_pred", noise_pred.shape) callback_latent = (latent_model_input[0].cpu() - noise_pred * t.cpu() / 1000).detach().permute(1,0,2,3) callback(i, callback_latent, None, steps) else: