diff --git a/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json b/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json new file mode 100644 index 0000000..1f215b5 --- /dev/null +++ b/example_workflows/wanvideo_I2V_InfiniteTalk_example_01.json @@ -0,0 +1,2429 @@ +{ + "id": "8b7a9a57-2303-4ef5-9fc2-bf41713bd1fc", + "revision": 0, + "last_node_id": 299, + "last_link_id": 540, + "nodes": [ + { + "id": 120, + "type": "MultiTalkModelLoader", + "pos": [ + 592.73583984375, + -2520.13525390625 + ], + "size": [ + 511.7737121582031, + 66.51400756835938 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "MULTITALKMODEL", + "links": [ + 355 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "MultiTalkModelLoader" + }, + "widgets_values": [ + "WanVideo\\InfiniteTalk\\InfiniteTalk\\Wan2_1-InfiniteTalk_Single_Q8.gguf" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 239, + "type": "MarkdownNote", + "pos": [ + 623.767822265625, + -2378.682861328125 + ], + "size": [ + 452.8328857421875, + 214.95587158203125 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": {}, + "widgets_values": [ + "You can't mix GGUF MultiTalk with non-GGUF main model, but you can mix different GGUF Qtypes with eachother.\n\nGGUF:\n\n[https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main](https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main)\n\n[https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk)\n\nFp8:\n\n[https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled](https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled/tree/main/InfiniteTalk)\n\nfp16:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk)\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 129, + "type": "WanVideoVAELoader", + "pos": [ + 1379.0877685546875, + -3038.031494140625 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 436 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoVAELoader" + }, + "widgets_values": [ + "wanvideo\\Wan2_1_VAE_bf16.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 240, + "type": "SetNode", + "pos": [ + 1761.82666015625, + -3007.7861328125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "link": 436 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_VAE", + "properties": { + "previousName": "VAE" + }, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 260, + "type": "SetNode", + "pos": [ + 1843.7393798828125, + -2738.34326171875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "link": 463 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_wanmodel", + "properties": { + "previousName": "wanmodel" + }, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 238, + "type": "CLIPVisionLoader", + "pos": [ + 1406.407470703125, + -2358.7119140625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 466 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 264, + "type": "SetNode", + "pos": [ + 1711.6546630859375, + -2336.708984375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "link": 466 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_clip_vision_model", + "properties": { + "previousName": "clip_vision_model" + }, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 247, + "type": "SetNode", + "pos": [ + 2497.053955078125, + -2320.50537109375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 441 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_width", + "properties": { + "previousName": "width" + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 248, + "type": "SetNode", + "pos": [ + 2495.547607421875, + -2143.893310546875 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 442 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_height", + "properties": { + "previousName": "height" + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 159, + "type": "AudioCrop", + "pos": [ + 676.1865234375, + -1132.495361328125 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 26, + "mode": 4, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 257 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 301, + 448 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioCrop" + }, + "widgets_values": [ + "0:02", + "2:00" + ] + }, + { + "id": 170, + "type": "AudioSeparation", + "pos": [ + 1064.037353515625, + -1347.1170654296875 + ], + "size": [ + 270, + 166 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 301 + } + ], + "outputs": [ + { + "name": "Bass", + "type": "AUDIO", + "links": null + }, + { + "name": "Drums", + "type": "AUDIO", + "links": null + }, + { + "name": "Other", + "type": "AUDIO", + "links": null + }, + { + "name": "Vocals", + "type": "AUDIO", + "links": [ + 335 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioSeparation" + }, + "widgets_values": [ + "linear", + 10, + 0.1 + ] + }, + { + "id": 137, + "type": "DownloadAndLoadWav2VecModel", + "pos": [ + 1400.8470458984375, + -1411.5140380859375 + ], + "size": [ + 330.96728515625, + 106 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "links": [ + 334 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "DownloadAndLoadWav2VecModel" + }, + "widgets_values": [ + "TencentGameMate/chinese-wav2vec2-base", + "fp16", + "main_device" + ] + }, + { + "id": 253, + "type": "SetNode", + "pos": [ + 1002.430908203125, + -1102.810302734375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "link": 448 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_input_audio", + "properties": { + "previousName": "input_audio" + }, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 282, + "type": "GetNode", + "pos": [ + 1040.8619384765625, + -1797.7919921875 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 495 + ] + } + ], + "title": "Get_height", + "properties": {}, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 261, + "type": "GetNode", + "pos": [ + 3279.65771484375, + -2056.792724609375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "links": [ + 464 + ] + } + ], + "title": "Get_wanmodel", + "properties": {}, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 138, + "type": "WanVideoLoraSelect", + "pos": [ + 589.2518920898438, + -2738.91064453125 + ], + "size": [ + 503.4073486328125, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 349 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoLoraSelect" + }, + "widgets_values": [ + "WanVideo\\Lightx2v\\lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 1, + false, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 125, + "type": "LoadAudio", + "pos": [ + 536.47412109375, + -1357.4945068359375 + ], + "size": [ + 404.9698791503906, + 136 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 257 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "hush.mp3", + null, + null + ] + }, + { + "id": 134, + "type": "WanVideoBlockSwap", + "pos": [ + 587.2186279296875, + -3026.727294921875 + ], + "size": [ + 281.404296875, + 202 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "links": [ + 362 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 20, + false, + false, + true, + 0, + 1, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 245, + "type": "INTConstant", + "pos": [ + 2401.922607421875, + -2438.11865234375 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Width", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 246, + "type": "INTConstant", + "pos": [ + 2404.592529296875, + -2258.70654296875 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 442 + ] + } + ], + "title": "Height", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 237, + "type": "WanVideoClipVisionEncode", + "pos": [ + 2538.0830078125, + -1204.498779296875 + ], + "size": [ + 280.9771423339844, + 262 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 467 + }, + { + "name": "image_1", + "type": "IMAGE", + "link": 539 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "negative_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_CLIPEMBEDS", + "links": [ + 433 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoClipVisionEncode" + }, + "widgets_values": [ + 1, + 1, + "center", + "average", + true, + 0, + 0.5 + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 265, + "type": "GetNode", + "pos": [ + 2549.920166015625, + -1268.2757568359375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 467 + ] + } + ], + "title": "Get_clip_vision_model", + "properties": {}, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 242, + "type": "GetNode", + "pos": [ + 3670.50830078125, + -1982.0732421875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 438 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 270, + "type": "INTConstant", + "pos": [ + 2655.538330078125, + -2261.7578125 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 472 + ] + } + ], + "title": "Max frames", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 1000 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 294, + "type": "SetNode", + "pos": [ + 1751.949462890625, + -1184.374267578125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 519 + } + ], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 520 + ] + } + ], + "title": "Set_actual_audio_frames", + "properties": { + "previousName": "actual_audio_frames" + }, + "widgets_values": [ + "actual_audio_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 293, + "type": "PreviewAny", + "pos": [ + 1772.05078125, + -1110.6668701171875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 520 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [] + }, + { + "id": 271, + "type": "SetNode", + "pos": [ + 2660.10888671875, + -2152.97998046875 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 472 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_max_frames", + "properties": { + "previousName": "max_frames" + }, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 194, + "type": "MultiTalkWav2VecEmbeds", + "pos": [ + 1400.6778564453125, + -1237.9234619140625 + ], + "size": [ + 291.08203125, + 278 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "link": 334 + }, + { + "name": "audio_1", + "type": "AUDIO", + "link": 335 + }, + { + "name": "audio_2", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_3", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_4", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "ref_target_masks", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 529 + } + ], + "outputs": [ + { + "name": "multitalk_embeds", + "type": "MULTITALK_EMBEDS", + "links": [ + 357 + ] + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "num_frames", + "type": "INT", + "links": [ + 519 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3d7801cee4c8e3106078dd9b9f146caee95069ba", + "Node name for S&R": "MultiTalkWav2VecEmbeds" + }, + "widgets_values": [ + true, + 400, + 25, + 1, + 1, + "para" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 263, + "type": "Note", + "pos": [ + 1719.4892578125, + -965.56591796875 + ], + "size": [ + 344.82928466796875, + 143.7332305908203 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "increase audio_scale for stronger effect\n\naudio_cfg is only used when sampler is using cfg as well\n\nnum_frames is the maximum frame count to generate, if the value is higher than your audio length would be, the audio length is used " + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "WanVideoModelLoader", + "pos": [ + 1121.4686279296875, + -2764.419921875 + ], + "size": [ + 668.1777954101562, + 314 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 362 + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 349 + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": 355 + }, + { + "name": "fantasyportrait_model", + "shape": 7, + "type": "FANTASYPORTRAITMODEL", + "link": null + }, + { + "name": "extra_model", + "shape": 7, + "type": "VACEPATH", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 463 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoModelLoader" + }, + "widgets_values": [ + "WanVideo\\wan2.1-i2v-14b-480p-Q8_0.gguf", + "fp16_fast", + "disabled", + "offload_device", + "sageattn" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 177, + "type": "WanVideoTorchCompileSettings", + "pos": [ + 970.6024169921875, + -3048.0654296875 + ], + "size": [ + 342.74609375, + 202 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "links": [] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "f3614e6720744247f3211d60f7b9333f43572384", + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 272, + "type": "GetNode", + "pos": [ + 1174.7862548828125, + -1059.2822265625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 529 + ] + } + ], + "title": "Get_max_frames", + "properties": {}, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 130, + "type": "WanVideoDecode", + "pos": [ + 3675.3330078125, + -1893.888671875 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 438 + }, + { + "name": "samples", + "type": "LATENT", + "link": 530 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 493 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 131, + "type": "VHS_VideoCombine", + "pos": [ + 3689.34814453125, + -1589.9613037109375 + ], + "size": [ + 991.5499877929688, + 334 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 493 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 449 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 25, + "loop_count": 0, + "filename_prefix": "WanVideo2_1_InfiniteTalk", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_1_InfiniteTalk_00004-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 25, + "workflow": "WanVideo2_1_InfiniteTalk_00004.png", + "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_1_InfiniteTalk_00004-audio.mp4" + } + } + } + }, + { + "id": 254, + "type": "GetNode", + "pos": [ + 3683.283935546875, + -1643.584228515625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 18, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 449 + ] + } + ], + "title": "Get_input_audio", + "properties": {}, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 291, + "type": "GetImageSizeAndCount", + "pos": [ + 2239.1484375, + -1265.372314453125 + ], + "size": [ + 190.86483764648438, + 86 + ], + "flags": { + "collapsed": false + }, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 538 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 539, + 540 + ] + }, + { + "label": "640 width", + "name": "width", + "type": "INT", + "links": [ + 516 + ] + }, + { + "label": "640 height", + "name": "height", + "type": "INT", + "links": [ + 517 + ] + }, + { + "label": "1 count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3", + "Node name for S&R": "GetImageSizeAndCount" + }, + "widgets_values": [] + }, + { + "id": 241, + "type": "WanVideoTextEncodeCached", + "pos": [ + 2281.288330078125, + -1946.174072265625 + ], + "size": [ + 465.5179138183594, + 386.354248046875 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "extender_args", + "shape": 7, + "type": "WANVIDEOPROMPTEXTENDER_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 437 + ] + }, + { + "name": "negative_text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + }, + { + "name": "positive_prompt", + "type": "STRING", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoTextEncodeCached" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "a man is talking", + "bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards", + "disabled", + false, + "gpu" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 192, + "type": "WanVideoImageToVideoMultiTalk", + "pos": [ + 2836.906982421875, + -1288.3162841796875 + ], + "size": [ + 349.6015625, + 266 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 440 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 540 + }, + { + "name": "clip_embeds", + "shape": 7, + "type": "WANVIDIMAGE_CLIPEMBEDS", + "link": 433 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 516 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 517 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 413 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "49430f900bf812a1e281560bafafd04c43dfdf13", + "Node name for S&R": "WanVideoImageToVideoMultiTalk" + }, + "widgets_values": [ + 832, + 480, + 81, + 9, + false, + "disabled", + false, + "infinitetalk" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 244, + "type": "GetNode", + "pos": [ + 2825.2236328125, + -1325.1824951171875 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 440 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 128, + "type": "WanVideoSampler", + "pos": [ + 3279.373046875, + -1999.322021484375 + ], + "size": [ + 315, + 999 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 464 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 413 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 437 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": null + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": 357 + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 530 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 7, + 1.0000000000000002, + 11.000000000000002, + 2, + "fixed", + true, + "dpm++_sde", + 0, + 1, + false, + "comfy", + 0, + -1, + true + ] + }, + { + "id": 283, + "type": "GetNode", + "pos": [ + 1043.0416259765625, + -1848.6282958984375 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 494 + ] + } + ], + "title": "Get_width", + "properties": {}, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 284, + "type": "LoadImage", + "pos": [ + 638.8305053710938, + -1938.324951171875 + ], + "size": [ + 274.080078125, + 314 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 496 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "woman.png", + "image" + ] + }, + { + "id": 281, + "type": "ImageResizeKJv2", + "pos": [ + 1211.8646240234375, + -1936.6402587890625 + ], + "size": [ + 270, + 336 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 496 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 494 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 495 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 538 + ] + }, + { + "name": "width", + "type": "INT", + "links": [] + }, + { + "name": "height", + "type": "INT", + "links": [] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 832, + 480, + "lanczos", + "crop", + "0, 0, 0", + "center", + 16, + "cpu", + "Output: 1 x 640 x 640 | 4.69MB" + ] + }, + { + "id": 299, + "type": "Note", + "pos": [ + 1424.5263671875, + -2250.27001953125 + ], + "size": [ + 290.9361267089844, + 88 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Clip vision is not strictly necessary\n\nAny I2V model should work, MAGREF can be interesting to play with as well." + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 257, + 125, + 0, + 159, + 0, + "AUDIO" + ], + [ + 301, + 159, + 0, + 170, + 0, + "AUDIO" + ], + [ + 334, + 137, + 0, + 194, + 0, + "WAV2VECMODEL" + ], + [ + 335, + 170, + 3, + 194, + 1, + "AUDIO" + ], + [ + 349, + 138, + 0, + 122, + 2, + "WANVIDLORA" + ], + [ + 355, + 120, + 0, + 122, + 6, + "MULTITALKMODEL" + ], + [ + 357, + 194, + 0, + 128, + 15, + "MULTITALK_EMBEDS" + ], + [ + 362, + 134, + 0, + 122, + 1, + "BLOCKSWAPARGS" + ], + [ + 413, + 192, + 0, + 128, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 433, + 237, + 0, + 192, + 2, + "WANVIDIMAGE_CLIPEMBEDS" + ], + [ + 436, + 129, + 0, + 240, + 0, + "*" + ], + [ + 437, + 241, + 0, + 128, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 438, + 242, + 0, + 130, + 0, + "WANVAE" + ], + [ + 440, + 244, + 0, + 192, + 0, + "WANVAE" + ], + [ + 441, + 245, + 0, + 247, + 0, + "*" + ], + [ + 442, + 246, + 0, + 248, + 0, + "*" + ], + [ + 448, + 159, + 0, + 253, + 0, + "*" + ], + [ + 449, + 254, + 0, + 131, + 1, + "AUDIO" + ], + [ + 463, + 122, + 0, + 260, + 0, + "*" + ], + [ + 464, + 261, + 0, + 128, + 0, + "WANVIDEOMODEL" + ], + [ + 466, + 238, + 0, + 264, + 0, + "*" + ], + [ + 467, + 265, + 0, + 237, + 0, + "CLIP_VISION" + ], + [ + 472, + 270, + 0, + 271, + 0, + "*" + ], + [ + 493, + 130, + 0, + 131, + 0, + "IMAGE" + ], + [ + 494, + 283, + 0, + 281, + 2, + "INT" + ], + [ + 495, + 282, + 0, + 281, + 3, + "INT" + ], + [ + 496, + 284, + 0, + 281, + 0, + "IMAGE" + ], + [ + 516, + 291, + 1, + 192, + 3, + "INT" + ], + [ + 517, + 291, + 2, + 192, + 4, + "INT" + ], + [ + 519, + 194, + 2, + 294, + 0, + "*" + ], + [ + 520, + 294, + 0, + 293, + 0, + "*" + ], + [ + 529, + 272, + 0, + 194, + 6, + "INT" + ], + [ + 530, + 128, + 0, + 130, + 1, + "LATENT" + ], + [ + 538, + 281, + 0, + 291, + 0, + "IMAGE" + ], + [ + 539, + 291, + 0, + 237, + 1, + "IMAGE" + ], + [ + 540, + 291, + 0, + 192, + 1, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 480.1058044433594, + -3147.42529296875, + 1615.13037109375, + 1070.3359375 + ], + "color": "#88A", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Audio", + "bounding": [ + 462.57562255859375, + -1529.8046875, + 1656.1551513671875, + 753.9797973632812 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "Input image", + "bounding": [ + 466.7610778808594, + -2043.424560546875, + 1652.696533203125, + 495.40948486328125 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.45949729863576383, + "offset": [ + 34.34672984465828, + 2847.63895576458 + ] + }, + "frontendVersion": "1.26.3", + "node_versions": { + "ComfyUI-WanVideoWrapper": "0a11c67a0c0062b534178920a0d6dcaa75e7b5fe", + "comfy-core": "0.3.43", + "audio-separation-nodes-comfyui": "31a4567726e035097cc2d1f767767908a6fda2ea", + "ComfyUI-KJNodes": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "comfyui-videohelpersuite": "a7ce59e381934733bfae03b1be029756d6ce936d" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/wanvideo_InfiniteTalk_V2V_example_01.json b/example_workflows/wanvideo_InfiniteTalk_V2V_example_01.json new file mode 100644 index 0000000..dd7babc --- /dev/null +++ b/example_workflows/wanvideo_InfiniteTalk_V2V_example_01.json @@ -0,0 +1,2891 @@ +{ + "id": "8b7a9a57-2303-4ef5-9fc2-bf41713bd1fc", + "revision": 0, + "last_node_id": 301, + "last_link_id": 551, + "nodes": [ + { + "id": 234, + "type": "Note", + "pos": [ + 539.4052734375, + -832.4791870117188 + ], + "size": [ + 289.1911926269531, + 100.4728012084961 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "If audio is longer than the video, the result will extend from the last video frame" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 120, + "type": "MultiTalkModelLoader", + "pos": [ + 592.73583984375, + -2520.13525390625 + ], + "size": [ + 511.7737121582031, + 66.51400756835938 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "MULTITALKMODEL", + "links": [ + 355 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "MultiTalkModelLoader" + }, + "widgets_values": [ + "WanVideo\\InfiniteTalk\\InfiniteTalk\\Wan2_1-InfiniteTalk_Single_Q8.gguf" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 239, + "type": "MarkdownNote", + "pos": [ + 623.767822265625, + -2378.682861328125 + ], + "size": [ + 452.8328857421875, + 214.95587158203125 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": {}, + "widgets_values": [ + "You can't mix GGUF MultiTalk with non-GGUF main model, but you can mix different GGUF Qtypes with eachother.\n\nGGUF:\n\n[https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main](https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main)\n\n[https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy_GGUF/tree/main/InfiniteTalk)\n\nFp8:\n\n[https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled](https://huggingface.co/Kijai/WanVideo_comfy_fp8_scaled/tree/main/InfiniteTalk)\n\nfp16:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk](https://huggingface.co/Kijai/WanVideo_comfy/tree/main/InfiniteTalk)\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 243, + "type": "GetNode", + "pos": [ + 1715.7088623046875, + -924.8023071289062 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 439 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 129, + "type": "WanVideoVAELoader", + "pos": [ + 1379.0877685546875, + -3038.031494140625 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + } + ], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 436 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoVAELoader" + }, + "widgets_values": [ + "wanvideo\\Wan2_1_VAE_bf16.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 240, + "type": "SetNode", + "pos": [ + 1761.82666015625, + -3007.7861328125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "link": 436 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_VAE", + "properties": { + "previousName": "VAE" + }, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 260, + "type": "SetNode", + "pos": [ + 1843.7393798828125, + -2738.34326171875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "link": 463 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_wanmodel", + "properties": { + "previousName": "wanmodel" + }, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 238, + "type": "CLIPVisionLoader", + "pos": [ + 1406.407470703125, + -2358.7119140625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 466 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 264, + "type": "SetNode", + "pos": [ + 1711.6546630859375, + -2336.708984375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "link": 466 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_clip_vision_model", + "properties": { + "previousName": "clip_vision_model" + }, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 247, + "type": "SetNode", + "pos": [ + 2497.053955078125, + -2320.50537109375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 441 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_width", + "properties": { + "previousName": "width" + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 248, + "type": "SetNode", + "pos": [ + 2495.547607421875, + -2143.893310546875 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 442 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_height", + "properties": { + "previousName": "height" + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 170, + "type": "AudioSeparation", + "pos": [ + 1099.1485595703125, + -1843.935302734375 + ], + "size": [ + 270, + 166 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 301 + } + ], + "outputs": [ + { + "name": "Bass", + "type": "AUDIO", + "links": null + }, + { + "name": "Drums", + "type": "AUDIO", + "links": null + }, + { + "name": "Other", + "type": "AUDIO", + "links": null + }, + { + "name": "Vocals", + "type": "AUDIO", + "links": [ + 335 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioSeparation" + }, + "widgets_values": [ + "linear", + 10, + 0.1 + ] + }, + { + "id": 137, + "type": "DownloadAndLoadWav2VecModel", + "pos": [ + 1435.958251953125, + -1908.332275390625 + ], + "size": [ + 330.96728515625, + 106 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "links": [ + 334 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "DownloadAndLoadWav2VecModel" + }, + "widgets_values": [ + "TencentGameMate/chinese-wav2vec2-base", + "fp16", + "main_device" + ] + }, + { + "id": 253, + "type": "SetNode", + "pos": [ + 1037.5419921875, + -1599.6285400390625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "link": 448 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_input_audio", + "properties": { + "previousName": "input_audio" + }, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 261, + "type": "GetNode", + "pos": [ + 3379.9423828125, + -1989.2093505859375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVIDEOMODEL", + "type": "WANVIDEOMODEL", + "links": [ + 464 + ] + } + ], + "title": "Get_wanmodel", + "properties": {}, + "widgets_values": [ + "wanmodel" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 138, + "type": "WanVideoLoraSelect", + "pos": [ + 589.2518920898438, + -2738.91064453125 + ], + "size": [ + 503.4073486328125, + 150 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 349 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoLoraSelect" + }, + "widgets_values": [ + "WanVideo\\Lightx2v\\lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 1, + false, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 134, + "type": "WanVideoBlockSwap", + "pos": [ + 587.2186279296875, + -3026.727294921875 + ], + "size": [ + 281.404296875, + 202 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "links": [ + 362 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 20, + false, + false, + true, + 0, + 1, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 245, + "type": "INTConstant", + "pos": [ + 2401.922607421875, + -2438.11865234375 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Width", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 246, + "type": "INTConstant", + "pos": [ + 2404.592529296875, + -2258.70654296875 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 442 + ] + } + ], + "title": "Height", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 640 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 241, + "type": "WanVideoTextEncodeCached", + "pos": [ + 2799.24609375, + -1790.9327392578125 + ], + "size": [ + 465.5179138183594, + 386.354248046875 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "extender_args", + "shape": 7, + "type": "WANVIDEOPROMPTEXTENDER_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 437 + ] + }, + { + "name": "negative_text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + }, + { + "name": "positive_prompt", + "type": "STRING", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoTextEncodeCached" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "a woman is singing a lullaby", + "bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards", + "disabled", + false, + "gpu" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 254, + "type": "GetNode", + "pos": [ + 3936.826171875, + -1674.468505859375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 449 + ] + } + ], + "title": "Get_input_audio", + "properties": {}, + "widgets_values": [ + "input_audio" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 249, + "type": "GetNode", + "pos": [ + 897.5274047851562, + -921.2979125976562 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 443, + 511 + ] + } + ], + "title": "Get_height", + "properties": {}, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 250, + "type": "GetNode", + "pos": [ + 895.64990234375, + -977.6257934570312 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 444, + 510 + ] + } + ], + "title": "Get_width", + "properties": {}, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 192, + "type": "WanVideoImageToVideoMultiTalk", + "pos": [ + 2848.897705078125, + -1217.5718994140625 + ], + "size": [ + 349.6015625, + 266 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 440 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 543 + }, + { + "name": "clip_embeds", + "shape": 7, + "type": "WANVIDIMAGE_CLIPEMBEDS", + "link": 433 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 516 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 517 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 413 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "49430f900bf812a1e281560bafafd04c43dfdf13", + "Node name for S&R": "WanVideoImageToVideoMultiTalk" + }, + "widgets_values": [ + 832, + 480, + 81, + 9, + false, + "disabled", + false, + "infinitetalk" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 244, + "type": "GetNode", + "pos": [ + 2865.99169921875, + -1271.2249755859375 + ], + "size": [ + 210, + 34 + ], + "flags": { + "collapsed": true + }, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 440 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 237, + "type": "WanVideoClipVisionEncode", + "pos": [ + 2538.0830078125, + -1204.498779296875 + ], + "size": [ + 280.9771423339844, + 262 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 467 + }, + { + "name": "image_1", + "type": "IMAGE", + "link": 542 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "negative_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_CLIPEMBEDS", + "links": [ + 433 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "ff779c91714d8ee3484cd4119b082c72a1734b72", + "Node name for S&R": "WanVideoClipVisionEncode" + }, + "widgets_values": [ + 1, + 1, + "center", + "average", + true, + 0, + 0.5 + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 265, + "type": "GetNode", + "pos": [ + 2549.920166015625, + -1268.2757568359375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 467 + ] + } + ], + "title": "Get_clip_vision_model", + "properties": {}, + "widgets_values": [ + "clip_vision_model" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 242, + "type": "GetNode", + "pos": [ + 3764.5048828125, + -2040.7044677734375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 18, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 438 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 270, + "type": "INTConstant", + "pos": [ + 2655.538330078125, + -2261.7578125 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 472 + ] + } + ], + "title": "Max frames", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 1000 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 294, + "type": "SetNode", + "pos": [ + 1787.0606689453125, + -1681.1925048828125 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 519 + } + ], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 520 + ] + } + ], + "title": "Set_actual_audio_frames", + "properties": { + "previousName": "actual_audio_frames" + }, + "widgets_values": [ + "actual_audio_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 293, + "type": "PreviewAny", + "pos": [ + 1807.1619873046875, + -1607.485107421875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 520 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [] + }, + { + "id": 271, + "type": "SetNode", + "pos": [ + 2660.10888671875, + -2152.97998046875 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 472 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_max_frames", + "properties": { + "previousName": "max_frames" + }, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 263, + "type": "Note", + "pos": [ + 1754.6004638671875, + -1462.3841552734375 + ], + "size": [ + 344.82928466796875, + 143.7332305908203 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "increase audio_scale for stronger effect\n\naudio_cfg is only used when sampler is using cfg as well\n\nnum_frames is the maximum frame count to generate, if the value is higher than your audio length would be, the audio length is used " + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "WanVideoModelLoader", + "pos": [ + 1121.4686279296875, + -2764.419921875 + ], + "size": [ + 668.1777954101562, + 314 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 362 + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 349 + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + }, + { + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL", + "link": 355 + }, + { + "name": "fantasyportrait_model", + "shape": 7, + "type": "FANTASYPORTRAITMODEL", + "link": null + }, + { + "name": "extra_model", + "shape": 7, + "type": "VACEPATH", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 463 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "058286fc0f3b0651a2f6b68309df3f06e8332cc0", + "Node name for S&R": "WanVideoModelLoader" + }, + "widgets_values": [ + "WanVideo\\wan2.1-i2v-14b-480p-Q8_0.gguf", + "fp16_fast", + "disabled", + "offload_device", + "sageattn" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 177, + "type": "WanVideoTorchCompileSettings", + "pos": [ + 970.6024169921875, + -3048.0654296875 + ], + "size": [ + 342.74609375, + 202 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "links": [] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "f3614e6720744247f3211d60f7b9333f43572384", + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 229, + "type": "WanVideoEncode", + "pos": [ + 1707.7315673828125, + -876.6842041015625 + ], + "size": [ + 270, + 242 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 439 + }, + { + "name": "image", + "type": "IMAGE", + "link": 417 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 528 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "5375a01fb6dae02db244b8848087e87fbd69f44a", + "Node name for S&R": "WanVideoEncode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + 0, + 1 + ] + }, + { + "id": 228, + "type": "VHS_LoadVideo", + "pos": [ + 892.2421264648438, + -853.955810546875 + ], + "size": [ + 247.455078125, + 451.9747314453125 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "custom_width", + "type": "INT", + "widget": { + "name": "custom_width" + }, + "link": 510 + }, + { + "name": "custom_height", + "type": "INT", + "widget": { + "name": "custom_height" + }, + "link": 511 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 416 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": [] + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "8e4d79471bf1952154768e8435a9300077b534fa", + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "10.mp4", + "force_rate": 0, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "Wan", + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "10.mp4", + "type": "input", + "format": "video/mp4", + "force_rate": 0, + "custom_width": null, + "custom_height": 480, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1 + } + } + } + }, + { + "id": 291, + "type": "GetImageSizeAndCount", + "pos": [ + 2225.578125, + -1042.865478515625 + ], + "size": [ + 190.86483764648438, + 86 + ], + "flags": { + "collapsed": true + }, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 540 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 542, + 543 + ] + }, + { + "label": "640 width", + "name": "width", + "type": "INT", + "links": [ + 516 + ] + }, + { + "label": "640 height", + "name": "height", + "type": "INT", + "links": [ + 517 + ] + }, + { + "label": "1 count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3", + "Node name for S&R": "GetImageSizeAndCount" + }, + "widgets_values": [] + }, + { + "id": 125, + "type": "LoadAudio", + "pos": [ + 571.5848999023438, + -1854.312744140625 + ], + "size": [ + 404.9698791503906, + 136 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 257 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.41", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "one-does-not-simply-walk-into-mordor-its-black-gates-are-guarded-by-more-than-just-orcs.mp3", + null, + null + ] + }, + { + "id": 159, + "type": "AudioCrop", + "pos": [ + 711.2973022460938, + -1629.3135986328125 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 257 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 301, + 448 + ] + } + ], + "properties": { + "cnr_id": "audio-separation-nodes-comfyui", + "ver": "31a4567726e035097cc2d1f767767908a6fda2ea", + "Node name for S&R": "AudioCrop" + }, + "widgets_values": [ + "0:02", + "2:00" + ] + }, + { + "id": 272, + "type": "GetNode", + "pos": [ + 1234.6712646484375, + -1557.0916748046875 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 544 + ] + } + ], + "title": "Get_max_frames", + "properties": {}, + "widgets_values": [ + "max_frames" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 130, + "type": "WanVideoDecode", + "pos": [ + 3769.32958984375, + -1952.5198974609375 + ], + "size": [ + 315, + 198 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 438 + }, + { + "name": "samples", + "type": "LATENT", + "link": 532 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 545 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 230, + "type": "ImageResizeKJv2", + "pos": [ + 1336.364990234375, + -852.6764526367188 + ], + "size": [ + 270, + 336 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 416 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 444 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 443 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 417, + 418, + 547 + ] + }, + { + "name": "width", + "type": "INT", + "links": [] + }, + { + "name": "height", + "type": "INT", + "links": [] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 832, + 480, + "lanczos", + "crop", + "0, 0, 0", + "center", + 16, + "cpu", + "Output: 97 x 640 x 640 | 454.69MB" + ] + }, + { + "id": 231, + "type": "GetImageRangeFromBatch", + "pos": [ + 1340.5838623046875, + -1083.07763671875 + ], + "size": [ + 340.3267517089844, + 102 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 418 + }, + { + "name": "masks", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 540 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "GetImageRangeFromBatch" + }, + "widgets_values": [ + 0, + 1 + ] + }, + { + "id": 300, + "type": "GetImageSizeAndCount", + "pos": [ + 4164.92578125, + -1949.7347412109375 + ], + "size": [ + 190.86483764648438, + 86 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 545 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 549 + ] + }, + { + "label": "640 width", + "name": "width", + "type": "INT", + "links": null + }, + { + "label": "640 height", + "name": "height", + "type": "INT", + "links": null + }, + { + "label": "225 count", + "name": "count", + "type": "INT", + "links": [ + 551 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3", + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 301, + "type": "GetImageRangeFromBatch", + "pos": [ + 4187.93701171875, + -1824.931396484375 + ], + "size": [ + 340.3267517089844, + 102 + ], + "flags": { + "collapsed": true + }, + "order": 47, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 549 + }, + { + "name": "masks", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 551 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 550 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "e435e999e4b1a828a6b5f6d8f037e66f4a798324", + "Node name for S&R": "GetImageRangeFromBatch" + }, + "widgets_values": [ + 0, + 1 + ] + }, + { + "id": 131, + "type": "VHS_VideoCombine", + "pos": [ + 3826.9462890625, + -1626.7916259765625 + ], + "size": [ + 725.5770263671875, + 700.7885131835938 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 548 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 449 + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 25, + "loop_count": 0, + "filename_prefix": "WanVideo2_1_InfiniteTalk", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideo2_1_InfiniteTalk_00007-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 25, + "workflow": "WanVideo2_1_InfiniteTalk_00007.png", + "fullpath": "N:\\AI\\ComfyUI\\temp\\WanVideo2_1_InfiniteTalk_00007-audio.mp4" + } + } + } + }, + { + "id": 299, + "type": "ImageConcatMulti", + "pos": [ + 4190.734375, + -1750.7587890625 + ], + "size": [ + 270, + 150 + ], + "flags": { + "collapsed": true + }, + "order": 48, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 550 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 547 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 548 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "6dfd2c2420260dbb321a3412b9f1dff439a0f2e3" + }, + "widgets_values": [ + 2, + "left", + false, + null + ] + }, + { + "id": 128, + "type": "WanVideoSampler", + "pos": [ + 3379.65771484375, + -1931.739013671875 + ], + "size": [ + 315, + 999 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 464 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 413 + }, + { + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 437 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": 528 + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS", + "link": null + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + }, + { + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS", + "link": null + }, + { + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS", + "link": 357 + }, + { + "name": "freeinit_args", + "shape": 7, + "type": "FREEINITARGS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 532 + ] + }, + { + "name": "denoised_samples", + "type": "LATENT", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "c3ee35f3ece76e38099dc516182d69b406e16772", + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 4, + 1.0000000000000002, + 11.000000000000002, + 2, + "fixed", + true, + "dpm++_sde", + 0, + 1, + false, + "comfy", + 2, + -1, + true + ] + }, + { + "id": 194, + "type": "MultiTalkWav2VecEmbeds", + "pos": [ + 1435.7890625, + -1734.74169921875 + ], + "size": [ + 291.08203125, + 278 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "wav2vec_model", + "type": "WAV2VECMODEL", + "link": 334 + }, + { + "name": "audio_1", + "type": "AUDIO", + "link": 335 + }, + { + "name": "audio_2", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_3", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "audio_4", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "ref_target_masks", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 544 + } + ], + "outputs": [ + { + "name": "multitalk_embeds", + "type": "MULTITALK_EMBEDS", + "links": [ + 357 + ] + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "num_frames", + "type": "INT", + "links": [ + 519 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "3d7801cee4c8e3106078dd9b9f146caee95069ba", + "Node name for S&R": "MultiTalkWav2VecEmbeds" + }, + "widgets_values": [ + true, + 400, + 25, + 1.5, + 1, + "para" + ], + "color": "#323", + "bgcolor": "#535" + } + ], + "links": [ + [ + 257, + 125, + 0, + 159, + 0, + "AUDIO" + ], + [ + 301, + 159, + 0, + 170, + 0, + "AUDIO" + ], + [ + 334, + 137, + 0, + 194, + 0, + "WAV2VECMODEL" + ], + [ + 335, + 170, + 3, + 194, + 1, + "AUDIO" + ], + [ + 349, + 138, + 0, + 122, + 2, + "WANVIDLORA" + ], + [ + 355, + 120, + 0, + 122, + 6, + "MULTITALKMODEL" + ], + [ + 357, + 194, + 0, + 128, + 15, + "MULTITALK_EMBEDS" + ], + [ + 362, + 134, + 0, + 122, + 1, + "BLOCKSWAPARGS" + ], + [ + 413, + 192, + 0, + 128, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 416, + 228, + 0, + 230, + 0, + "IMAGE" + ], + [ + 417, + 230, + 0, + 229, + 1, + "IMAGE" + ], + [ + 418, + 230, + 0, + 231, + 0, + "IMAGE" + ], + [ + 433, + 237, + 0, + 192, + 2, + "WANVIDIMAGE_CLIPEMBEDS" + ], + [ + 436, + 129, + 0, + 240, + 0, + "*" + ], + [ + 437, + 241, + 0, + 128, + 2, + "WANVIDEOTEXTEMBEDS" + ], + [ + 438, + 242, + 0, + 130, + 0, + "WANVAE" + ], + [ + 439, + 243, + 0, + 229, + 0, + "WANVAE" + ], + [ + 440, + 244, + 0, + 192, + 0, + "WANVAE" + ], + [ + 441, + 245, + 0, + 247, + 0, + "*" + ], + [ + 442, + 246, + 0, + 248, + 0, + "*" + ], + [ + 443, + 249, + 0, + 230, + 3, + "INT" + ], + [ + 444, + 250, + 0, + 230, + 2, + "INT" + ], + [ + 448, + 159, + 0, + 253, + 0, + "*" + ], + [ + 449, + 254, + 0, + 131, + 1, + "AUDIO" + ], + [ + 463, + 122, + 0, + 260, + 0, + "*" + ], + [ + 464, + 261, + 0, + 128, + 0, + "WANVIDEOMODEL" + ], + [ + 466, + 238, + 0, + 264, + 0, + "*" + ], + [ + 467, + 265, + 0, + 237, + 0, + "CLIP_VISION" + ], + [ + 472, + 270, + 0, + 271, + 0, + "*" + ], + [ + 510, + 250, + 0, + 228, + 2, + "INT" + ], + [ + 511, + 249, + 0, + 228, + 3, + "INT" + ], + [ + 516, + 291, + 1, + 192, + 3, + "INT" + ], + [ + 517, + 291, + 2, + 192, + 4, + "INT" + ], + [ + 519, + 194, + 2, + 294, + 0, + "*" + ], + [ + 520, + 294, + 0, + 293, + 0, + "*" + ], + [ + 528, + 229, + 0, + 128, + 3, + "LATENT" + ], + [ + 532, + 128, + 0, + 130, + 1, + "LATENT" + ], + [ + 540, + 231, + 0, + 291, + 0, + "IMAGE" + ], + [ + 542, + 291, + 0, + 237, + 1, + "IMAGE" + ], + [ + 543, + 291, + 0, + 192, + 1, + "IMAGE" + ], + [ + 544, + 272, + 0, + 194, + 6, + "INT" + ], + [ + 545, + 130, + 0, + 300, + 0, + "IMAGE" + ], + [ + 547, + 230, + 0, + 299, + 1, + "IMAGE" + ], + [ + 548, + 299, + 0, + 131, + 0, + "IMAGE" + ], + [ + 549, + 300, + 0, + 301, + 0, + "IMAGE" + ], + [ + 550, + 301, + 0, + 299, + 0, + "IMAGE" + ], + [ + 551, + 300, + 3, + 301, + 2, + "INT" + ] + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 480.1058044433594, + -3147.42529296875, + 1615.13037109375, + 1070.3359375 + ], + "color": "#88A", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Video2video", + "bounding": [ + 498.3595886230469, + -1241.007080078125, + 1720.7049560546875, + 926.4686889648438 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Audio", + "bounding": [ + 497.6866149902344, + -2026.6229248046875, + 1656.1551513671875, + 753.9797973632812 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.41772481694160374, + "offset": [ + -263.000991979628, + 2820.474862310073 + ] + }, + "frontendVersion": "1.26.3", + "node_versions": { + "ComfyUI-WanVideoWrapper": "0a11c67a0c0062b534178920a0d6dcaa75e7b5fe", + "comfy-core": "0.3.43", + "audio-separation-nodes-comfyui": "31a4567726e035097cc2d1f767767908a6fda2ea", + "ComfyUI-KJNodes": "f7eb33abc80a2aded1b46dff0dd14d07856a7d50", + "comfyui-videohelpersuite": "a7ce59e381934733bfae03b1be029756d6ce936d" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/fantasytalking/nodes.py b/fantasytalking/nodes.py index 2458407..928aa1b 100644 --- a/fantasytalking/nodes.py +++ b/fantasytalking/nodes.py @@ -20,8 +20,8 @@ class DownloadAndLoadWav2VecModel: "required": { "model": ( [ - "facebook/wav2vec2-base-960h", - "TencentGameMate/chinese-wav2vec2-base" + "TencentGameMate/chinese-wav2vec2-base", + "facebook/wav2vec2-base-960h" ], ), diff --git a/multitalk/nodes.py b/multitalk/nodes.py index e4ee204..628c5d2 100644 --- a/multitalk/nodes.py +++ b/multitalk/nodes.py @@ -52,7 +52,8 @@ class MultiTalkModelLoader: multitalk = { "proj_model": multitalk_proj_model, "sd": sd, - "is_gguf": model_path.endswith(".gguf") + "is_gguf": model_path.endswith(".gguf"), + "model_type": "InfiniteTalk" if "infinite" in model.lower() else "MultiTalk", } return (multitalk,) @@ -75,8 +76,8 @@ class MultiTalkWav2VecEmbeds: return {"required": { "wav2vec_model": ("WAV2VECMODEL",), "audio_1": ("AUDIO",), - "normalize_loudness": ("BOOLEAN", {"default": True}), - "num_frames": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 1}), + "normalize_loudness": ("BOOLEAN", {"default": True, "tooltip": "Normalize the audio loudness to -23 LUFS"}), + "num_frames": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 1, "tooltip": "The total frame count to generate."}), "fps": ("FLOAT", {"default": 25.0, "min": 1.0, "max": 60.0, "step": 0.1}), "audio_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "tooltip": "Strength of the audio conditioning"}), "audio_cfg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "tooltip": "When not 1.0, an extra model pass without audio conditioning is done: slower inference but more motion is allowed"}), @@ -90,8 +91,8 @@ class MultiTalkWav2VecEmbeds: } } - RETURN_TYPES = ("MULTITALK_EMBEDS", "AUDIO", ) - RETURN_NAMES = ("multitalk_embeds", "audio", ) + RETURN_TYPES = ("MULTITALK_EMBEDS", "AUDIO", "INT", ) + RETURN_NAMES = ("multitalk_embeds", "audio", "num_frames", ) FUNCTION = "process" CATEGORY = "WanVideoWrapper" @@ -230,12 +231,30 @@ class MultiTalkWav2VecEmbeds: offset += w.shape[-1] out_audio = {"waveform": mixed, "sample_rate": sr} + # Calculate actual frames based on audio duration + actual_num_frames = num_frames + if len(audio_outputs) > 0: + if multi_audio_type == "para": + # For parallel mode, use the longest audio duration + max_audio_duration = max([ao["waveform"].shape[-1] / sr for ao in audio_outputs]) + actual_frames_from_audio = int(max_audio_duration * fps) + else: # "add" + # For sequential mode, use the total audio duration + total_audio_duration = sum([ao["waveform"].shape[-1] / sr for ao in audio_outputs]) + actual_frames_from_audio = int(total_audio_duration * fps) + + # Use the smaller of requested frames or actual audio frames + actual_num_frames = min(num_frames, actual_frames_from_audio) + + if actual_frames_from_audio < num_frames: + log.info(f"[MultiTalk] Audio duration ({actual_frames_from_audio} frames) is shorter than requested ({num_frames} frames). Using {actual_num_frames} frames.") + # Debug: log final mixed audio length and mode total_samples_raw = sum([ao["waveform"].shape[-1] for ao in audio_outputs]) log.info(f"[MultiTalk] total raw duration = {total_samples_raw/sr:.3f}s") log.info(f"[MultiTalk] multi_audio_type={multi_audio_type} | final waveform shape={out_audio['waveform'].shape} | length={out_audio['waveform'].shape[-1]} samples | seconds={out_audio['waveform'].shape[-1]/sr:.3f}s (expected {'sum' if multi_audio_type=='add' else 'max'} of raw)") - return (multitalk_embeds, out_audio) + return (multitalk_embeds, out_audio, actual_num_frames) class WanVideoImageToVideoMultiTalk: @@ -243,11 +262,11 @@ class WanVideoImageToVideoMultiTalk: def INPUT_TYPES(s): return {"required": { "vae": ("WANVAE",), - "width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8, "tooltip": "Width of the image to encode"}), - "height": ("INT", {"default": 480, "min": 64, "max": 29048, "step": 8, "tooltip": "Height of the image to encode"}), - "frame_window_size": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 4, "tooltip": "Number of frames to encode"}), - "motion_frame": ("INT", {"default": 25, "min": 1, "max": 10000, "step": 1, "tooltip": "Driven frame length used in the long video generation."}), - "force_offload": ("BOOLEAN", {"default": True}), + "width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8, "tooltip": "Width of the generation"}), + "height": ("INT", {"default": 480, "min": 64, "max": 29048, "step": 8, "tooltip": "Height of the generation"}), + "frame_window_size": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 4, "tooltip": "The number of frames to process at once, should be a value the model is generally good at."}), + "motion_frame": ("INT", {"default": 25, "min": 1, "max": 10000, "step": 1, "tooltip": "Driven frame length used in the long video generation. Basically the overlap length."}), + "force_offload": ("BOOLEAN", {"default": False, "tooltip": "Whether to force offload the model within the loop for VAE operations, enable if you encounter memory issues."}), "colormatch": ( [ 'disabled', @@ -258,17 +277,18 @@ class WanVideoImageToVideoMultiTalk: 'hm-mvgd-hm', 'hm-mkl-hm', ], { - "default": 'disabled' - }), + "default": 'disabled', "tooltip": "Color matching method to use between the windows" + },), }, "optional": { - "start_image": ("IMAGE", {"tooltip": "Image to encode"}), + "start_image": ("IMAGE", {"tooltip": "Images to encode"}), "tiled_vae": ("BOOLEAN", {"default": False, "tooltip": "Use tiled VAE encoding for reduced memory use"}), "clip_embeds": ("WANVIDIMAGE_CLIPEMBEDS", {"tooltip": "Clip vision encoded image"}), "mode": ([ + "auto", "multitalk", "infinitetalk" - ], {"default": "multitalk", "tooltip": "The sampling strategy to use in the long video generation loop, should match the model used"}) + ], {"default": "auto", "tooltip": "The sampling strategy to use in the long video generation loop, should match the model used"}) } } @@ -276,6 +296,7 @@ class WanVideoImageToVideoMultiTalk: RETURN_NAMES = ("image_embeds",) FUNCTION = "process" CATEGORY = "WanVideoWrapper" + DESCRIPTION = "Enables Multi/InfiniteTalk long video generation sampling method, the video is created in windows with overlapping frames. Not compatible or necessary to be used with context windows and many other features besides Multi/InfiniteTalk." def process(self, vae, width, height, frame_window_size, motion_frame, force_offload, colormatch, start_image=None, tiled_vae=False, clip_embeds=None, mode="multitalk"): @@ -320,7 +341,7 @@ NODE_CLASS_MAPPINGS = { } NODE_DISPLAY_NAME_MAPPINGS = { - "MultiTalkModelLoader": "MultiTalk Model Loader", - "MultiTalkWav2VecEmbeds": "MultiTalk Wav2Vec Embeds", - "WanVideoImageToVideoMultiTalk": "WanVideo Image To Video MultiTalk" + "MultiTalkModelLoader": "Multi/InfiniteTalk Model Loader", + "MultiTalkWav2VecEmbeds": "Multi/InfiniteTalk Wav2Vec Embeds", + "WanVideoImageToVideoMultiTalk": "WanVideo Long I2V Multi/InfiniteTalk" } \ No newline at end of file diff --git a/nodes.py b/nodes.py index 55bdcb0..8b236d7 100644 --- a/nodes.py +++ b/nodes.py @@ -874,9 +874,9 @@ class WanVideoImageToVideoEncode: H = height W = width - lat_h = H // 8 - lat_w = W // 8 - + lat_h = H // vae.upsampling_factor + lat_w = W // vae.upsampling_factor + num_frames = ((num_frames - 1) // 4) * 4 + 1 two_ref_images = start_image is not None and end_image is not None @@ -1280,8 +1280,6 @@ class WanVideoVACEEncode: CATEGORY = "WanVideoWrapper" def process(self, vae, width, height, num_frames, strength, vace_start_percent, vace_end_percent, input_frames=None, ref_images=None, input_masks=None, prev_vace_embeds=None, tiled_vae=False): - vae = vae.to(device) - width = (width // 16) * 16 height = (height // 16) * 16 @@ -1292,8 +1290,8 @@ class WanVideoVACEEncode: if input_frames is None: input_frames = torch.zeros((1, 3, num_frames, height, width), device=device, dtype=vae.dtype) else: - input_frames = input_frames[:num_frames] - input_frames = common_upscale(input_frames.clone().movedim(-1, 1), width, height, "lanczos", "disabled").movedim(1, -1) + input_frames = input_frames.clone()[:num_frames, :, :, :3] + input_frames = common_upscale(input_frames.movedim(-1, 1), width, height, "lanczos", "disabled").movedim(1, -1) input_frames = input_frames.to(vae.dtype).to(device).unsqueeze(0).permute(0, 4, 1, 2, 3) # B, C, T, H, W input_frames = input_frames * 2 - 1 if input_masks is None: @@ -1306,10 +1304,11 @@ class WanVideoVACEEncode: input_masks = input_masks.unsqueeze(-1).unsqueeze(0).permute(0, 4, 1, 2, 3).repeat(1, 3, 1, 1, 1) # B, C, T, H, W if ref_images is not None: + ref_images = ref_images.clone()[..., :3] # Create padded image if ref_images.shape[0] > 1: ref_images = torch.cat([ref_images[i] for i in range(ref_images.shape[0])], dim=1).unsqueeze(0) - + B, H, W, C = ref_images.shape current_aspect = W / H target_aspect = width / height @@ -1331,12 +1330,12 @@ class WanVideoVACEEncode: ref_images = ref_images.to(vae.dtype).to(device).unsqueeze(0).permute(0, 4, 1, 2, 3).unsqueeze(0) ref_images = ref_images * 2 - 1 - + + vae = vae.to(device) z0 = self.vace_encode_frames(vae, input_frames, ref_images, masks=input_masks, tiled_vae=tiled_vae) vae.model.clear_cache() m0 = self.vace_encode_masks(input_masks, ref_images) z = self.vace_latent(z0, m0) - vae.to(offload_device) vace_input = { @@ -1357,6 +1356,7 @@ class WanVideoVACEEncode: vace_input["additional_vace_inputs"].append(prev_vace_embeds) return (vace_input,) + def vace_encode_frames(self, vae, frames, ref_images, masks=None, tiled_vae=False): if ref_images is None: ref_images = [None] * len(frames) @@ -1675,6 +1675,10 @@ class WanVideoSampler: transformer = compile_model(transformer, model["compile_args"]) multitalk_sampling = image_embeds.get("multitalk_sampling", False) + + if multitalk_sampling and context_options is not None: + raise Exception("context_options are not compatible or necessary with 'WanVideoImageToVideoMultiTalk' node, since it's already an alternative method that creates the video in a loop.") + if not multitalk_sampling and scheduler == "multitalk": raise Exception("multitalk scheduler is only for multitalk sampling when using ImagetoVideoMultiTalk -node") @@ -1803,7 +1807,7 @@ class WanVideoSampler: control_embeds = image_embeds.get("control_embeds", None) if control_embeds is not None: - if transformer.in_dim not in [52, 48, 36, 32]: + if transformer.in_dim not in [148, 52, 48, 36, 32]: raise ValueError("Control signal only works with Fun-Control model") control_latents = control_embeds.get("control_images", None) @@ -1900,10 +1904,10 @@ class WanVideoSampler: patcher = apply_lora(patcher, device, device, low_mem_load=False, control_lora=True) patcher.model.is_patched = True else: - if transformer.in_dim not in [48, 36, 32, 52]: + if transformer.in_dim not in [148, 48, 36, 32, 52]: raise ValueError("Control signal only works with Fun-Control model") image_cond = torch.zeros_like(noise).to(device) #fun control - if transformer.in_dim == 52 or transformer.control_adapter is not None: #fun 2.2 control + if transformer.in_dim in [148, 52] or transformer.control_adapter is not None: #fun 2.2 control mask_latents = torch.tile( torch.zeros_like(noise[:1]), [4, 1, 1, 1] ) @@ -1917,7 +1921,7 @@ class WanVideoSampler: control_start_percent = control_embeds.get("start_percent", 0.0) control_end_percent = control_embeds.get("end_percent", 1.0) else: - if transformer.in_dim == 36: #fun inp + if transformer.in_dim in [148, 52]: #fun inp mask_latents = torch.tile( torch.zeros_like(noise[:1]), [4, 1, 1, 1] ) @@ -2113,7 +2117,7 @@ class WanVideoSampler: mtv_freqs = mtv_freqs.to(device, dtype) # vid2vid - if samples is not None: + if samples is not None and not multitalk_sampling: saved_generator_state = samples.get("generator_state", None) if saved_generator_state is not None: seed_g.set_state(saved_generator_state) @@ -2127,29 +2131,29 @@ class WanVideoSampler: else: noise = input_samples - mask = samples.get("noise_mask", None) - if mask is not None: - log.info(f"Latent mask shape: {mask.shape}") + noise_mask = samples.get("noise_mask", None) + if noise_mask is not None: + log.info(f"Latent noise_mask shape: {noise_mask.shape}") original_image = input_samples.to(device) - if len(mask.shape) == 4: - mask = mask.squeeze(1) + if len(noise_mask.shape) == 4: + noise_mask = noise_mask.squeeze(1) - mask = torch.nn.functional.interpolate( - mask.unsqueeze(0).unsqueeze(0), # Add batch and channel dims [1,1,T,H,W] + noise_mask = torch.nn.functional.interpolate( + noise_mask.unsqueeze(0).unsqueeze(0), # Add batch and channel dims [1,1,T,H,W] size=(noise.shape[1], noise.shape[2], noise.shape[3]), mode='trilinear', align_corners=False ).squeeze(0) # Remove batch dim, keep channel dim # Add batch & channel dims for final output - mask = mask.unsqueeze(0).repeat(1, noise.shape[0], 1, 1, 1) + noise_mask = noise_mask.unsqueeze(0).repeat(1, noise.shape[0], 1, 1, 1) - if mask.shape[2] != noise.shape[1]: - mask = torch.cat([torch.zeros(1, noise.shape[0], noise.shape[1] - mask.shape[2], noise.shape[2], noise.shape[3]), mask], dim=2) + if noise_mask.shape[2] != noise.shape[1]: + noise_mask = torch.cat([torch.zeros(1, noise.shape[0], noise.shape[1] - noise_mask.shape[2], noise.shape[2], noise.shape[3]), noise_mask], dim=2) # extra latents (Pusa) and 5b latents_to_insert = add_index = None - if (extra_latents := image_embeds.get("extra_latents", None)) is not None: + if (extra_latents := image_embeds.get("extra_latents", None)) is not None and transformer.multitalk_model_type.lower() != "infinitetalk": all_indices = [] for entry in extra_latents: add_index = entry["index"] @@ -2178,7 +2182,7 @@ class WanVideoSampler: if uni3c_embeds is not None: transformer.controlnet = uni3c_embeds["controlnet"] pcd_data = { - "render_latent": uni3c_embeds["render_latent"].to(dtype), + "render_latent": uni3c_embeds["render_latent"], "render_mask": uni3c_embeds["render_mask"], "camera_embedding": uni3c_embeds["camera_embedding"], "controlnet_weight": uni3c_embeds["controlnet_weight"], @@ -2701,18 +2705,19 @@ class WanVideoSampler: from .latent_preview import prepare_callback #custom for tiny VAE previews callback = prepare_callback(patcher, len(timesteps)) - log.info(f"Input sequence length: {seq_len}") - log.info(f"Sampling {(latent_video_length-1) * 4 + 1} frames at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") + if not multitalk_sampling: + log.info(f"Input sequence length: {seq_len}") + log.info(f"Sampling {(latent_video_length-1) * 4 + 1} frames at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") intermediate_device = device # diff diff prep masks = None - if samples is not None and mask is not None: - mask = 1 - mask + if not multitalk_sampling and samples is not None and noise_mask is not None: + noise_mask = 1 - noise_mask thresholds = torch.arange(len(timesteps), dtype=original_image.dtype) / len(timesteps) thresholds = thresholds.unsqueeze(1).unsqueeze(1).unsqueeze(1).unsqueeze(1).to(device) - masks = mask.repeat(len(timesteps), 1, 1, 1, 1).to(device) + masks = noise_mask.repeat(len(timesteps), 1, 1, 1, 1).to(device) masks = masks > thresholds latent_shift_loop = False @@ -2789,27 +2794,24 @@ class WanVideoSampler: try: pbar = ProgressBar(len(timesteps)) #region main loop start - for idx, t in enumerate(tqdm(timesteps)): + for idx, t in enumerate(tqdm(timesteps, disable=multitalk_sampling)): if flowedit_args is not None: if idx < skip_steps: continue - # diff diff - if masks is not None: - if idx < len(timesteps) - 1: - noise_timestep = timesteps[idx+1] - image_latent = sample_scheduler.scale_noise( - original_image, torch.tensor([noise_timestep]), noise.to(device) - ) - mask = masks[idx] - mask = mask.to(latent) - latent = image_latent * mask + latent * (1-mask) - # end diff diff - if bidirectional_sampling: latent_flipped = torch.flip(latent, dims=[1]) latent_model_input_flipped = latent_flipped.to(device) + #InfiniteTalk first frame handling + if (extra_latents is not None + and not multitalk_sampling + and transformer.multitalk_model_type=="InfiniteTalk"): + for entry in extra_latents: + add_index = entry["index"] + num_extra_frames = entry["samples"].shape[2] + latent[:, add_index:add_index+num_extra_frames] = entry["samples"].to(latent) + latent_model_input = latent.to(device) current_step_percentage = idx / len(timesteps) @@ -3097,8 +3099,10 @@ class WanVideoSampler: #region multitalk elif multitalk_sampling: mode = image_embeds.get("multitalk_mode", "multitalk") + if mode == "auto": + mode = transformer.multitalk_model_type.lower() log.info(f"Multitalk mode: {mode}") - original_images = cond_image = image_embeds.get("multitalk_start_image", None) + cond_frame = None offload = image_embeds.get("force_offload", False) tiled_vae = image_embeds.get("tiled_vae", False) frame_num = clip_length = image_embeds.get("num_frames", 81) @@ -3110,23 +3114,20 @@ class WanVideoSampler: motion_frame = image_embeds.get("motion_frame", 25) target_w = image_embeds.get("target_w", None) target_h = image_embeds.get("target_h", None) + original_images = cond_image = image_embeds.get("multitalk_start_image", None) + if original_images is None: + original_images = torch.zeros([noise.shape[0], 1, target_h, target_w], device=device) if len(multitalk_embeds['audio_features'])==2 and (multitalk_embeds['ref_target_masks'] is None): face_scale = 0.1 x_min, x_max = int(target_h * face_scale), int(target_h * (1 - face_scale)) - background_mask = torch.zeros([target_h, target_w]) - background_mask = torch.zeros([target_h, target_w]) - human_mask1 = torch.zeros([target_h, target_w]) - human_mask2 = torch.zeros([target_h, target_w]) lefty_min, lefty_max = int((target_w//2) * face_scale), int((target_w//2) * (1 - face_scale)) righty_min, righty_max = int((target_w//2) * face_scale + (target_w//2)), int((target_w//2) * (1 - face_scale) + (target_w//2)) + human_mask1, human_mask2 = (torch.zeros([target_h, target_w]) for _ in range(2)) human_mask1[x_min:x_max, lefty_min:lefty_max] = 1 human_mask2[x_min:x_max, righty_min:righty_max] = 1 - background_mask += human_mask1 - background_mask += human_mask2 - human_masks = [human_mask1, human_mask2] - background_mask = torch.where(background_mask > 0, torch.tensor(0), torch.tensor(1)) - human_masks.append(background_mask) + background_mask = torch.where((human_mask1 + human_mask2) > 0, torch.tensor(0), torch.tensor(1)) + human_masks = [human_mask1, human_mask2, background_mask] ref_target_masks = torch.stack(human_masks, dim=0) multitalk_embeds['ref_target_masks'] = ref_target_masks @@ -3134,7 +3135,7 @@ class WanVideoSampler: is_first_clip = True arrive_last_frame = False cur_motion_frames_num = 1 - audio_start_idx = iteration_count = 0 + audio_start_idx = iteration_count = step_iteration_count= 0 audio_end_idx = audio_start_idx + clip_length indices = (torch.arange(4 + 1) - 2) * 1 current_condframe_index = 0 @@ -3146,7 +3147,7 @@ class WanVideoSampler: if uni3c_embeds is not None: transformer.controlnet = uni3c_embeds["controlnet"] pcd_data = { - "render_latent": uni3c_embeds["render_latent"].to(dtype), + "render_latent": uni3c_embeds["render_latent"], "render_mask": uni3c_embeds["render_mask"], "camera_embedding": uni3c_embeds["camera_embedding"], "controlnet_weight": uni3c_embeds["controlnet_weight"], @@ -3155,19 +3156,19 @@ class WanVideoSampler: } estimated_iterations = total_frames // (frame_num - motion_frame) + 1 - log.info(f"Total frames: {total_frames}, frame_num: {frame_num}, motion_frame: {motion_frame}") - log.info(f"Estimated iterations: {estimated_iterations}") - loop_pbar = tqdm(total=estimated_iterations, desc="Generating video clips") + loop_pbar = tqdm(total=estimated_iterations, desc="Total progress", position=1, leave=True) callback = prepare_callback(patcher, estimated_iterations) audio_embedding = multitalk_audio_embedding human_num = len(audio_embedding) audio_embs = None + + log.info(f"Sampling {total_frames} frames in {estimated_iterations} windows, at {latent.shape[3]*vae_upscale_factor}x{latent.shape[2]*vae_upscale_factor} with {steps} steps") + while True: # start video generation iteratively + cur_motion_frames_latent_num = int(1 + (cur_motion_frames_num-1) // 4) if mode == "infinitetalk": - cond_image = original_images[:, :, current_condframe_index:current_condframe_index+1] - log.info(f"current_condframe_index: {current_condframe_index}") - log.info(f"audio_start_idx: {audio_start_idx}") + cond_image = original_images[:, :, current_condframe_index:current_condframe_index+1] if cond_image is not None else None if multitalk_embeds is not None: audio_embs = [] # split audio with window size @@ -3180,18 +3181,83 @@ class WanVideoSampler: if uni3c_embeds is not None: vae.to(device) - render_latent = vae.encode(original_images[:, :, audio_start_idx:audio_end_idx].to(device, vae.dtype), device=device, tiled=tiled_vae).to(dtype) + # Pad original_images if needed + num_frames = original_images.shape[2] + required_frames = audio_end_idx - audio_start_idx + if audio_end_idx > num_frames: + pad_len = audio_end_idx - num_frames + last_frame = original_images[:, :, -1:].repeat(1, 1, pad_len, 1, 1) + padded_images = torch.cat([original_images, last_frame], dim=2) + else: + padded_images = original_images + render_latent = vae.encode( + padded_images[:, :, audio_start_idx:audio_end_idx].to(device, vae.dtype), + device=device, tiled=tiled_vae + ).to(dtype) pcd_data['render_latent'] = render_latent - h, w = cond_image.shape[-2], cond_image.shape[-1] + h, w = (cond_image.shape[-2], cond_image.shape[-1]) if cond_image is not None else (target_h, target_w) lat_h, lat_w = h // VAE_STRIDE[1], w // VAE_STRIDE[2] seq_len = ((frame_num - 1) // VAE_STRIDE[0] + 1) * lat_h * lat_w // (PATCH_SIZE[1] * PATCH_SIZE[2]) noise = torch.randn( 16, (frame_num - 1) // 4 + 1, lat_h, lat_w, dtype=torch.float32, device=torch.device("cpu"), generator=seed_g).to(device) + + # Calculate the correct latent slice based on current iteration + if is_first_clip: + latent_start_idx = 0 + latent_end_idx = noise.shape[1] + else: + new_frames_per_iteration = frame_num - motion_frame + new_latent_frames_per_iteration = ((new_frames_per_iteration - 1) // 4 + 1) + latent_start_idx = iteration_count * new_latent_frames_per_iteration + latent_end_idx = latent_start_idx + noise.shape[1] + + if samples is not None: + input_samples = samples["samples"].squeeze(0).to(noise) + # Check if we have enough frames in input_samples + if latent_end_idx > input_samples.shape[1]: + # We need more frames than available - pad the input_samples at the end + pad_length = latent_end_idx - input_samples.shape[1] + last_frame = input_samples[:, -1:].repeat(1, pad_length, 1, 1) + input_samples = torch.cat([input_samples, last_frame], dim=1) + input_samples = input_samples[:, latent_start_idx:latent_end_idx] - # get mask + assert input_samples.shape[1] == noise.shape[1], f"Slice mismatch: {input_samples.shape[1]} vs {noise.shape[1]}" + + if add_noise_to_samples: + latent_timestep = timesteps[0] + noise = noise * latent_timestep / 1000 + (1 - latent_timestep / 1000) * input_samples + else: + noise = input_samples + + # diff diff prep + masks = None + if noise_mask is not None: + noise_mask = 1 - noise_mask + thresholds = torch.arange(len(timesteps), dtype=original_image.dtype) / len(timesteps) + thresholds = thresholds.unsqueeze(1).unsqueeze(1).unsqueeze(1).unsqueeze(1).to(device) + masks = noise_mask.repeat(len(timesteps), 1, 1, 1, 1).to(device) + masks = masks > thresholds + + window_vace_data = None + if vace_data is not None: + window_vace_data = [] + for vace_entry in vace_data: + partial_context = vace_entry["context"][0][:, latent_start_idx:latent_end_idx] + if has_ref: + partial_context[:, 0] = vace_entry["context"][0][:, 0] + + window_vace_data.append({ + "context": [partial_context], + "scale": vace_entry["scale"], + "start": vace_entry["start"], + "end": vace_entry["end"], + "seq_len": vace_entry["seq_len"] + }) + + # get image cond mask msk = torch.ones(1, frame_num, lat_h, lat_w, device=device) if mode == "multitalk": msk[:, cur_motion_frames_num:] = 0 @@ -3206,26 +3272,28 @@ class WanVideoSampler: mm.soft_empty_cache() # zero padding and vae encode - video_frames = torch.zeros(1, cond_image.shape[1], frame_num-cond_image.shape[2], target_h, target_w, device=device, dtype=vae.dtype) - padding_frames_pixels_values = torch.concat([cond_image.to(device, vae.dtype), video_frames], dim=2) + if cond_image is not None: + video_frames = torch.zeros(1, 3, frame_num-cond_image.shape[2], target_h, target_w, device=device, dtype=vae.dtype) + padding_frames_pixels_values = torch.concat([cond_image.to(device, vae.dtype), video_frames], dim=2) - # encode - vae.to(device) - y = vae.encode(padding_frames_pixels_values, device=device, tiled=tiled_vae).to(dtype) - cur_motion_frames_latent_num = int(1 + (cur_motion_frames_num-1) // 4) + # encode + vae.to(device) + y = vae.encode(padding_frames_pixels_values, device=device, tiled=tiled_vae, pbar=False).to(dtype) - if mode == "multitalk": - latent_motion_frames = y[:, :, :cur_motion_frames_latent_num][0] # C T H W - else: - if is_first_clip: - latent_motion_frames = vae.encode(cond_image.to(device, vae.dtype), device=device, tiled=tiled_vae).to(dtype) + if mode == "multitalk": + latent_motion_frames = y[:, :, :cur_motion_frames_latent_num][0] # C T H W else: - latent_motion_frames = vae.encode(cond_frame.to(device, vae.dtype), device=device, tiled=tiled_vae).to(dtype) - latent_motion_frames = latent_motion_frames[0] - vae.to(offload_device) - - y = torch.concat([msk, y], dim=1) # B 4+C T H W - mm.soft_empty_cache() + if is_first_clip: + latent_motion_frames = vae.encode(cond_image.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) + else: + latent_motion_frames = vae.encode(cond_frame.to(device, vae.dtype), device=device, tiled=tiled_vae, pbar=False).to(dtype) + latent_motion_frames = latent_motion_frames[0] + vae.to(offload_device) + y = torch.concat([msk, y], dim=1).squeeze(0) # 4+C T H W + mm.soft_empty_cache() + else: + y = None + latent_motion_frames = noise[:, :1] if scheduler == "multitalk": timesteps = list(np.linspace(1000, 1, steps, dtype=np.float32)) @@ -3235,6 +3303,24 @@ class WanVideoSampler: else: sample_scheduler, timesteps = get_scheduler(scheduler, steps, shift, device, transformer.dim, flowedit_args, denoise_strength, sigmas=sigmas) + steps = len(timesteps) + if end_step != -1 and start_step >= end_step: + raise ValueError("start_step must be less than end_step") + if denoise_strength < 1.0: + if start_step != 0: + raise ValueError("start_step must be 0 when denoise_strength is used") + start_step = steps - int(steps * denoise_strength) - 1 + if (end_step != -1 or end_step >= steps): + timesteps = timesteps[:end_step] + sample_scheduler.sigmas = sample_scheduler.sigmas[:end_step+1] + if start_step > 0: + timesteps = timesteps[start_step:] + sample_scheduler.sigmas = sample_scheduler.sigmas[start_step:] + + if sample_scheduler is not None: + if hasattr(sample_scheduler, 'timesteps'): + sample_scheduler.timesteps = timesteps + transformed_timesteps = [] for t in timesteps: t_tensor = torch.tensor([t.item()], device=device) @@ -3287,8 +3373,16 @@ class WanVideoSampler: else: transformer.to(device) - comfy_pbar = ProgressBar(len(timesteps)-1) - for i in tqdm(range(len(timesteps)-1)): + # Use the appropriate prompt for this section + if len(text_embeds["prompt_embeds"]) > 1: + prompt_index = min(iteration_count, len(text_embeds["prompt_embeds"]) - 1) + positive = [text_embeds["prompt_embeds"][prompt_index]] + log.info(f"Using prompt index: {prompt_index}") + else: + positive = text_embeds["prompt_embeds"] + + sampling_pbar = tqdm(total=len(timesteps)-1, desc=f"Sampling audio indices {audio_start_idx}-{audio_end_idx}", position=0, leave=True) + for i in range(len(timesteps)-1): timestep = timesteps[i] latent_model_input = latent.to(device) if mode == "infinitetalk": @@ -3297,14 +3391,18 @@ class WanVideoSampler: noise_pred, self.cache_state = predict_with_cfg( latent_model_input, cfg[idx], - text_embeds["prompt_embeds"], + positive, text_embeds["negative_prompt_embeds"], - timestep, idx, y.squeeze(0), clip_embeds, control_latents, vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, + timestep, idx, y, clip_embeds, control_latents, window_vace_data, unianim_data, audio_proj, control_camera_latents, add_cond, cache_state=self.cache_state, multitalk_audio_embeds=audio_embs) + sampling_pbar.update(1) + if callback is not None: callback_latent = (latent_model_input.to(device) - noise_pred.to(device) * t.to(device) / 1000).detach().permute(1,0,2,3) - callback(iteration_count, callback_latent, None, estimated_iterations) + callback(step_iteration_count, callback_latent, None, estimated_iterations*(len(timesteps)-1)) + + step_iteration_count += 1 # update latent if scheduler == "multitalk": @@ -3314,13 +3412,22 @@ class WanVideoSampler: latent = latent + noise_pred * dt[:, None, None, None] else: latent = latent.to(intermediate_device) - temp_x0 = sample_scheduler.step( noise_pred.unsqueeze(0), timestep, latent.unsqueeze(0), **scheduler_step_args)[0] latent = temp_x0.squeeze(0) + + # differential diffusion inpaint + if masks is not None: + if idx < len(timesteps) - 1: + noise_timestep = timesteps[idx+1] + image_latent = sample_scheduler.scale_noise( + original_image, torch.tensor([noise_timestep]), noise.to(device) + ) + mask = masks[idx].to(latent) + latent = image_latent * mask + latent * (1-mask) # injecting motion frames if not is_first_clip and mode == "multitalk": @@ -3334,13 +3441,14 @@ class WanVideoSampler: x0 = latent.to(device) del latent_model_input, timestep - comfy_pbar.update(1) if offload: offload_transformer(transformer) vae.to(device) - videos = vae.decode(x0.unsqueeze(0).to(vae.dtype), device=device, tiled=tiled_vae) + videos = vae.decode(x0.unsqueeze(0).to(vae.dtype), device=device, tiled=tiled_vae, pbar=False) vae.to(offload_device) + + sampling_pbar.close() # cache generated samples videos = torch.stack(videos).cpu() # B C T H W @@ -3411,9 +3519,7 @@ class WanVideoSampler: del noise, latent if force_offload: if not model["auto_cpu_offload"]: - transformer.to(offload_device) - mm.soft_empty_cache() - gc.collect() + offload_transformer(transformer) try: print_memory(device) torch.cuda.reset_peak_memory_stats(device) @@ -3486,6 +3592,25 @@ class WanVideoSampler: latent_backwards = torch.flip(latent_backwards, dims=[1]) latent = latent * 0.5 + latent_backwards * 0.5 + #InfiniteTalk first frame handling + if (extra_latents is not None + and not multitalk_sampling + and transformer.multitalk_model_type=="InfiniteTalk"): + for entry in extra_latents: + add_index = entry["index"] + num_extra_frames = entry["samples"].shape[2] + latent[:, add_index:add_index+num_extra_frames] = entry["samples"].to(latent) + + # differential diffusion inpaint + if masks is not None: + if idx < len(timesteps) - 1: + noise_timestep = timesteps[idx+1] + image_latent = sample_scheduler.scale_noise( + original_image, torch.tensor([noise_timestep]), noise.to(device) + ) + mask = masks[idx].to(latent) + latent = image_latent * mask + latent * (1-mask) + if freeinit_args is not None: current_latent = latent.clone() diff --git a/nodes_model_loading.py b/nodes_model_loading.py index aaa5209..3432b54 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -503,7 +503,7 @@ class WanVideoVACEModelSelect: def INPUT_TYPES(s): return { "required": { - "vace_model": (folder_paths.get_filename_list("diffusion_models"), {"tooltip": "These models are loaded from the 'ComfyUI/models/diffusion_models' VACE model to use when not using model that has it included"}), + "vace_model": (folder_paths.get_filename_list("unet_gguf") + folder_paths.get_filename_list("diffusion_models"), {"tooltip": "These models are loaded from the 'ComfyUI/models/diffusion_models' VACE model to use when not using model that has it included"}), }, } @@ -1021,15 +1021,9 @@ class WanVideoModelLoader: if extra_model is not None: if gguf: - if not extra_model["path"].endswith(".gguf"): - raise ValueError("With GGUF main model the extra model must also be a GGUF quantized, if the main model already has extra included, you can disconnect the extra model loader") - from diffusers.models.model_loading_utils import load_gguf_checkpoint - extra_sd = load_gguf_checkpoint(extra_model["path"]) - new_keys = {} - for k, v in extra_sd.items(): - if "vace" in k: - new_keys[k] = v - extra_sd = new_keys + if not vace_model["path"].endswith(".gguf"): + raise ValueError("With GGUF main model the VACE module must also be a GGUF quantized, if the main model already has VACE included, you can disconnect the VACE module loader") + vace_sd = load_gguf_checkpoint(vace_model["path"]) else: extra_sd = load_torch_file(extra_model["path"], device=transformer_load_device, safe_load=True) sd.update(extra_sd) @@ -1209,6 +1203,7 @@ class WanVideoModelLoader: if multitalk_model is not None: if multitalk_model["is_gguf"] and not gguf: raise ValueError("Multitalk/InfiniteTalk model is a GGUF model, main model also has to be a GGUF model.") + multitalk_model_type = multitalk_model.get("model_type", "MultiTalk") # init audio module from .multitalk.multitalk import SingleStreamMultiAttention from .wanvideo.modules.model import WanRMSNorm, WanLayerNorm @@ -1228,8 +1223,9 @@ class WanVideoModelLoader: attention_mode=attention_mode, ) block.norm_x = WanLayerNorm(dim, transformer.eps, elementwise_affine=True) if norm_input_visual else nn.Identity() - log.info("MultiTalk model detected, patching model...") + log.info(f"{multitalk_model_type} detected, patching model...") transformer.audio_proj = multitalk_model["proj_model"] + transformer.multitalk_model_type = multitalk_model_type sd.update(multitalk_model["sd"]) # Additional cond latents diff --git a/uni3c/nodes.py b/uni3c/nodes.py index 3720356..064b6dc 100644 --- a/uni3c/nodes.py +++ b/uni3c/nodes.py @@ -21,7 +21,7 @@ class WanVideoUni3C_ControlnetLoader: "model": (folder_paths.get_filename_list("controlnet"), {"tooltip": "These models are loaded from the 'ComfyUI/models/controlnet' -folder",}), "base_precision": (["fp32", "bf16", "fp16"], {"default": "fp16"}), - "quantization": (['disabled', 'fp8_e4m3fn', 'fp8_e4m3fn_fast', 'fp8_e5m2', 'fp8_e4m3fn_fast_no_ffn'], {"default": 'disabled', "tooltip": "optional quantization method"}), + "quantization": (['disabled', 'fp8_e4m3fn', 'fp8_e5m2'], {"default": 'disabled', "tooltip": "optional quantization method"}), "load_device": (["main_device", "offload_device"], {"default": "main_device", "tooltip": "Initial device to load the model to, NOT recommended with the larger models unless you have 48GB+ VRAM"}), "attention_mode": ([ "sdpa", @@ -138,12 +138,12 @@ class WanVideoUni3C_embeds: def INPUT_TYPES(s): return {"required": { "controlnet": ("WANVIDEOCONTROLNET",), - "render_latent": ("LATENT",), "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001}), "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Start percent of the steps to apply the controlnet"}), "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "End percent of the steps to apply the controlnet"}), }, "optional": { + "render_latent": ("LATENT",), "render_mask": ("MASK", {"tooltip": "NOT IMPLEMENTED!"}), }, } @@ -153,16 +153,16 @@ class WanVideoUni3C_embeds: FUNCTION = "process" CATEGORY = "WanVideoWrapper" - def process(self, controlnet, render_latent, strength, start_percent, end_percent, render_mask=None): + def process(self, controlnet, strength, start_percent, end_percent, render_latent=None, render_mask=None): device = mm.get_torch_device() - latent_mask = None - - latents = render_latent["samples"] - nframe = latents.shape[2] * 4 - height = latents.shape[3] * 8 - width = latents.shape[4] * 8 + latent_mask = latents = None + if render_latent is not None: + latents = render_latent["samples"] + # nframe = latents.shape[2] * 4 + # height = latents.shape[3] * 8 + # width = latents.shape[4] * 8 if render_mask is not None: raise NotImplementedError("render_mask is not implemented at this time") @@ -224,7 +224,7 @@ class WanVideoUni3C_embeds: "controlnet_weight": strength, "start": start_percent, "end": end_percent, - "render_latent": latents.to(device), + "render_latent": latents, "render_mask": latent_mask, "camera_embedding": None } diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index 07ea579..74b862b 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -1305,6 +1305,8 @@ class WanModel(torch.nn.Module): self.video_attention_split_steps = [] self.lora_scheduling_enabled = False + self.multitalk_model_type = "none" + # embeddings self.patch_embedding = nn.Conv3d( in_dim, dim, kernel_size=patch_size, stride=patch_size) @@ -1666,7 +1668,7 @@ class WanModel(torch.nn.Module): #uni3c controlnet if pcd_data is not None: hidden_states = x[0].unsqueeze(0).clone().float() - render_latent = torch.cat([hidden_states[:, :20], pcd_data["render_latent"]], dim=1) + render_latent = torch.cat([hidden_states[:, :20], pcd_data["render_latent"].to(x[0].dtype)], dim=1) # embeddings if control_lora_enabled: diff --git a/wanvideo/wan_video_vae.py b/wanvideo/wan_video_vae.py index 7ac9a05..2f55bc4 100644 --- a/wanvideo/wan_video_vae.py +++ b/wanvideo/wan_video_vae.py @@ -1019,12 +1019,13 @@ class VideoVAE_(nn.Module): return mu - def encode(self, x): + def encode(self, x, pbar=True): self.clear_cache() ## cache - pbar = ProgressBar(x.shape[2]) t = x.shape[2] iter_ = 1 + (t - 1) // 4 + if pbar: + pbar = ProgressBar(iter_) for i in range(iter_): self._enc_conv_idx = [0] @@ -1037,10 +1038,13 @@ class VideoVAE_(nn.Module): feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) out = torch.cat([out, out_], 2) - pbar.update(iter_) + if pbar: + pbar.update(1) mu = self.conv1(out).chunk(2, dim=1)[0] mu = (mu - self.mean.to(mu)) * self.inv_std.to(mu) + if pbar: + pbar.update_absolute(0) return mu @@ -1076,14 +1080,13 @@ class VideoVAE_(nn.Module): - def decode(self, z): + def decode(self, z, pbar=True): self.clear_cache() # z: [b,c,t,h,w] - pbar = ProgressBar(z.shape[2]) - z = z / self.inv_std.to(z) + self.mean.to(z) - iter_ = z.shape[2] + if pbar: + pbar = ProgressBar(iter_) x = self.conv2(z) for i in range(iter_): self._conv_idx = [0] @@ -1096,7 +1099,11 @@ class VideoVAE_(nn.Module): feat_cache=self._feat_map, feat_idx=self._conv_idx) out = torch.cat([out, out_], 2) # may add tensor offload - pbar.update(1) + + if pbar: + pbar.update(1) + if pbar: + pbar.update_absolute(0) return out def reparameterize(self, mu, log_var): @@ -1167,7 +1174,7 @@ class WanVideoVAE(nn.Module): return mask - def tiled_decode(self, hidden_states, device, tile_size, tile_stride): + def tiled_decode(self, hidden_states, device, tile_size, tile_stride, pbar=True): _, _, T, H, W = hidden_states.shape size_h, size_w = tile_size stride_h, stride_w = tile_stride @@ -1187,8 +1194,8 @@ class WanVideoVAE(nn.Module): out_T = T * 4 - 3 weight = torch.zeros((1, 1, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) values = torch.zeros((1, 3, out_T, H * self.upsampling_factor, W * self.upsampling_factor), dtype=hidden_states.dtype, device=data_device) - - pbar = ProgressBar(len(tasks)) + if pbar: + pbar = ProgressBar(len(tasks)) for h, h_, w, w_ in tqdm(tasks, desc="VAE decoding"): hidden_states_batch = hidden_states[:, :, :, h:h_, w:w_].to(computation_device) hidden_states_batch = self.model.decode(hidden_states_batch).to(data_device) @@ -1215,13 +1222,14 @@ class WanVideoVAE(nn.Module): target_h: target_h + hidden_states_batch.shape[3], target_w: target_w + hidden_states_batch.shape[4], ] += mask - pbar.update(1) + if pbar: + pbar.update(1) values = values / weight values = values.float().clamp_(-1, 1) return values - def tiled_encode(self, video, device, tile_size, tile_stride, end_=False): + def tiled_encode(self, video, device, tile_size, tile_stride, end_=False, pbar=True): _, _, T, H, W = video.shape if tile_size is None and tile_stride is None: @@ -1248,8 +1256,8 @@ class WanVideoVAE(nn.Module): out_T += 1 weight = torch.zeros((1, 1, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) values = torch.zeros((1, self.z_dim, out_T, H // self.upsampling_factor, W // self.upsampling_factor), dtype=video.dtype, device=data_device) - - pbar = ProgressBar(len(tasks)) + if pbar: + pbar = ProgressBar(len(tasks)) for h, h_, w, w_ in tqdm(tasks, desc="VAE encoding"): hidden_states_batch = video[:, :, :, h:h_, w:w_].to(computation_device) if end_: @@ -1279,21 +1287,22 @@ class WanVideoVAE(nn.Module): target_h: target_h + hidden_states_batch.shape[3], target_w: target_w + hidden_states_batch.shape[4], ] += mask - pbar.update(1) + if pbar: + pbar.update(1) values = values / weight values = values.float() return values - def single_encode(self, video, device): + def single_encode(self, video, device, pbar=True): video = video.to(device) - x = self.model.encode(video) + x = self.model.encode(video, pbar=pbar) return x.float() - def single_decode(self, hidden_state, device): + def single_decode(self, hidden_state, device, pbar=True): hidden_state = hidden_state.to(device) - video = self.model.decode(hidden_state) + video = self.model.decode(hidden_state, pbar=pbar) return video def double_encode(self, video, device): @@ -1308,36 +1317,36 @@ class WanVideoVAE(nn.Module): video = self.model.decode_2(hidden_state) return video - def encode(self, videos, device, tiled=False,end_=False, tile_size=None, tile_stride=None): + def encode(self, videos, device, tiled=False,end_=False, tile_size=None, tile_stride=None, pbar=True): videos = [video.to("cpu") for video in videos] hidden_states = [] for video in videos: video = video.unsqueeze(0) if tiled: - hidden_state = self.tiled_encode(video, device, tile_size, tile_stride, end_=end_) + hidden_state = self.tiled_encode(video, device, tile_size, tile_stride, end_=end_, pbar=pbar) else: if end_: hidden_state = self.double_encode(video, device) else: - hidden_state = self.single_encode(video, device) + hidden_state = self.single_encode(video, device, pbar=pbar) hidden_state = hidden_state.squeeze(0) hidden_states.append(hidden_state) hidden_states = torch.stack(hidden_states) return hidden_states - def decode(self, hidden_states, device, tiled=False, end_=False, tile_size=(34, 34), tile_stride=(18, 16)): + def decode(self, hidden_states, device, tiled=False, end_=False, tile_size=(34, 34), tile_stride=(18, 16), pbar=True): hidden_states = [hidden_state.to("cpu") for hidden_state in hidden_states] videos = [] for hidden_state in hidden_states: hidden_state = hidden_state.unsqueeze(0) if tiled: - video = self.tiled_decode(hidden_state, device, tile_size, tile_stride) + video = self.tiled_decode(hidden_state, device, tile_size, tile_stride, pbar=pbar) else: if end_: video = self.double_decode(hidden_state, device) else: - video = self.single_decode(hidden_state, device) + video = self.single_decode(hidden_state, device, pbar=pbar) video = video.squeeze(0) videos.append(video) return videos @@ -1397,11 +1406,13 @@ class VideoVAE38_(VideoVAE_): attn_scales, self.temperal_upsample, dropout) - def encode(self, x): + def encode(self, x, pbar=True): self.clear_cache() x = patchify(x, patch_size=2) t = x.shape[2] iter_ = 1 + (t - 1) // 4 + if pbar: + pbar = ProgressBar(iter_) for i in range(iter_): self._enc_conv_idx = [0] if i == 0: @@ -1413,6 +1424,8 @@ class VideoVAE38_(VideoVAE_): feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) out = torch.cat([out, out_], 2) + if pbar: + pbar.update(1) mu = self.conv1(out).chunk(2, dim=1)[0] mu = (mu - self.mean.to(mu)) * self.inv_std.to(mu) @@ -1421,12 +1434,13 @@ class VideoVAE38_(VideoVAE_): return mu - def decode(self, z): + def decode(self, z, pbar=True): self.clear_cache() - z = z / self.inv_std.to(z) + self.mean.to(z) iter_ = z.shape[2] + if pbar: + pbar = ProgressBar(iter_) x = self.conv2(z) for i in range(iter_): self._conv_idx = [0] @@ -1440,6 +1454,8 @@ class VideoVAE38_(VideoVAE_): feat_cache=self._feat_map, feat_idx=self._conv_idx) out = torch.cat([out, out_], 2) + if pbar: + pbar.update(1) out = unpatchify(out, patch_size=2) self.clear_cache() return out