diff --git a/.gitignore b/.gitignore index 5f07abb..d15ec35 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ *.zip +.tracking diff --git a/examples/example_start_end_frames.json b/examples/example_start_end_frames.json index 3992bd9..ea160d2 100644 --- a/examples/example_start_end_frames.json +++ b/examples/example_start_end_frames.json @@ -1,8 +1,8 @@ { "id": "f7892458-72a2-4d5b-a285-afa7a53cbfb9", "revision": 0, - "last_node_id": 80, - "last_link_id": 208, + "last_node_id": 86, + "last_link_id": 214, "nodes": [ { "id": 17, @@ -37,114 +37,6 @@ "clip_vision_h.safetensors" ] }, - { - "id": 13, - "type": "CLIPLoader", - "pos": [ - 780, - -1570 - ], - "size": [ - 315, - 98 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 12, - 14 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "CLIPLoader" - }, - "widgets_values": [ - "umt5_xxl_fp8_e4m3fn_scaled.safetensors", - "wan", - "default" - ] - }, - { - "id": 19, - "type": "UnetLoaderGGUF", - "pos": [ - 650, - -1820 - ], - "size": [ - 315, - 58 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 157 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-GGUF", - "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", - "Node name for S&R": "UnetLoaderGGUF" - }, - "widgets_values": [ - "wan2.1-i2v-14b-480p-Q8_0.gguf" - ] - }, - { - "id": 12, - "type": "CLIPTextEncode", - "pos": [ - 1270, - -1560 - ], - "size": [ - 400, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 12 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 179 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "" - ] - }, { "id": 14, "type": "CLIPTextEncode", @@ -157,7 +49,7 @@ 200 ], "flags": {}, - "order": 8, + "order": 12, "mode": 0, "inputs": [ { @@ -196,7 +88,7 @@ 58 ], "flags": {}, - "order": 3, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -218,77 +110,6 @@ "wan_2.1_vae.safetensors" ] }, - { - "id": 58, - "type": "UNETLoader", - "pos": [ - 650, - -1950 - ], - "size": [ - 315, - 82 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": null - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "UNETLoader" - }, - "widgets_values": [ - "wan2.1_i2v_720p_14B_fp16.safetensors", - "fp8_e4m3fn_fast" - ] - }, - { - "id": 48, - "type": "ModelSamplingSD3", - "pos": [ - 1000, - -1820 - ], - "size": [ - 210, - 58 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 157 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 158 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "ModelSamplingSD3" - }, - "widgets_values": [ - 8.000000000000002 - ] - }, { "id": 46, "type": "ImageNoiseAugmentation", @@ -301,7 +122,7 @@ 106 ], "flags": {}, - "order": 11, + "order": 10, "mode": 0, "inputs": [ { @@ -342,7 +163,7 @@ 106 ], "flags": {}, - "order": 10, + "order": 9, "mode": 0, "inputs": [ { @@ -372,213 +193,6 @@ "fixed" ] }, - { - "id": 47, - "type": "SkipLayerGuidanceWanVideo", - "pos": [ - 1230, - -1820 - ], - "size": [ - 235.1999969482422, - 106 - ], - "flags": {}, - "order": 12, - "mode": 4, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 158 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 156 - ] - } - ], - "properties": { - "cnr_id": "comfyui-kjnodes", - "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", - "Node name for S&R": "SkipLayerGuidanceWanVideo" - }, - "widgets_values": [ - "9, 10", - 0.20000000000000004, - 1 - ] - }, - { - "id": 50, - "type": "WanVideoEnhanceAVideoKJ", - "pos": [ - 1610, - -1690 - ], - "size": [ - 277.20001220703125, - 80 - ], - "flags": {}, - "order": 16, - "mode": 4, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 162 - }, - { - "name": "latent", - "type": "LATENT", - "link": 188 - } - ], - "outputs": [ - { - "name": "model", - "type": "MODEL", - "links": [ - 172 - ] - } - ], - "properties": { - "cnr_id": "comfyui-kjnodes", - "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", - "Node name for S&R": "WanVideoEnhanceAVideoKJ" - }, - "widgets_values": [ - 2.0000000000000004 - ] - }, - { - "id": 55, - "type": "TorchCompileModelWanVideo", - "pos": [ - 2200, - -1890 - ], - "size": [ - 315, - 178 - ], - "flags": {}, - "order": 18, - "mode": 4, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 175 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 176 - ] - } - ], - "properties": { - "cnr_id": "comfyui-kjnodes", - "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", - "Node name for S&R": "TorchCompileModelWanVideo" - }, - "widgets_values": [ - "inductor", - false, - "default", - false, - 64, - true - ] - }, - { - "id": 59, - "type": "LoadImage", - "pos": [ - 490, - -1130 - ], - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 189 - ] - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "start.jpg", - "image" - ] - }, - { - "id": 60, - "type": "LoadImage", - "pos": [ - 490, - -740 - ], - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 190 - ] - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.27", - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "end.jpg", - "image" - ] - }, { "id": 16, "type": "CLIPVisionEncode", @@ -591,7 +205,7 @@ 78 ], "flags": {}, - "order": 13, + "order": 14, "mode": 0, "inputs": [ { @@ -635,7 +249,7 @@ 46 ], "flags": {}, - "order": 20, + "order": 23, "mode": 0, "inputs": [ { @@ -665,88 +279,6 @@ }, "widgets_values": [] }, - { - "id": 28, - "type": "PathchSageAttentionKJ", - "pos": [ - 1490, - -1820 - ], - "size": [ - 210, - 58 - ], - "flags": {}, - "order": 14, - "mode": 4, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 156 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 162 - ] - } - ], - "properties": { - "cnr_id": "comfyui-kjnodes", - "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", - "Node name for S&R": "PathchSageAttentionKJ" - }, - "widgets_values": [ - "auto" - ] - }, - { - "id": 54, - "type": "WanVideoTeaCacheKJ", - "pos": [ - 1830, - -1890 - ], - "size": [ - 340.20001220703125, - 154 - ], - "flags": {}, - "order": 17, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 172 - } - ], - "outputs": [ - { - "name": "model", - "type": "MODEL", - "links": [ - 175 - ] - } - ], - "properties": { - "cnr_id": "comfyui-kjnodes", - "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", - "Node name for S&R": "WanVideoTeaCacheKJ" - }, - "widgets_values": [ - 0.23000000000000007, - 0.1, - 1, - "offload_device", - "i2v_480" - ] - }, { "id": 78, "type": "GetImagesFromBatchRanged_F2", @@ -759,7 +291,7 @@ 82 ], "flags": {}, - "order": 21, + "order": 24, "mode": 0, "inputs": [ { @@ -799,7 +331,7 @@ 206 ], "flags": {}, - "order": 15, + "order": 16, "mode": 0, "inputs": [ { @@ -882,7 +414,7 @@ 571 ], "flags": {}, - "order": 19, + "order": 22, "mode": 0, "inputs": [ { @@ -921,7 +453,7 @@ "Node name for S&R": "KSampler" }, "widgets_values": [ - 81762998615878, + 928474147271562, "randomize", 20, 4, @@ -943,7 +475,7 @@ 542.7587890625 ], "flags": {}, - "order": 22, + "order": 25, "mode": 0, "inputs": [ { @@ -997,16 +529,591 @@ "hidden": false, "paused": false, "params": { - "filename": "Wan_00017.mp4", + "filename": "Wan_00022.mp4", "subfolder": "", "type": "output", "format": "video/h264-mp4", "frame_rate": 16, - "workflow": "Wan_00017.png", - "fullpath": "D:\\ai\\ComfyUI_windows_portable\\ComfyUI\\output\\Wan_00017.mp4" + "workflow": "Wan_00022.png", + "fullpath": "D:\\ai\\ComfyUI_windows_portable\\ComfyUI\\output\\Wan_00022.mp4" } } } + }, + { + "id": 48, + "type": "ModelSamplingSD3", + "pos": [ + 1000, + -1940 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 212 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 158 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "ModelSamplingSD3" + }, + "widgets_values": [ + 8.000000000000002 + ] + }, + { + "id": 28, + "type": "PathchSageAttentionKJ", + "pos": [ + 1270, + -1820 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 17, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 156 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 213 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", + "Node name for S&R": "PathchSageAttentionKJ" + }, + "widgets_values": [ + "auto" + ] + }, + { + "id": 84, + "type": "PatchModelPatcherOrder", + "pos": [ + 1270, + -1960 + ], + "size": [ + 270, + 82 + ], + "flags": { + "collapsed": false + }, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 213 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 214 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "52c2e31a903fec2dd654fb614ea82ba2757d5028", + "Node name for S&R": "PatchModelPatcherOrder" + }, + "widgets_values": [ + "weight_patch_first", + "disabled" + ] + }, + { + "id": 50, + "type": "WanVideoEnhanceAVideoKJ", + "pos": [ + 1560, + -1820 + ], + "size": [ + 277.20001220703125, + 80 + ], + "flags": {}, + "order": 19, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 214 + }, + { + "name": "latent", + "type": "LATENT", + "link": 188 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 172 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", + "Node name for S&R": "WanVideoEnhanceAVideoKJ" + }, + "widgets_values": [ + 2.0000000000000004 + ] + }, + { + "id": 54, + "type": "WanVideoTeaCacheKJ", + "pos": [ + 1900, + -1890 + ], + "size": [ + 340.20001220703125, + 154 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 172 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 175 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", + "Node name for S&R": "WanVideoTeaCacheKJ" + }, + "widgets_values": [ + 0.23000000000000007, + 0.1, + 1, + "offload_device", + "i2v_480" + ] + }, + { + "id": 55, + "type": "TorchCompileModelWanVideo", + "pos": [ + 2280, + -1890 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 21, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 175 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 176 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", + "Node name for S&R": "TorchCompileModelWanVideo" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true + ] + }, + { + "id": 19, + "type": "UnetLoaderGGUF", + "pos": [ + 210, + -1820 + ], + "size": [ + 360, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 211 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-GGUF", + "ver": "bc5223b0e37e053dbec2ea5e5f52c2fd4b8f712a", + "Node name for S&R": "UnetLoaderGGUF" + }, + "widgets_values": [ + "wan2.1-i2v-14b-480p-Q8_0.gguf" + ] + }, + { + "id": 58, + "type": "UNETLoader", + "pos": [ + 200, + -1980 + ], + "size": [ + 380, + 82 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "UNETLoader" + }, + "widgets_values": [ + "wan2.1_i2v_720p_14B_fp16.safetensors", + "fp8_e4m3fn_fast" + ] + }, + { + "id": 47, + "type": "SkipLayerGuidanceWanVideo", + "pos": [ + 1000, + -1820 + ], + "size": [ + 235.1999969482422, + 106 + ], + "flags": {}, + "order": 15, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 158 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 156 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", + "Node name for S&R": "SkipLayerGuidanceWanVideo" + }, + "widgets_values": [ + "9, 10", + 0.20000000000000004, + 1 + ] + }, + { + "id": 59, + "type": "LoadImage", + "pos": [ + 490, + -1130 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 189 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "start.jpg", + "image", + "" + ] + }, + { + "id": 60, + "type": "LoadImage", + "pos": [ + 490, + -740 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 190 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "end.jpg", + "image", + "" + ] + }, + { + "id": 83, + "type": "LoraLoaderModelOnly", + "pos": [ + 640, + -1820 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 211 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 212 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "LoraLoaderModelOnly" + }, + "widgets_values": [ + "gun_epoch20.safetensors", + 1 + ] + }, + { + "id": 12, + "type": "CLIPTextEncode", + "pos": [ + 1270, + -1560 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 12 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 179 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "p5lls g4un pulls a cat starts shooting with a gun, after that puts down the gun." + ] + }, + { + "id": 13, + "type": "CLIPLoader", + "pos": [ + 730, + -1570 + ], + "size": [ + 370, + 100 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 12, + 14 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.27", + "Node name for S&R": "CLIPLoader" + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 86, + "type": "Note", + "pos": [ + -40, + -1690 + ], + "size": [ + 600, + 320 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "MODEL DOWNLOAD URL", + "properties": {}, + "widgets_values": [ + "MODEL (select one):\nhttps://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main\nhttps://huggingface.co/city96/Wan2.1-I2V-14B-720P-gguf/tree/main\nhttps://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models\n\n\nCLIP (select one):\nhttps://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp16.safetensors?download=true\nhttps://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors?download=true\n\n\nCLIP VISION:\nhttps://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/clip_vision/clip_vision_h.safetensors?download=true\n\n\nVAE:\nhttps://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/vae/wan_2.1_vae.safetensors?download=true\n\n\nLORA (example):\nhttps://civitai.com/models/1344352/gun-shooting-wan21-i2v-lora" + ], + "shape": 2 } ], "links": [ @@ -1058,14 +1165,6 @@ 0, "MODEL" ], - [ - 157, - 19, - 0, - 48, - 0, - "MODEL" - ], [ 158, 48, @@ -1074,14 +1173,6 @@ 0, "MODEL" ], - [ - 162, - 28, - 0, - 50, - 0, - "MODEL" - ], [ 172, 50, @@ -1225,16 +1316,48 @@ 38, 0, "IMAGE" + ], + [ + 211, + 19, + 0, + 83, + 0, + "MODEL" + ], + [ + 212, + 83, + 0, + 48, + 0, + "MODEL" + ], + [ + 213, + 28, + 0, + 84, + 0, + "MODEL" + ], + [ + 214, + 84, + 0, + 50, + 0, + "MODEL" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.8830511255241463, + "scale": 0.9713562380765651, "offset": [ - -894.7564176246019, - 1937.3825376682698 + 103.42137242231036, + 2189.838184971042 ] }, "VHS_latentpreview": true, diff --git a/nodes/nodes.py b/nodes/nodes.py index 4f1d398..463a738 100644 --- a/nodes/nodes.py +++ b/nodes/nodes.py @@ -147,7 +147,8 @@ class WanImageToVideo_F2: CATEGORY = "Flow2/Wan 2.1" def encode(self, positive, negative, vae, width, height, length, start_image=None, end_image=None, clip_vision_output=None): - device = mm.intermediate_device() + offload_device = mm.unet_offload_device() + device = offload_device valid_end_image = end_image is not None diff --git a/pyproject.toml b/pyproject.toml index 1ef984f..941ac5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ComfyUI-WanStartEndFramesNative" description = "Start and end frames video generation node that supports native ComfyUI." -version = "1.0.1" +version = "1.0.2" license = {file = "LICENSE"} dependencies = []