diff --git a/__init__.py b/__init__.py index fe65d44..5634205 100644 --- a/__init__.py +++ b/__init__.py @@ -3,15 +3,18 @@ from .recammaster.nodes import NODE_CLASS_MAPPINGS as RECAM_MASTER_NODE_CLASS_MA from .unianimate.nodes import NODE_CLASS_MAPPINGS as UNIANIMATE_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as UNIANIMATE_NODE_DISPLAY_NAME_MAPPINGS from .skyreels.nodes import NODE_CLASS_MAPPINGS as SKYREELS_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as SKYREELS_NODE_DISPLAY_NAME_MAPPINGS from .fantasytalking.nodes import NODE_CLASS_MAPPINGS as FANTASYTALKING_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as FANTASYTALKING_NODE_DISPLAY_NAME_MAPPINGS +from .fun_camera.nodes import NODE_CLASS_MAPPINGS as FUN_CAMERA_NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS as FUN_CAMERA_NODE_DISPLAY_NAME_MAPPINGS NODE_CLASS_MAPPINGS.update(RECAM_MASTER_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(UNIANIMATE_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(SKYREELS_NODE_CLASS_MAPPINGS) NODE_CLASS_MAPPINGS.update(FANTASYTALKING_NODE_CLASS_MAPPINGS) +NODE_CLASS_MAPPINGS.update(FUN_CAMERA_NODE_CLASS_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(RECAM_MASTER_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(UNIANIMATE_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(SKYREELS_NODE_DISPLAY_NAME_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(FANTASYTALKING_NODE_DISPLAY_NAME_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(FUN_CAMERA_NODE_DISPLAY_NAME_MAPPINGS) __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] \ No newline at end of file diff --git a/example_workflows/wanvideo_Fun_control_camera_example_01.json b/example_workflows/wanvideo_Fun_control_camera_example_01.json new file mode 100644 index 0000000..5e100e4 --- /dev/null +++ b/example_workflows/wanvideo_Fun_control_camera_example_01.json @@ -0,0 +1,1915 @@ +{ + "id": "206247b6-9fec-4ed2-8927-e4f388c674d4", + "revision": 0, + "last_node_id": 112, + "last_link_id": 171, + "nodes": [ + { + "id": 46, + "type": "WanVideoTextEmbedBridge", + "pos": [ + 1303.4276123046875, + 1086.20947265625 + ], + "size": [ + 315, + 46 + ], + "flags": {}, + "order": 27, + "mode": 2, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 54 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 55 + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": null + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoTextEmbedBridge" + }, + "widgets_values": [] + }, + { + "id": 50, + "type": "CLIPTextEncode", + "pos": [ + 853.4276123046875, + 1346.2091064453125 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 23, + "mode": 2, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 53 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 55 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 49, + "type": "CLIPTextEncode", + "pos": [ + 853.4276123046875, + 1096.20947265625 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 22, + "mode": 2, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 52 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 54 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "high quality nature video featuring a red panda balancing on a bamboo stem while a bird lands on it's head, on the background there is a waterfall" + ] + }, + { + "id": 42, + "type": "Note", + "pos": [ + -990, + -1030 + ], + "size": [ + 314.96246337890625, + 152.77333068847656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "Adjust the blocks to swap based on your VRAM, this is a tradeoff between speed and memory usage.\n\nAlternatively there's option to use VRAM management introduced in DiffSynt-Studios. This is usually slower, but saves even more VRAM compared to BlockSwap" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 51, + "type": "Note", + "pos": [ + 523.4271850585938, + 926.2090454101562 + ], + "size": [ + 253.16725158691406, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "You can also use native ComfyUI text encoding with these nodes instead of the original, the models are node specific and can't otherwise be mixed." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 44, + "type": "Note", + "pos": [ + -1030, + -1320 + ], + "size": [ + 303.0501403808594, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "If you have Triton installed, connect this for ~30% speed increase" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 35, + "type": "WanVideoTorchCompileSettings", + "pos": [ + -690, + -1320 + ], + "size": [ + 421.6000061035156, + 202 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "torch_compile_args", + "type": "WANCOMPILEARGS", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoTorchCompileSettings" + }, + "widgets_values": [ + "inductor", + false, + "default", + false, + 64, + true, + 128 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 36, + "type": "Note", + "pos": [ + -250, + -1280 + ], + "size": [ + 374.3061828613281, + 171.9547576904297 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "fp8_fast seems to cause huge quality degradation\n\nfp_16_fast enables \"Full FP16 Accmumulation in FP16 GEMMs\" feature available in the very latest pytorch nightly, this is around 20% speed boost. \n\nSageattn if you have it installed can be used for almost double inference speed" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "GetNode", + "pos": [ + -797.9415893554688, + -324.1277770996094 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 123 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 11, + "type": "LoadWanVideoT5TextEncoder", + "pos": [ + -153.58168029785156, + -752.5652465820312 + ], + "size": [ + 377.1661376953125, + 130 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "wan_t5_model", + "type": "WANTEXTENCODER", + "slot_index": 0, + "links": [ + 15 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "LoadWanVideoT5TextEncoder" + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "offload_device", + "disabled" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 38, + "type": "WanVideoVAELoader", + "pos": [ + -140.5011749267578, + -566.3262939453125 + ], + "size": [ + 372.7727966308594, + 82 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 103 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoVAELoader" + }, + "widgets_values": [ + "wanvideo\\Wan2_1_VAE_bf16.safetensors", + "bf16" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 83, + "type": "SetNode", + "pos": [ + 267.4593505859375, + -535.8521728515625 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "link": 103 + } + ], + "outputs": [ + { + "name": "*", + "type": "*", + "links": null + } + ], + "title": "Set_VAE", + "properties": { + "previousName": "VAE" + }, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 86, + "type": "GetNode", + "pos": [ + 1757.5093994140625, + -644.2652587890625 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "WANVAE", + "type": "WANVAE", + "links": [ + 106 + ] + } + ], + "title": "Get_VAE", + "properties": {}, + "widgets_values": [ + "VAE" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 89, + "type": "GetNode", + "pos": [ + 1738.6708984375, + -343.6011657714844 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 136 + ] + } + ], + "title": "Get_InputImage", + "properties": {}, + "widgets_values": [ + "InputImage" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 63, + "type": "WanVideoImageToVideoEncode", + "pos": [ + -800.4992065429688, + -261.5985107421875 + ], + "size": [ + 352.79998779296875, + 398 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 123 + }, + { + "name": "clip_embeds", + "shape": 7, + "type": "WANVIDIMAGE_CLIPEMBEDS", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 169 + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_embeds", + "shape": 7, + "type": "WANVIDIMAGE_EMBEDS", + "link": 153 + }, + { + "name": "temporal_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "extra_latents", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 137 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 138 + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 156 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 100 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoImageToVideoEncode" + }, + "widgets_values": [ + 832, + 480, + 81, + 0.030000000000000006, + 1, + 1, + true, + true + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 16, + "type": "WanVideoTextEncode", + "pos": [ + 706.5767822265625, + -377.3152160644531 + ], + "size": [ + 420.30511474609375, + 261.5306701660156 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "t5", + "type": "WANTEXTENCODER", + "link": 15 + }, + { + "name": "model_to_offload", + "shape": 7, + "type": "WANVIDEOMODEL", + "link": 79 + } + ], + "outputs": [ + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "slot_index": 0, + "links": [ + 30 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoTextEncode" + }, + "widgets_values": [ + "high quality video of an old man", + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + true + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 1745.494873046875, + -597.1070556640625 + ], + "size": [ + 315, + 174 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 106 + }, + { + "name": "samples", + "type": "LATENT", + "link": 33 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 135 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoDecode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 102, + "type": "CameraPoseVisualizer", + "pos": [ + -865.4253540039062, + 707.3152465820312 + ], + "size": [ + 315, + 202 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "cameractrl_poses", + "shape": 7, + "type": "CAMERACTRL_POSES", + "link": 145 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 146, + 147 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "c3dc82108a2a86c17094107ead61d63f8c76200e", + "Node name for S&R": "CameraPoseVisualizer" + }, + "widgets_values": [ + "", + 0.2, + 0.3, + 1, + false, + true, + false + ] + }, + { + "id": 52, + "type": "WanVideoTeaCache", + "pos": [ + 1307.6705322265625, + -787.4303588867188 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "teacache_args", + "type": "TEACACHEARGS", + "links": [ + 56 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoTeaCache" + }, + "widgets_values": [ + 0.08000000000000002, + 6, + -1, + "offload_device", + "true", + "e0" + ] + }, + { + "id": 90, + "type": "WanVideoExperimentalArgs", + "pos": [ + 947.8186645507812, + 317.6713562011719 + ], + "size": [ + 327.5999755859375, + 226 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "exp_args", + "type": "EXPERIMENTALARGS", + "links": [ + 114 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d5085d4e879b5c9d56e39f57e656255ce72e81fc", + "Node name for S&R": "WanVideoExperimentalArgs" + }, + "widgets_values": [ + "", + true, + false, + 0, + true, + 1, + 1.25, + 20 + ] + }, + { + "id": 105, + "type": "INTConstant", + "pos": [ + -1567.3739013671875, + 473.9447937011719 + ], + "size": [ + 210, + 58 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 156, + 164 + ] + } + ], + "title": "Frames", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "c3dc82108a2a86c17094107ead61d63f8c76200e", + "Node name for S&R": "INTConstant" + }, + "widgets_values": [ + 81 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 80, + "type": "Reroute", + "pos": [ + -85.4590835571289, + 246.93772888183594 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 100 + } + ], + "outputs": [ + { + "name": "", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 166 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 87, + "type": "ImageConcatMulti", + "pos": [ + 1735.2471923828125, + -292.62054443359375 + ], + "size": [ + 315, + 170 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 135 + }, + { + "name": "image_2", + "type": "IMAGE", + "link": 136 + }, + { + "name": "image_3", + "type": "IMAGE", + "link": 147 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 168 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "52c2e31a903fec2dd654fb614ea82ba2757d5028" + }, + "widgets_values": [ + 3, + "left", + true, + null + ] + }, + { + "id": 58, + "type": "LoadImage", + "pos": [ + -1700.562744140625, + -315.5335693359375 + ], + "size": [ + 413.10479736328125, + 498.3180847167969 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 132 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "oldman_upscaled.png", + "image" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 97, + "type": "ImageResizeKJv2", + "pos": [ + -1194.7802734375, + -218.37701416015625 + ], + "size": [ + 315, + 242 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 132 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 134 + ] + }, + { + "name": "width", + "type": "INT", + "links": [ + 137, + 154 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 138, + 155 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "c3dc82108a2a86c17094107ead61d63f8c76200e", + "Node name for S&R": "ImageResizeKJv2" + }, + "widgets_values": [ + 624, + 624, + "lanczos", + "crop", + "0, 0, 0", + "center", + 16 + ] + }, + { + "id": 99, + "type": "ADE_CameraPoseBasic", + "pos": [ + -1236.113037109375, + 554.797607421875 + ], + "size": [ + 310.9779968261719, + 106 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "prev_poses", + "shape": 7, + "type": "CAMERACTRL_POSES", + "link": null + }, + { + "name": "frame_length", + "type": "INT", + "widget": { + "name": "frame_length" + }, + "link": 164 + } + ], + "outputs": [ + { + "name": "CAMERACTRL_POSES", + "type": "CAMERACTRL_POSES", + "links": [ + 145, + 151 + ] + } + ], + "properties": { + "cnr_id": "comfyui-animatediff-evolved", + "ver": "3795f515b8faa1f4cfd9393a50cdcca299a68993", + "Node name for S&R": "ADE_CameraPoseBasic" + }, + "widgets_values": [ + "Zoom Out", + 0.10000000000000002, + 40 + ] + }, + { + "id": 103, + "type": "PreviewImage", + "pos": [ + -307.6606750488281, + 518.0542602539062 + ], + "size": [ + 554.8445434570312, + 491.0906066894531 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 146 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.29", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 98, + "type": "SetNode", + "pos": [ + -1021.9185180664062, + -287.0921630859375 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "link": 134 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 169 + ] + } + ], + "title": "Set_InputImage", + "properties": { + "previousName": "InputImage" + }, + "widgets_values": [ + "InputImage" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 104, + "type": "WanVideoFunCameraEmbeds", + "pos": [ + -712.9072875976562, + 503.0683288574219 + ], + "size": [ + 315, + 154 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "poses", + "type": "CAMERACTRL_POSES", + "link": 151 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 154 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 155 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 153 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "4a41fb0aaff6a9f3a8ba86906ada7d53be4f94d0", + "Node name for S&R": "WanVideoFunCameraEmbeds" + }, + "widgets_values": [ + 832, + 480, + 1, + 0, + 1 + ] + }, + { + "id": 27, + "type": "WanVideoSampler", + "pos": [ + 1315.2401123046875, + -401.48028564453125 + ], + "size": [ + 357.6112060546875, + 909.6112060546875 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "link": 29 + }, + { + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "link": 30 + }, + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 166 + }, + { + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": null + }, + { + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": null + }, + { + "name": "teacache_args", + "shape": 7, + "type": "TEACACHEARGS", + "link": 56 + }, + { + "name": "flowedit_args", + "shape": 7, + "type": "FLOWEDITARGS", + "link": null + }, + { + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": 114 + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null + }, + { + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS", + "link": null + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "slot_index": 0, + "links": [ + 33 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoSampler" + }, + "widgets_values": [ + 30, + 6, + 5, + 43, + "fixed", + true, + "unipc", + 0, + 1, + "", + "comfy", + "" + ] + }, + { + "id": 39, + "type": "WanVideoBlockSwap", + "pos": [ + -620, + -1030 + ], + "size": [ + 315, + 154 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "slot_index": 0, + "links": [] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 15, + false, + false, + true, + 0 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 68, + "type": "MarkdownNote", + "pos": [ + 159.35009765625, + -1353.13427734375 + ], + "size": [ + 652.1258544921875, + 234.3824005126953 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model download links", + "properties": {}, + "widgets_values": [ + "1.3B BF16:\n\n[https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control-Camera/blob/main/diffusion_pytorch_model.safetensors](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control-Camera/blob/main/diffusion_pytorch_model.safetensors)\n\n14B BF16:\n\n[https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control-Camera/blob/main/diffusion_pytorch_model.safetensors](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control-Camera/blob/main/diffusion_pytorch_model.safetensors)\n\n14B FP8:\n\n[https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan2_1-Wan2_1-Fun-V1_1-14B-Control-Camera_fp8_e4m3fn.safetensors](https://huggingface.co/Kijai/WanVideo_comfy/blob/main/Wan2_1-Wan2_1-Fun-V1_1-14B-Control-Camera_fp8_e4m3fn.safetensors)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 22, + "type": "WanVideoModelLoader", + "pos": [ + -260, + -1050 + ], + "size": [ + 477.4410095214844, + 254 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS", + "link": null + }, + { + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": null + }, + { + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS", + "link": null + }, + { + "name": "vace_model", + "shape": 7, + "type": "VACEPATH", + "link": null + }, + { + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "WANVIDEOMODEL", + "slot_index": 0, + "links": [ + 29, + 79 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "d20baf00247fd06553fdc9253e18732244e54172", + "Node name for S&R": "WanVideoModelLoader" + }, + "widgets_values": [ + "WanVideo\\Wan2.1-Fun-V1.1-1.3B-Control-Camera.safetensors", + "bf16", + "disabled", + "offload_device", + "sdpa" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 48, + "type": "CLIPLoader", + "pos": [ + 493.4272766113281, + 1096.20947265625 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 17, + "mode": 2, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 52, + 53 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.26", + "Node name for S&R": "CLIPLoader" + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 30, + "type": "VHS_VideoCombine", + "pos": [ + 2127.332275390625, + -314.9173583984375 + ], + "size": [ + 1245.8460693359375, + 725.5557250976562 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 168 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "WanVideoWrapper_FunControlCamera", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "trim_to_audio": false, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "WanVideoWrapper_FunControl_00318.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16, + "workflow": "WanVideoWrapper_FunControl_00318.png", + "fullpath": "N:\\AI\\ComfyUI\\output\\WanVideoWrapper_FunControl_00318.mp4" + } + } + } + } + ], + "links": [ + [ + 15, + 11, + 0, + 16, + 0, + "WANTEXTENCODER" + ], + [ + 29, + 22, + 0, + 27, + 0, + "WANVIDEOMODEL" + ], + [ + 30, + 16, + 0, + 27, + 1, + "WANVIDEOTEXTEMBEDS" + ], + [ + 33, + 27, + 0, + 28, + 1, + "LATENT" + ], + [ + 52, + 48, + 0, + 49, + 0, + "CLIP" + ], + [ + 53, + 48, + 0, + 50, + 0, + "CLIP" + ], + [ + 54, + 49, + 0, + 46, + 0, + "CONDITIONING" + ], + [ + 55, + 50, + 0, + 46, + 1, + "CONDITIONING" + ], + [ + 56, + 52, + 0, + 27, + 6, + "TEACACHEARGS" + ], + [ + 79, + 22, + 0, + 16, + 1, + "WANVIDEOMODEL" + ], + [ + 100, + 63, + 0, + 80, + 0, + "*" + ], + [ + 103, + 38, + 0, + 83, + 0, + "*" + ], + [ + 106, + 86, + 0, + 28, + 0, + "WANVAE" + ], + [ + 114, + 90, + 0, + 27, + 10, + "EXPERIMENTALARGS" + ], + [ + 123, + 85, + 0, + 63, + 0, + "WANVAE" + ], + [ + 132, + 58, + 0, + 97, + 0, + "IMAGE" + ], + [ + 134, + 97, + 0, + 98, + 0, + "*" + ], + [ + 135, + 28, + 0, + 87, + 0, + "IMAGE" + ], + [ + 136, + 89, + 0, + 87, + 1, + "IMAGE" + ], + [ + 137, + 97, + 1, + 63, + 7, + "INT" + ], + [ + 138, + 97, + 2, + 63, + 8, + "INT" + ], + [ + 145, + 99, + 0, + 102, + 0, + "CAMERACTRL_POSES" + ], + [ + 146, + 102, + 0, + 103, + 0, + "IMAGE" + ], + [ + 147, + 102, + 0, + 87, + 2, + "IMAGE" + ], + [ + 151, + 99, + 0, + 104, + 0, + "CAMERACTRL_POSES" + ], + [ + 153, + 104, + 0, + 63, + 4, + "WANVIDIMAGE_EMBEDS" + ], + [ + 154, + 97, + 1, + 104, + 1, + "INT" + ], + [ + 155, + 97, + 2, + 104, + 2, + "INT" + ], + [ + 156, + 105, + 0, + 63, + 9, + "INT" + ], + [ + 164, + 105, + 0, + 99, + 1, + "INT" + ], + [ + 166, + 80, + 0, + 27, + 2, + "WANVIDIMAGE_EMBEDS" + ], + [ + 168, + 87, + 0, + 30, + 0, + "IMAGE" + ], + [ + 169, + 98, + 0, + 63, + 2, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "ComfyUI text encoding alternative", + "bounding": [ + 430.61114501953125, + 782.2755126953125, + 1210.621337890625, + 805.9080810546875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Start image", + "bounding": [ + -1762.3848876953125, + -569.3446655273438, + 1364.3648681640625, + 982.508544921875 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.5131581182307067, + "offset": [ + 2047.9575205645494, + 937.8470574906838 + ] + }, + "frontendVersion": "1.17.3", + "node_versions": { + "ComfyUI-WanVideoWrapper": "5a2383621a05825d0d0437781afcb8552d9590fd", + "comfy-core": "0.3.26", + "ComfyUI-KJNodes": "a5bd3c86c8ed6b83c55c2d0e7a59515b15a0137f", + "ComfyUI-VideoHelperSuite": "0a75c7958fe320efcb052f1d9f8451fd20c730a8" + }, + "VHS_latentpreview": true, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/fun_camera/nodes.py b/fun_camera/nodes.py new file mode 100644 index 0000000..d7e9479 --- /dev/null +++ b/fun_camera/nodes.py @@ -0,0 +1,186 @@ +import numpy as np +import os +import torch +from einops import rearrange + +script_directory = os.path.dirname(os.path.abspath(__file__)) + +class Camera(object): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + def __init__(self, entry): + fx, fy, cx, cy = entry[1:5] + self.fx = fx + self.fy = fy + self.cx = cx + self.cy = cy + w2c_mat = np.array(entry[7:]).reshape(3, 4) + w2c_mat_4x4 = np.eye(4) + w2c_mat_4x4[:3, :] = w2c_mat + self.w2c_mat = w2c_mat_4x4 + self.c2w_mat = np.linalg.inv(w2c_mat_4x4) + +def custom_meshgrid(*args): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + # ref: https://pytorch.org/docs/stable/generated/torch.meshgrid.html?highlight=meshgrid#torch.meshgrid + return torch.meshgrid(*args) + + +def get_relative_pose(cam_params): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + abs_w2cs = [cam_param.w2c_mat for cam_param in cam_params] + abs_c2ws = [cam_param.c2w_mat for cam_param in cam_params] + cam_to_origin = 0 + target_cam_c2w = np.array([ + [1, 0, 0, 0], + [0, 1, 0, -cam_to_origin], + [0, 0, 1, 0], + [0, 0, 0, 1] + ]) + abs2rel = target_cam_c2w @ abs_w2cs[0] + ret_poses = [target_cam_c2w, ] + [abs2rel @ abs_c2w for abs_c2w in abs_c2ws[1:]] + ret_poses = np.array(ret_poses, dtype=np.float32) + return ret_poses + +def ray_condition(K, c2w, H, W, device): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + # c2w: B, V, 4, 4 + # K: B, V, 4 + + B = K.shape[0] + + j, i = custom_meshgrid( + torch.linspace(0, H - 1, H, device=device, dtype=c2w.dtype), + torch.linspace(0, W - 1, W, device=device, dtype=c2w.dtype), + ) + i = i.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + j = j.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + + fx, fy, cx, cy = K.chunk(4, dim=-1) # B,V, 1 + + zs = torch.ones_like(i) # [B, HxW] + xs = (i - cx) / fx * zs + ys = (j - cy) / fy * zs + zs = zs.expand_as(ys) + + directions = torch.stack((xs, ys, zs), dim=-1) # B, V, HW, 3 + directions = directions / directions.norm(dim=-1, keepdim=True) # B, V, HW, 3 + + rays_d = directions @ c2w[..., :3, :3].transpose(-1, -2) # B, V, 3, HW + rays_o = c2w[..., :3, 3] # B, V, 3 + rays_o = rays_o[:, :, None].expand_as(rays_d) # B, V, 3, HW + # c2w @ dirctions + rays_dxo = torch.cross(rays_o, rays_d) + plucker = torch.cat([rays_dxo, rays_d], dim=-1) + plucker = plucker.reshape(B, c2w.shape[1], H, W, 6) # B, V, H, W, 6 + # plucker = plucker.permute(0, 1, 4, 2, 3) + return plucker + +def process_poses(poses, width=672, height=384, original_pose_width=1280, original_pose_height=720, device='cpu', return_poses=False): + """Modified from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + + cam_params = [[float(x) for x in pose] for pose in poses] + if return_poses: + return cam_params + else: + cam_params = [Camera(cam_param) for cam_param in cam_params] + + sample_wh_ratio = width / height + pose_wh_ratio = original_pose_width / original_pose_height # Assuming placeholder ratios, change as needed + + if pose_wh_ratio > sample_wh_ratio: + resized_ori_w = height * pose_wh_ratio + for cam_param in cam_params: + cam_param.fx = resized_ori_w * cam_param.fx / width + else: + resized_ori_h = width / pose_wh_ratio + for cam_param in cam_params: + cam_param.fy = resized_ori_h * cam_param.fy / height + + intrinsic = np.asarray([[cam_param.fx * width, + cam_param.fy * height, + cam_param.cx * width, + cam_param.cy * height] + for cam_param in cam_params], dtype=np.float32) + + K = torch.as_tensor(intrinsic)[None] # [1, 1, 4] + c2ws = get_relative_pose(cam_params) # Assuming this function is defined elsewhere + c2ws = torch.as_tensor(c2ws)[None] # [1, n_frame, 4, 4] + plucker_embedding = ray_condition(K, c2ws, height, width, device=device)[0].permute(0, 3, 1, 2).contiguous() # V, 6, H, W + plucker_embedding = plucker_embedding[None] + plucker_embedding = rearrange(plucker_embedding, "b f c h w -> b f h w c")[0] + return plucker_embedding + +class WanVideoFunCameraEmbeds: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "poses": ("CAMERACTRL_POSES", ), + "width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8, "tooltip": "Width of the image to encode"}), + "height": ("INT", {"default": 480, "min": 64, "max": 29048, "step": 8, "tooltip": "Height of the image to encode"}), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Strength of the camera motion"}), + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "Start percent of the steps to apply camera motion"}), + "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, "tooltip": "End percent of the steps to apply camera motion"}), + }, + # "optional": { + # "fun_ref_image": ("LATENT", {"tooltip": "Reference latent for the Fun 1.1 -model"}), + # } + } + + + RETURN_TYPES = ("WANVIDIMAGE_EMBEDS",) + RETURN_NAMES = ("image_embeds",) + FUNCTION = "process" + CATEGORY = "WanVideoWrapper" + + def process(self, poses, width, height, strength, start_percent, end_percent, fun_ref_image=None): + num_frames = len(poses) + + control_camera_video = process_poses(poses, width, height) + control_camera_video = control_camera_video.permute([3, 0, 1, 2]).unsqueeze(0) + print("control_camera_video.shape", control_camera_video.shape) + + # Rearrange dimensions + # Concatenate and transpose dimensions + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_video[:, :, 0:1], repeats=4, dim=2), + control_camera_video[:, :, 1:] + ], dim=2 + ).transpose(1, 2) + + # Reshape, transpose, and view into desired shape + b, f, c, h, w = control_camera_latents.shape + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, 4, c, h, w).transpose(2, 3) + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, c * 4, h, w).transpose(1, 2) + print("control_camera_latents.shape", control_camera_latents.shape) + + vae_stride = (4, 8, 8) + + target_shape = (16, (num_frames - 1) // vae_stride[0] + 1, + height // vae_stride[1], + width // vae_stride[2]) + + embeds = { + "target_shape": target_shape, + "num_frames": num_frames, + "control_embeds": { + "control_camera_latents": control_camera_latents * strength, + "control_camera_start_percent": start_percent, + "control_camera_end_percent": end_percent, + "fun_ref_image": fun_ref_image["samples"][:,:, 0] if fun_ref_image is not None else None, + } + } + + return (embeds,) + +NODE_CLASS_MAPPINGS = { + "WanVideoFunCameraEmbeds": WanVideoFunCameraEmbeds, + } +NODE_DISPLAY_NAME_MAPPINGS = { + "WanVideoFunCameraEmbeds": "WanVideo FunCamera Embeds", + } diff --git a/nodes.py b/nodes.py index 601c915..2578281 100644 --- a/nodes.py +++ b/nodes.py @@ -559,6 +559,9 @@ class WanVideoModelLoader: model_type = "i2v" elif in_channels == 16: model_type = "t2v" + elif "control_adapter.conv.weight" in sd: + model_type = "t2v" + num_heads = 40 if dim == 5120 else 12 num_layers = 40 if dim == 5120 else 30 @@ -625,8 +628,7 @@ class WanVideoModelLoader: "add_ref_conv": True if "ref_conv.weight" in sd else False, "in_dim_ref_conv": sd["ref_conv.weight"].shape[1] if "ref_conv.weight" in sd else None, "add_control_adapter": True if "control_adapter.conv.weight" in sd else False, - "in_dim_control_adapter": sd["control_adapter.conv.weight"].shape[1] if "control_adapter.conv.weight" in sd else None, - } + } with init_empty_weights(): transformer = WanModel(**TRANSFORMER_CONFIG) @@ -668,7 +670,7 @@ class WanVideoModelLoader: dtype = torch.float8_e5m2 else: dtype = base_dtype - params_to_keep = {"norm", "head", "bias", "time_in", "vector_in", "patch_embedding", "time_", "img_emb", "modulation", "text_embedding"} + params_to_keep = {"norm", "head", "bias", "time_in", "vector_in", "patch_embedding", "time_", "img_emb", "modulation", "text_embedding", "adapter"} #if lora is not None: # transformer_load_device = device if not lora_low_mem_load: @@ -1588,13 +1590,13 @@ class WanVideoImageToVideoEncode: "width": ("INT", {"default": 832, "min": 64, "max": 2048, "step": 8, "tooltip": "Width of the image to encode"}), "height": ("INT", {"default": 480, "min": 64, "max": 29048, "step": 8, "tooltip": "Height of the image to encode"}), "num_frames": ("INT", {"default": 81, "min": 1, "max": 10000, "step": 4, "tooltip": "Number of frames to encode"}), - "clip_embeds": ("WANVIDIMAGE_CLIPEMBEDS", {"tooltip": "Clip vision encoded image"}), "noise_aug_strength": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.001, "tooltip": "Strength of noise augmentation, helpful for I2V where some noise can add motion and give sharper results"}), "start_latent_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001, "tooltip": "Additional latent multiplier, helpful for I2V where lower values allow for more motion"}), "end_latent_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.001, "tooltip": "Additional latent multiplier, helpful for I2V where lower values allow for more motion"}), "force_offload": ("BOOLEAN", {"default": True}), }, "optional": { + "clip_embeds": ("WANVIDIMAGE_CLIPEMBEDS", {"tooltip": "Clip vision encoded image"}), "start_image": ("IMAGE", {"tooltip": "Image to encode"}), "end_image": ("IMAGE", {"tooltip": "end frame"}), "control_embeds": ("WANVIDIMAGE_EMBEDS", {"tooltip": "Control signal for the Fun -model"}), @@ -1609,9 +1611,9 @@ class WanVideoImageToVideoEncode: FUNCTION = "process" CATEGORY = "WanVideoWrapper" - def process(self, vae, width, height, num_frames, clip_embeds, force_offload, noise_aug_strength, + def process(self, vae, width, height, num_frames, force_offload, noise_aug_strength, start_latent_strength, end_latent_strength, start_image=None, end_image=None, control_embeds=None, fun_or_fl2v_model=False, - temporal_mask=None, extra_latents=None): + temporal_mask=None, extra_latents=None, clip_embeds=None): device = mm.get_torch_device() offload_device = mm.unet_offload_device() @@ -1722,8 +1724,8 @@ class WanVideoImageToVideoEncode: image_embeds = { "image_embeds": y, - "clip_context": clip_embeds.get("clip_embeds", None), - "negative_clip_context": clip_embeds.get("negative_clip_embeds", None), + "clip_context": clip_embeds.get("clip_embeds", None) if clip_embeds is not None else None, + "negative_clip_context": clip_embeds.get("negative_clip_embeds", None) if clip_embeds is not None else None, "max_seq_len": max_seq_len, "num_frames": num_frames, "lat_h": lat_h, @@ -2400,9 +2402,9 @@ class WanVideoSampler: seed_g = torch.Generator(device=torch.device("cpu")) seed_g.manual_seed(seed) - control_latents, clip_fea, clip_fea_neg, end_image, recammaster, camera_embed, unianim_data = None, None, None, None, None, None, None - vace_data, vace_context, vace_scale = None, None, None - fun_or_fl2v_model, has_ref, drop_last, = False, False, False + control_latents = control_camera_latents = clip_fea = clip_fea_neg = end_image = recammaster = camera_embed = unianim_data = None + vace_data = vace_context = vace_scale = None + fun_or_fl2v_model = has_ref = drop_last = False phantom_latents = None fun_ref_image = None @@ -2435,9 +2437,12 @@ class WanVideoSampler: control_embeds = image_embeds.get("control_embeds", None) if control_embeds is not None: - if transformer.in_dim != 48: + if transformer.in_dim not in [48, 32]: raise ValueError("Control signal only works with Fun-Control model") - control_latents = control_embeds["control_images"].to(device) + control_latents = control_embeds.get("control_images", None) + control_camera_latents = control_embeds.get("control_camera_latents", None) + control_camera_start_percent = control_embeds.get("control_camera_start_percent", 0.0) + control_camera_end_percent = control_embeds.get("control_camera_end_percent", 1.0) control_start_percent = control_embeds.get("start_percent", 0.0) control_end_percent = control_embeds.get("end_percent", 1.0) drop_last = image_embeds.get("drop_last", False) @@ -2498,7 +2503,15 @@ class WanVideoSampler: control_embeds = image_embeds.get("control_embeds", None) if control_embeds is not None: - control_latents = control_embeds["control_images"].to(device) + control_latents = control_embeds.get("control_images", None) + if control_latents is not None: + control_latents = control_latents.to(device) + control_camera_latents = control_embeds.get("control_camera_latents", None) + control_camera_start_percent = control_embeds.get("control_camera_start_percent", 0.0) + control_camera_end_percent = control_embeds.get("control_camera_end_percent", 1.0) + if control_camera_latents is not None: + control_camera_latents = control_camera_latents.to(device) + if control_lora: image_cond = control_latents.to(device) if not patcher.model.is_patched: @@ -2506,9 +2519,9 @@ class WanVideoSampler: patcher = apply_lora(patcher, device, device, low_mem_load=False) patcher.model.is_patched = True else: - if transformer.in_dim != 48: + if transformer.in_dim not in [48, 32]: raise ValueError("Control signal only works with Fun-Control model") - image_cond = torch.zeros_like(control_latents).to(device) #fun control + image_cond = torch.zeros_like(noise).to(device) #fun control clip_fea = None fun_ref_image = control_embeds.get("fun_ref_image", None) control_start_percent = control_embeds.get("start_percent", 0.0) @@ -2691,6 +2704,8 @@ class WanVideoSampler: for name, param in transformer.named_parameters(): if "block" not in name: param.data = param.data.to(device) + if "control_adapter" in name: + param.data = param.data.to(device) elif block_swap_args["offload_txt_emb"] and "txt_emb" in name: param.data = param.data.to(offload_device, non_blocking=transformer.use_non_blocking) elif block_swap_args["offload_img_emb"] and "img_emb" in name: @@ -2846,10 +2861,17 @@ class WanVideoSampler: if not patcher.model.is_patched: log.info("Loading LoRA...") patcher = apply_lora(patcher, device, device, low_mem_load=False) - patcher.model.is_patched = True + patcher.model.is_patched = True else: image_cond_input = image_cond.to(z) if image_cond is not None else None + if control_camera_latents is not None: + if (control_camera_start_percent <= current_step_percentage <= control_camera_end_percent) or \ + (control_end_percent > 0 and idx == 0 and current_step_percentage >= control_camera_start_percent): + control_camera_input = control_camera_latents.to(z) + else: + control_camera_input = None + if recammaster is not None: z = torch.cat([z, recam_latents.to(z)], dim=1) use_phantom = False @@ -2876,6 +2898,7 @@ class WanVideoSampler: 'camera_embed': camera_embed, 'unianim_data': unianim_data, 'fun_ref': fun_ref_input if fun_ref_image is not None else None, + 'fun_camera': control_camera_input if control_camera_latents is not None else None, 'audio_proj': audio_proj if fantasytalking_embeds is not None else None, 'audio_context_lens': audio_context_lens if fantasytalking_embeds is not None else None, 'audio_scale': audio_scale if fantasytalking_embeds is not None else None, diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index 15151f3..586a609 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -1066,7 +1066,7 @@ class WanModel(ModelMixin, ConfigMixin): camera_embed=None, unianim_data=None, fps_embeds=None, - fun_ref = None, + fun_ref=None, fun_camera=None, audio_proj=None, audio_context_lens=None, @@ -1214,7 +1214,7 @@ class WanModel(ModelMixin, ConfigMixin): self.text_embedding.to(self.offload_device, non_blocking=self.use_non_blocking) clip_embed = None - if clip_fea is not None: + if clip_fea is not None and hasattr(self, "img_emb"): clip_fea = clip_fea.to(self.main_device) if self.offload_img_emb: self.img_emb.to(self.main_device)