nodes/HelloVision-ComfyUI_HelloMeme
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add hellomeme v4

Browse Source
This commit is contained in:
songkey-win
2025-03-23 02:02:04 +08:00
parent 834163be7c
commit 7c67b9e4f0
18 changed files with 1027 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
example_workflows/hellomeme_image_workflow.json
-1
View File
File diff suppressed because one or more lines are too long
example_workflows/hellomeme_video_v2_workflow.json
-1
View File
File diff suppressed because one or more lines are too long
example_workflows/hellomeme_video_workflow.json
-1
View File
File diff suppressed because one or more lines are too long
example_workflows/image_generation_ctrl1.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 164 KiB

example_workflows/image_generation_ctrl1.json
+1
View File
@@ -0,0 +1 @@
{"last_node_id":63,"last_link_id":129,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,127],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,126],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,125],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":124},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":null}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,859627586602455,"randomize",2,0]},{"id":61,"type":"GetDriveExpression","pos":[857,823],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":127},{"name":"images","type":"IMAGE","link":125},{"name":"landmarks","type":"FACELANDMARKS222","link":126}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[124],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[210,246],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[124,61,0,58,4,"DRIVE_EXPRESSION"],[125,57,0,61,1,"IMAGE"],[126,43,0,61,2,"FACELANDMARKS222"],[127,2,0,61,0,"FACE_TOOLKITS"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4}
example_workflows/image_generation_ctrl2.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 197 KiB

example_workflows/hellomeme_image_v2_workflow.json → example_workflows/image_generation_ctrl2.json
+1 -1
View File
@@ -1 +1 @@
{"last_node_id":64,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":133,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293309029136420,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[290.7474365234375,283.26800537109375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4}
{"last_node_id":64,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","shape":7,"link":null},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","shape":7,"link":133}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,404637369688717,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[290.7474365234375,283.26800537109375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]MoreDetails","same as checkpoint","v4","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4}
example_workflows/video_generation_ctrl1.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 207 KiB

example_workflows/video_generation_ctrl1.json
+682
View File
@@ -0,0 +1,682 @@
{
"last_node_id": 60,
"last_link_id": 122,
"nodes": [
{
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": [
43,
699
],
"size": [
315,
82
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"links": [
57,
62,
65,
97,
119
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": [
0,
"huggingface"
]
},
{
"id": 39,
"type": "CropPortrait",
"pos": [
516,
537
],
"size": [
241.79998779296875,
46
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 116
},
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 57
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
99
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CropPortrait"
},
"widgets_values": []
},
{
"id": 43,
"type": "GetFaceLandmarks",
"pos": [
465,
771
],
"size": [
292.20001220703125,
46
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 62
},
{
"name": "images",
"type": "IMAGE",
"link": 82
}
],
"outputs": [
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"links": [
64,
120
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetFaceLandmarks"
},
"widgets_values": []
},
{
"id": 44,
"type": "GetDrivePose",
"pos": [
842,
677
],
"size": [
304.79998779296875,
66
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 65
},
{
"name": "images",
"type": "IMAGE",
"link": 83
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 64
}
],
"outputs": [
{
"name": "drive_pose",
"type": "DRIVE_POSE",
"links": [
100
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetDrivePose"
},
"widgets_values": []
},
{
"id": 53,
"type": "HMPipelineVideo",
"pos": [
1258,
562
],
"size": [
315,
350
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"link": 115
},
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 97
},
{
"name": "ref_image",
"type": "IMAGE",
"link": 99
},
{
"name": "drive_pose",
"type": "DRIVE_POSE",
"link": 100
},
{
"name": "drive_exp",
"type": "DRIVE_EXPRESSION",
"shape": 7,
"link": 122
},
{
"name": "drive_exp2",
"type": "DRIVE_EXPRESSION2",
"shape": 7,
"link": null
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
117
],
"slot_index": 0
},
{
"name": "LATENT",
"type": "LATENT",
"links": null
}
],
"properties": {
"Node name for S&R": "HMPipelineVideo"
},
"widgets_values": [
0,
4,
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
990809923057900,
"randomize",
2,
0
]
},
{
"id": 60,
"type": "GetDriveExpression",
"pos": [
845,
813
],
"size": [
292.20001220703125,
66
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 119
},
{
"name": "images",
"type": "IMAGE",
"link": 121
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 120
}
],
"outputs": [
{
"name": "drive_exp",
"type": "DRIVE_EXPRESSION",
"links": [
122
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetDriveExpression"
},
"widgets_values": []
},
{
"id": 58,
"type": "LoadImage",
"pos": [
56,
273
],
"size": [
315,
314
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
116
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"toon.png",
"image"
]
},
{
"id": 59,
"type": "VHS_VideoCombine",
"pos": [
1619,
800
],
"size": [
280.4215087890625,
584.4215087890625
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 117
},
{
"name": "audio",
"type": "AUDIO",
"shape": 7,
"link": 118
},
{
"name": "meta_batch",
"type": "VHS_BatchManager",
"shape": 7,
"link": null
},
{
"name": "vae",
"type": "VAE",
"shape": 7,
"link": null
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 15,
"loop_count": 0,
"filename_prefix": "AnimateDiff",
"format": "video/h264-mp4",
"pix_fmt": "yuv420p",
"crf": 18,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "AnimateDiff_00409-audio.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4",
"frame_rate": 15
},
"muted": false
}
}
},
{
"id": 48,
"type": "VHS_LoadVideo",
"pos": [
106,
867
],
"size": [
247.455078125,
304
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "meta_batch",
"type": "VHS_BatchManager",
"shape": 7,
"link": null
},
{
"name": "vae",
"type": "VAE",
"shape": 7,
"link": null
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
82,
83,
121
],
"slot_index": 0
},
{
"name": "frame_count",
"type": "INT",
"links": null
},
{
"name": "audio",
"type": "AUDIO",
"links": [
118
],
"slot_index": 2
},
{
"name": "video_info",
"type": "VHS_VIDEOINFO",
"links": null
}
],
"properties": {
"Node name for S&R": "VHS_LoadVideo"
},
"widgets_values": {
"video": "tiktok.mp4",
"force_rate": 15,
"force_size": "Disabled",
"custom_width": 512,
"custom_height": 512,
"frame_load_cap": 0,
"skip_first_frames": 0,
"select_every_nth": 1,
"choose video to upload": "image",
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"force_rate": 15,
"frame_load_cap": 0,
"skip_first_frames": 0,
"select_every_nth": 1,
"filename": "tiktok.mp4",
"type": "input",
"format": "video/mp4"
},
"muted": false
}
}
},
{
"id": 57,
"type": "HMVideoPipelineLoader",
"pos": [
813,
343
],
"size": [
352.79998779296875,
202
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"links": [
115
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMVideoPipelineLoader"
},
"widgets_values": [
"[preset]DisneyPixarCartoonB",
"[preset]MoreDetails",
"same as checkpoint",
"v4",
"x1",
"huggingface",
1
]
}
],
"links": [
[
57,
2,
0,
39,
1,
"FACE_TOOLKITS"
],
[
62,
2,
0,
43,
0,
"FACE_TOOLKITS"
],
[
64,
43,
0,
44,
2,
"FACELANDMARKS222"
],
[
65,
2,
0,
44,
0,
"FACE_TOOLKITS"
],
[
82,
48,
0,
43,
1,
"IMAGE"
],
[
83,
48,
0,
44,
1,
"IMAGE"
],
[
97,
2,
0,
53,
1,
"FACE_TOOLKITS"
],
[
99,
39,
0,
53,
2,
"IMAGE"
],
[
100,
44,
0,
53,
3,
"DRIVE_POSE"
],
[
115,
57,
0,
53,
0,
"HMVIDEOPIPELINE"
],
[
116,
58,
0,
39,
0,
"IMAGE"
],
[
117,
53,
0,
59,
0,
"IMAGE"
],
[
118,
48,
2,
59,
1,
"AUDIO"
],
[
119,
2,
0,
60,
0,
"FACE_TOOLKITS"
],
[
120,
43,
0,
60,
2,
"FACELANDMARKS222"
],
[
121,
48,
0,
60,
1,
"IMAGE"
],
[
122,
60,
0,
53,
4,
"DRIVE_EXPRESSION"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.7513148009015779,
"offset": [
634.1362099561715,
-70.9744212566187
]
}
},
"version": 0.4
}
example_workflows/video_generation_ctrl2.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 186 KiB

example_workflows/video_generation_ctrl2.json
+1
View File
File diff suppressed because one or more lines are too long
hellomeme/models/__init__.py
+1 -1
View File
@@ -9,7 +9,7 @@
"""
from .hm_denoising_motion import HMDenoisingMotion
from .hm_control import HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2, HMV3ControlNet
from .hm_control import HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2, HMV3ControlNet, HMControlNetBase, HM4SD15ControlProj
from .hm_blocks import HMReferenceAdapter, HM3ReferenceAdapter, HM3MotionAdapter, HMPipeline
from .hm_denoising_3d import HMDenoising3D
from .hm3_denoising_3d import HM3Denoising3D
hellomeme/models/hm_blocks.py
+152
View File
@@ -270,6 +270,158 @@ class SKReferenceAttentionV3(nn.Module):
return hidden_states + self.proj(res2.contiguous())
class SmallUnet(nn.Module):
def __init__(self, in_channels: int = 3,
out_channels: int = 16,
cross_attention_dim: int = 1024,
mid_channels: Tuple[int] = (320, 640, 1280),
temporal_attn: bool = False):
super(SmallUnet, self).__init__()
down_channels = [in_channels] + mid_channels
up_channels = mid_channels[::-1] + [out_channels]
self.down_blocks = nn.ModuleList([
BasicBlock(
inplanes=down_channels[i-1],
planes=down_channels[i],
stride=2,
downsample=nn.Sequential(
nn.Conv2d(down_channels[i-1], down_channels[i], kernel_size=1, stride=2, bias=False),
nn.InstanceNorm2d(down_channels[i]),
nn.SiLU(),
),
norm_layer=nn.InstanceNorm2d,
) for i in range(1, len(down_channels))
])
self.up_blocks = nn.ModuleList([
nn.Sequential(
nn.Conv2d(up_channels[i-1], up_channels[i], kernel_size=3, padding=1),
nn.InstanceNorm2d(up_channels[i]),
nn.SiLU(),
nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False),
) for i in range(1, len(up_channels))
])
self.attn_blocks = nn.ModuleList([
STKCrossAttention(
channel_in=c,
channel_mid=c,
cross_attention_dim=cross_attention_dim,
num_positional_embeddings=512,
num_positional_embeddings_hidden=1024,
temporal_attn=temporal_attn,
) for c in mid_channels
])
def forward(self, x, condition):
f = x.size(2)
x = rearrange(x, "b c f h w -> (b f) c h w")
skips = []
ret_dict = {}
for down_block, attn_block in zip(self.down_blocks, self.attn_blocks):
x = rearrange(down_block(x), "(b f) c h w -> b f c h w", f=f)
x = attn_block(x, condition)
x = rearrange(x, "b f c h w -> (b f) c h w")
skips.append(x)
skips = skips[::-1][1:]
ret_dict['feat_0'] = rearrange(x, "(b f) c h w -> b c f h w", f=f)
for i, block in enumerate(self.up_blocks[:-1]):
x = block(x) + skips[i]
ret_dict[f'feat_{i+1}'] = rearrange(x, "(b f) c h w -> b c f h w", f=f)
x = self.up_blocks[-1](x)
ret_dict[f'feat_{i+2}'] = rearrange(x, "(b f) c h w -> b c f h w", f=f)
return ret_dict
class STKCrossAttention(nn.Module):
def __init__(
self,
channel_in: int,
channel_mid: int,
heads: int = 8,
cross_attention_dim: int = 320,
norm_elementwise_affine: bool = True,
norm_eps: float = 1e-5,
num_positional_embeddings: int = 64,
num_positional_embeddings_hidden: int = 64,
temporal_attn: bool = False,
):
super().__init__()
self.conv_in = nn.Conv2d(channel_in, channel_mid, kernel_size=3, padding=1)
self.pos_embed = SinusoidalPositionalEmbedding(channel_mid, max_seq_length=num_positional_embeddings)
self.pos_embed_hidden = SinusoidalPositionalEmbedding(cross_attention_dim, max_seq_length=num_positional_embeddings_hidden)
self.norm = nn.LayerNorm(channel_mid, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
self.attn1 = Attention(
query_dim=channel_mid,
heads=heads,
dim_head=channel_mid // heads,
dropout=0.0,
bias=False,
cross_attention_dim=cross_attention_dim,
upcast_attention=False,
out_bias=True,
)
self.attn2 = Attention(
query_dim=channel_mid,
heads=heads,
dim_head=channel_mid // heads,
dropout=0.0,
bias=False,
cross_attention_dim=cross_attention_dim,
upcast_attention=False,
out_bias=True,
)
if temporal_attn:
self.attn3 = Attention(
query_dim=channel_mid,
heads=heads,
dim_head=channel_mid // heads,
dropout=0.0,
bias=False,
cross_attention_dim=cross_attention_dim,
upcast_attention=False,
out_bias=True,
)
self.ff = FeedForward(
channel_mid,
dropout=0.0,
activation_fn="geglu",
final_dropout=False,
inner_dim=channel_mid*16,
bias=True,
)
self.proj = nn.Conv2d(channel_mid, channel_in, kernel_size=3, padding=1)
def forward(self, input, hidden_stats):
b, f, _, h, w = input.shape
x = rearrange(input, "b f c h w -> (b f) c h w")
x = self.conv_in(x)
hidden_stats = rearrange(hidden_stats, "b f c d -> (b f) c d")
x = rearrange(x, "b c h w -> (b h) w c")
x = self.attn1(self.norm(self.pos_embed(x)),
self.pos_embed_hidden(hidden_stats.repeat_interleave(h, dim=0)))
x = rearrange(x, "(b h) w c -> (b w) h c", h=h)
x = self.attn2(self.norm(self.pos_embed(x)),
self.pos_embed_hidden(hidden_stats.repeat_interleave(w, dim=0)))
if hasattr(self, 'attn3'):
x = rearrange(x, "(b f w) h c -> (b w h) f c", w=w, f=f)
x = self.attn3(self.norm(self.pos_embed(x)),
self.pos_embed_hidden(hidden_stats.repeat_interleave(w*h, dim=0)))
x = rearrange(x, "(b w h) f c -> (b f) (h w) c", w=w, h=h)
else:
x = rearrange(x, "(b f w) h c -> (b f) (h w) c", w=w, f=f)
x = self.norm(self.ff(x))
x = rearrange(x, "(b f) (h w) c -> (b f) c h w", w=w, f=f)
x = self.proj(x)
return rearrange(x, "(b f) c h w -> b f c h w", f=f)
class SKCrossAttention(nn.Module):
def __init__(
self,
hellomeme/models/hm_control.py
+107 -1
View File
@@ -17,8 +17,114 @@ from diffusers.models.modeling_utils import ModelMixin
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.models.embeddings import TimestepEmbedding, Timesteps
from einops import rearrange
from .hm_blocks import SKCrossAttention
from .hm_blocks import SKCrossAttention, SmallUnet
class HMControlNetBase(ModelMixin, ConfigMixin):
@register_to_config
def __init__(
self,
in_channels: int = 3,
cross_attention_dim: int = 2048,
):
super().__init__()
self.emo_embedding = Timesteps(1024, True, 0)
self.emo_pre_proj = TimestepEmbedding(548, 16)
self.emo_proj = TimestepEmbedding(1024, cross_attention_dim)
self.exp_embedding = Timesteps(1024, True, 0)
self.exp_pre_proj = TimestepEmbedding(51, 16)
self.face_pre_proj = TimestepEmbedding(3, 16)
self.exp_proj = TimestepEmbedding(1024, cross_attention_dim)
self.face_proj = TimestepEmbedding(1024, cross_attention_dim)
self.ctrl_attn = SmallUnet(in_channels=in_channels, out_channels=320,
mid_channels=[320, 640, 1280],
cross_attention_dim=cross_attention_dim,
temporal_attn=False)
def forward(self, condition, drive_coeff=None, face_parts=None, emo_embedding=None):
bs, video_length = condition.size(0), condition.size(2)
condition = rearrange(condition, "b c f h w -> (b f) c h w")
condition = F.interpolate(condition, size=(condition.size(2)//8, condition.size(3)//8), mode='bilinear', align_corners=False)
condition = rearrange(condition, "(b f) c h w -> b c f h w", f=video_length)
if drive_coeff is None:
emo_coeff = rearrange(emo_embedding * 20., "b f c -> (b f c)")
emo_embedding = self.emo_embedding(emo_coeff)
emo_embedding = rearrange(emo_embedding, "(b f c) d -> (b f) d c", b=bs, f=video_length)
emo_embedding = self.emo_pre_proj(emo_embedding.to(dtype=condition.dtype))
emo_embedding = rearrange(emo_embedding, "(b f) d c -> (b f) c d", f=video_length)
drive_embedding = self.emo_proj(emo_embedding)
else:
drive_coeff = rearrange(drive_coeff * 500., "b f c -> (b f c)")
exp_embedding = self.exp_embedding(drive_coeff)
exp_embedding = rearrange(exp_embedding, "(b f c) d -> (b f) d c", b=bs, f=video_length)
exp_embedding = self.exp_pre_proj(exp_embedding.to(dtype=condition.dtype))
exp_embedding = rearrange(exp_embedding, "(b f) d c -> (b f) c d", f=video_length)
face_parts = rearrange(face_parts, "b f c d -> (b f) d c")
face_embedding = self.face_pre_proj(face_parts)
face_embedding = rearrange(face_embedding, "(b f) d c -> (b f) c d", f=video_length)
drive_embedding = self.exp_proj(exp_embedding) + self.face_proj(face_embedding)
drive_embedding = rearrange(drive_embedding, "(b f) c d -> b f c d", f=video_length)
return self.ctrl_attn(condition, drive_embedding)
class HM4SD15ControlProj(ModelMixin, ConfigMixin):
@register_to_config
def __init__(
self,
):
super().__init__()
self.map640_to_640 = nn.Conv2d(640, 640, kernel_size=3, padding=1, bias=False)
self.map640_to_1280 = nn.Conv2d(640, 1280, kernel_size=3, padding=1, bias=False)
self.map320_to_320 = nn.Conv2d(320, 320, kernel_size=3, padding=1, bias=False)
self.map320_to_640 = nn.Conv2d(320, 640, kernel_size=3, padding=1, bias=False)
self.map320_to_1280 = nn.Conv2d(320, 1280, kernel_size=3, padding=1, bias=False)
self.map1280_to_1280 = nn.Conv2d(1280, 1280, kernel_size=3, padding=1, bias=False)
self.map1280_to_1280_2 = nn.Conv2d(1280, 1280, kernel_size=3, padding=1, bias=False)
self.map1280_to_1280_3 = nn.Conv2d(1280, 1280, kernel_size=3, padding=1, bias=False)
def forward(self, control_dict):
video_length = control_dict['feat_0'].size(2)
ret_dict = dict(
up3_0=rearrange(
self.map1280_to_1280(
rearrange(control_dict['feat_0'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
up3_1=rearrange(
self.map640_to_1280(
rearrange(control_dict['feat_1'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
up3_2=rearrange(
self.map320_to_1280(
rearrange(control_dict['feat_2'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
up3_3=rearrange(
self.map320_to_640(
rearrange(control_dict['feat_3'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
down3_0=rearrange(
self.map320_to_320(
rearrange(control_dict['feat_2'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
down3_1=rearrange(
self.map640_to_640(
rearrange(control_dict['feat_1'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
down3_2=rearrange(
self.map1280_to_1280_2(
rearrange(control_dict['feat_0'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
down3_3=rearrange(
self.map1280_to_1280_3(
rearrange(control_dict['feat_0'], "b c f h w -> (b f) c h w"),
), "(b f) c h w -> b c f h w", f=video_length),
)
return ret_dict
class HMControlNet(ModelMixin, ConfigMixin):
@register_to_config
hellomeme/pipelines/pipline_hm3_image.py
+35 -9
View File
@@ -20,9 +20,7 @@ from diffusers.utils.torch_utils import randn_tensor
from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput
from diffusers import DPMSolverMultistepScheduler
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents
from ..models import HM3Denoising3D, HMV3ControlNet, HMPipeline, HM3ReferenceAdapter
from ..models import HM3Denoising3D, HMV3ControlNet, HMPipeline, HM3ReferenceAdapter, HMControlNetBase, HM4SD15ControlProj
class HM3ImagePipeline(HMPipeline):
def caryomitosis(self, **kwargs):
@@ -45,14 +43,24 @@ class HM3ImagePipeline(HMPipeline):
self.safety_checker.cpu()
def insert_hm_modules(self, version='v3', dtype=torch.float16, modelscope=False):
self.version = version
if modelscope:
from modelscope import snapshot_download
hm_reference_dir = snapshot_download('songkey/hm3_reference')
hm_control_dir = snapshot_download('songkey/hm3_control_mix')
if version == 'v3':
hm_reference_dir = snapshot_download('songkey/hm3_reference')
hm_control_dir = snapshot_download('songkey/hm3_control_mix')
else:
hm_reference_dir = snapshot_download('songkey/hm4_reference')
hm_control_dir = snapshot_download('songkey/hm_control_base')
hm_control_proj_dir = snapshot_download('songkey/hm4_control_proj')
else:
hm_reference_dir = 'songkey/hm3_reference'
hm_control_dir = 'songkey/hm3_control_mix'
if version == 'v3':
hm_reference_dir = 'songkey/hm3_reference'
hm_control_dir = 'songkey/hm3_control_mix'
else:
hm_reference_dir = 'songkey/hm4_reference'
hm_control_dir = 'songkey/hm_control_base'
hm_control_proj_dir = 'songkey/hm4_control_proj'
if isinstance(self.unet, HM3Denoising3D):
hm_adapter = HM3ReferenceAdapter.from_pretrained(hm_reference_dir)
@@ -65,7 +73,17 @@ class HM3ImagePipeline(HMPipeline):
if hasattr(self, "mp_control"):
del self.mp_control
self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir)
if hasattr(self, "mp_control_proj"):
del self.mp_control_proj
if version == 'v3':
self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir)
else:
self.mp_control = HMControlNetBase.from_pretrained(hm_control_dir)
self.mp_control_proj = HM4SD15ControlProj.from_pretrained(hm_control_proj_dir)
self.mp_control_proj.to(device='cpu', dtype=dtype).eval()
self.mp_control.to(device='cpu', dtype=dtype).eval()
self.vae.to(device='cpu', dtype=dtype).eval()
@@ -233,6 +251,8 @@ class HM3ImagePipeline(HMPipeline):
control_latents = {}
self.mp_control.to(device=device)
if hasattr(self, 'mp_control_proj') and self.version == 'v4':
self.mp_control_proj.to(device=device)
if 'drive_coeff' in drive_params:
drive_coeff = drive_params['drive_coeff'].clone().to(device=device)
face_parts = drive_params['face_parts'].clone().to(device=device)
@@ -240,14 +260,20 @@ class HM3ImagePipeline(HMPipeline):
drive_coeff = torch.cat([torch.zeros_like(drive_coeff), drive_coeff], dim=0)
face_parts = torch.cat([torch.zeros_like(face_parts), face_parts], dim=0)
control_latents1 = self.mp_control(condition=condition, drive_coeff=drive_coeff, face_parts=face_parts)
if self.version == 'v4':
control_latents1 = self.mp_control_proj(control_latents1)
control_latents.update(control_latents1)
elif 'pd_fpg' in drive_params:
pd_fpg = drive_params['pd_fpg'].clone().to(device=device)
if self.do_classifier_free_guidance:
pd_fpg = torch.cat([torch.zeros_like(pd_fpg), pd_fpg], dim=0)
control_latents2 = self.mp_control(condition=condition, emo_embedding=pd_fpg)
if self.version == 'v4':
control_latents2 = self.mp_control_proj(control_latents2)
control_latents.update(control_latents2)
self.mp_control.cpu()
if self.version == 'v4':
self.mp_control_proj.cpu()
# 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
hellomeme/pipelines/pipline_hm3_video.py
+42 -9
View File
@@ -24,7 +24,7 @@ from diffusers import MotionAdapter, DPMSolverMultistepScheduler
from diffusers.utils.torch_utils import randn_tensor
from ..models import (HM3Denoising3D, HM3DenoisingMotion, HM3MotionAdapter,
HMV3ControlNet, HM3ReferenceAdapter, HMPipeline)
HMV3ControlNet, HM3ReferenceAdapter, HMPipeline, HMControlNetBase, HM4SD15ControlProj)
class HM3VideoPipeline(HMPipeline):
def caryomitosis(self, version='v3', modelscope=False, **kwargs):
@@ -58,19 +58,32 @@ class HM3VideoPipeline(HMPipeline):
self.safety_checker.cpu()
def insert_hm_modules(self, version='v3', dtype=torch.float16, modelscope=False):
self.version = version
if modelscope:
from modelscope import snapshot_download
hm_reference_dir = snapshot_download('songkey/hm3_reference')
hm_control_dir = snapshot_download('songkey/hm3_control_mix')
hm_motion_dir = snapshot_download('songkey/hm3_motion')
if version == 'v3':
hm_reference_dir = snapshot_download('songkey/hm3_reference')
hm_control_dir = snapshot_download('songkey/hm3_control_mix')
hm_motion_dir = snapshot_download('songkey/hm3_motion')
else:
hm_reference_dir = snapshot_download('songkey/hm4_reference')
hm_control_dir = snapshot_download('songkey/hm_control_base')
hm_control_proj_dir = snapshot_download('songkey/hm4_control_proj')
hm_motion_dir = snapshot_download('songkey/hm4_motion')
else:
hm_reference_dir = 'songkey/hm3_reference'
hm_control_dir = 'songkey/hm3_control_mix'
hm_motion_dir = 'songkey/hm3_motion'
if version == 'v3':
hm_reference_dir = 'songkey/hm3_reference'
hm_control_dir = 'songkey/hm3_control_mix'
hm_motion_dir = 'songkey/hm3_motion'
else:
hm_reference_dir = 'songkey/hm4_reference'
hm_control_dir = 'songkey/hm_control_base'
hm_control_proj_dir = 'songkey/hm4_control_proj'
hm_motion_dir = 'songkey/hm4_motion'
hm_adapter = HM3ReferenceAdapter.from_pretrained(hm_reference_dir)
motion_adapter = HM3MotionAdapter.from_pretrained(hm_motion_dir)
motion_adapter = motion_adapter.to(device='cpu', dtype=dtype).eval()
if isinstance(self.unet, HM3DenoisingMotion):
self.unet.insert_reference_adapter(hm_adapter)
@@ -87,8 +100,16 @@ class HM3VideoPipeline(HMPipeline):
if hasattr(self, "mp_control"):
del self.mp_control
if hasattr(self, "mp_control_proj"):
del self.mp_control_proj
self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir)
if version == 'v3':
self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir)
else:
self.mp_control = HMControlNetBase.from_pretrained(hm_control_dir)
self.mp_control_proj = HM4SD15ControlProj.from_pretrained(hm_control_proj_dir)
self.mp_control_proj.to(device='cpu', dtype=dtype).eval()
self.mp_control.to(device='cpu', dtype=dtype).eval()
self.vae.to(device='cpu', dtype=dtype).eval()
@@ -456,6 +477,8 @@ class HM3VideoPipeline(HMPipeline):
control_latents = []
control_latent1_neg, control_latent2_neg = None, None
self.mp_control.to(device=device)
if self.version == 'v4':
self.mp_control_proj.to(device=device)
with self.progress_bar(total=raw_video_len) as progress_bar:
for idx in range(raw_video_len):
progress_bar.set_description(f"GEN stage1")
@@ -472,10 +495,14 @@ class HM3VideoPipeline(HMPipeline):
control_latent1_neg = self.mp_control(condition=torch.ones_like(condition)*-1,
drive_coeff=torch.zeros_like(drive_coeff),
face_parts=torch.zeros_like(face_parts))
if hasattr(self, 'mp_control_proj') and self.version == 'v4':
control_latent1_neg = self.mp_control_proj(control_latent1_neg)
control_latent1 = self.mp_control(condition=condition,
drive_coeff=drive_coeff,
face_parts=face_parts)
if hasattr(self, 'mp_control_proj') and self.version == 'v4':
control_latent1 = self.mp_control_proj(control_latent1)
if self.do_classifier_free_guidance:
control_latent1 = cat_dicts([control_latent1_neg, control_latent1], dim=0)
@@ -486,8 +513,12 @@ class HM3VideoPipeline(HMPipeline):
if control_latent2_neg is None and self.do_classifier_free_guidance:
control_latent2_neg = self.mp_control(condition=torch.ones_like(condition)*-1,
emo_embedding=torch.zeros_like(pd_fpg))
if hasattr(self, 'mp_control_proj') and self.version == 'v4':
control_latent2_neg = self.mp_control_proj(control_latent2_neg)
control_latent2 = self.mp_control(condition=condition, emo_embedding=pd_fpg)
if hasattr(self, 'mp_control_proj') and self.version == 'v4':
control_latent2 = self.mp_control_proj(control_latent2)
if self.do_classifier_free_guidance:
control_latent2 = cat_dicts([control_latent2_neg, control_latent2], dim=0)
@@ -495,6 +526,8 @@ class HM3VideoPipeline(HMPipeline):
control_latents.append(dicts_to_device([control_latent_input], device='cpu')[0])
self.mp_control.cpu()
if self.version == 'v4':
self.mp_control_proj.cpu()
more_params = dict(
timesteps=timesteps,
meme.py
+4 -4
View File
@@ -124,7 +124,7 @@ class HMImagePipelineLoader:
"checkpoint": (checkpoint_files, ),
"lora": (lora_files, ),
"vae": (vae_files, ),
"version": (['v1', 'v2', 'v3'], ),
"version": (['v1', 'v2', 'v3', 'v4'], ),
"stylize": (['x1', 'x2'], ),
"deployment": (['huggingface', 'modelscope'], ),
"lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}),
@@ -142,7 +142,7 @@ class HMImagePipelineLoader:
sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5')
else:
sd1_5_dir = "songkey/stable-diffusion-v1-5"
if version == 'v3':
if version == 'v3' or version == 'v4':
pipeline = HM3ImagePipeline.from_pretrained(sd1_5_dir)
else:
pipeline = HMImagePipeline.from_pretrained(sd1_5_dir)
@@ -166,7 +166,7 @@ class HMVideoPipelineLoader:
"checkpoint": (checkpoint_files, ),
"lora": (lora_files, ),
"vae": (vae_files, ),
"version": (['v1', 'v2', 'v3'], ),
"version": (['v1', 'v2', 'v3', 'v4'], ),
"stylize": (['x1', 'x2'], ),
"deployment": (['huggingface', 'modelscope'], ),
"lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}),
@@ -187,7 +187,7 @@ class HMVideoPipelineLoader:
else:
sd1_5_dir = "songkey/stable-diffusion-v1-5"
if version == 'v3':
if version == 'v3' or version == 'v4':
pipeline = HM3VideoPipeline.from_pretrained(sd1_5_dir)
else:
pipeline = HMVideoPipeline.from_pretrained(sd1_5_dir)
pyproject.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "comfyui_hellomeme"
description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface"
version = "1.1.8"
version = "1.2.0"
license = {file = "LICENSE"}
[project.urls]