nodes/HelloVision-ComfyUI_HelloMeme
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fp32 supported

Browse Source
This commit is contained in:
songkey-win
2025-06-03 13:41:37 +08:00
parent aec3e172bf
commit 1142e98655
4 changed files with 767 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
example_workflows/image_generation.json
+703 -193
View File
@@ -1,206 +1,716 @@
{
"2": {
"inputs": {
"gpu_id": 0,
"deployment": "huggingface"
},
"class_type": "HMFaceToolkitsLoader",
"_meta": {
"title": "Load Face Toolkits"
}
},
"58": {
"inputs": {
"image": "chillout.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "加载图像"
}
},
"60": {
"inputs": {
"face_toolkits": [
"2",
0
"id": "a8e8ebf3-8dc8-4d76-ad73-dfe0b945cc77",
"revision": 0,
"last_node_id": 80,
"last_link_id": 187,
"nodes": [
{
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": [
45.41996383666992,
769.9866333007812
],
"images": [
"58",
"size": [
315,
82
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"slot_index": 0,
"links": [
122,
129,
147,
149,
173,
174
]
}
],
"properties": {
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": [
0,
"huggingface"
]
},
{
"id": 61,
"type": "GetFaceLandmarks",
"pos": [
519.7196655273438,
938.032958984375
],
"size": [
170.62362670898438,
46
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 129
},
{
"name": "images",
"type": "IMAGE",
"link": 179
}
],
"outputs": [
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"links": [
145,
177
]
}
],
"properties": {
"Node name for S&R": "GetFaceLandmarks"
},
"widgets_values": []
},
{
"id": 60,
"type": "GetFaceLandmarks",
"pos": [
518.9132690429688,
612.9461669921875
],
"size": [
170.62362670898438,
46
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 122
},
{
"name": "images",
"type": "IMAGE",
"link": 121
}
],
"outputs": [
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"links": [
143,
178
]
}
],
"properties": {
"Node name for S&R": "GetFaceLandmarks"
},
"widgets_values": []
},
{
"id": 77,
"type": "LoadImage",
"pos": [
61.306365966796875,
1023.740478515625
],
"size": [
270,
314
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
179,
180,
181
]
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"yao.jpg",
"image"
]
},
{
"id": 67,
"type": "GetHeadPose",
"pos": [
814.1528930664062,
495.9793701171875
],
"size": [
270,
98
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 147
},
{
"name": "images",
"type": "IMAGE",
"link": 151
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 143
}
],
"outputs": [
{
"name": "head_pose",
"type": "HEAD_POSE",
"links": [
183
]
}
],
"properties": {
"Node name for S&R": "GetHeadPose"
},
"widgets_values": [
true
]
},
{
"id": 75,
"type": "GetExpression2",
"pos": [
814.1359252929688,
689.6795654296875
],
"size": [
191.63729858398438,
66
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 173
},
{
"name": "images",
"type": "IMAGE",
"link": 175
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 178
}
],
"outputs": [
{
"name": "expression",
"type": "EXPRESSION",
"links": [
184
]
}
],
"properties": {
"Node name for S&R": "GetExpression2"
},
"widgets_values": []
},
{
"id": 69,
"type": "GetHeadPose",
"pos": [
819.799560546875,
860.5927734375
],
"size": [
270,
98
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 149
},
{
"name": "images",
"type": "IMAGE",
"link": 180
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 145
}
],
"outputs": [
{
"name": "head_pose",
"type": "HEAD_POSE",
"links": [
185
]
}
],
"properties": {
"Node name for S&R": "GetHeadPose"
},
"widgets_values": [
true
]
},
{
"id": 76,
"type": "GetExpression2",
"pos": [
833.6573486328125,
1042.0396728515625
],
"size": [
191.63729858398438,
66
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 174
},
{
"name": "images",
"type": "IMAGE",
"link": 181
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 177
}
],
"outputs": [
{
"name": "expression",
"type": "EXPRESSION",
"links": [
186
]
}
],
"properties": {
"Node name for S&R": "GetExpression2"
},
"widgets_values": []
},
{
"id": 79,
"type": "HMPipelineImage",
"pos": [
1334.226318359375,
613.9535522460938
],
"size": [
270,
306
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"link": 182
},
{
"name": "ref_head_pose",
"type": "HEAD_POSE",
"link": 183
},
{
"name": "ref_expression",
"type": "EXPRESSION",
"link": 184
},
{
"name": "drive_head_pose",
"type": "HEAD_POSE",
"link": 185
},
{
"name": "drive_expression",
"type": "EXPRESSION",
"link": 186
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
187
]
},
{
"name": "LATENT",
"type": "LATENT",
"links": null
}
],
"properties": {
"Node name for S&R": "HMPipelineImage"
},
"widgets_values": [
0,
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
815482659530374,
"randomize",
2,
0
]
},
"class_type": "GetFaceLandmarks",
"_meta": {
"title": "Get Face Landmarks"
}
},
"61": {
"inputs": {
"face_toolkits": [
"2",
0
{
"id": 58,
"type": "LoadImage",
"pos": [
37.4466552734375,
260.9000244140625
],
"images": [
"77",
0
"size": [
315,
314
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"slot_index": 0,
"links": [
121,
151,
175
]
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"chillout.jpg",
"image"
]
},
"class_type": "GetFaceLandmarks",
"_meta": {
"title": "Get Face Landmarks"
{
"id": 80,
"type": "PreviewImage",
"pos": [
1703.6793212890625,
613.9529418945312
],
"size": [
210,
258
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 187
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 78,
"type": "HMImagePipelineLoader",
"pos": [
797.7930297851562,
188.84007263183594
],
"size": [
298.3472595214844,
226
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"links": [
182
]
}
],
"properties": {
"Node name for S&R": "HMImagePipelineLoader"
},
"widgets_values": [
"[preset]DisneyPixarCartoonB",
"None",
"same as checkpoint",
"v5",
"x1",
"huggingface",
1,
"fp32"
]
}
},
"67": {
"inputs": {
"crop": true,
"face_toolkits": [
"2",
0
],
"images": [
"58",
0
],
"landmarks": [
"60",
0
],
"links": [
[
121,
58,
0,
60,
1,
"IMAGE"
],
[
122,
2,
0,
60,
0,
"FACE_TOOLKITS"
],
[
129,
2,
0,
61,
0,
"FACE_TOOLKITS"
],
[
143,
60,
0,
67,
2,
"FACELANDMARKS222"
],
[
145,
61,
0,
69,
2,
"FACELANDMARKS222"
],
[
147,
2,
0,
67,
0,
"FACE_TOOLKITS"
],
[
149,
2,
0,
69,
0,
"FACE_TOOLKITS"
],
[
151,
58,
0,
67,
1,
"IMAGE"
],
[
173,
2,
0,
75,
0,
"FACE_TOOLKITS"
],
[
174,
2,
0,
76,
0,
"FACE_TOOLKITS"
],
[
175,
58,
0,
75,
1,
"IMAGE"
],
[
177,
61,
0,
76,
2,
"FACELANDMARKS222"
],
[
178,
60,
0,
75,
2,
"FACELANDMARKS222"
],
[
179,
77,
0,
61,
1,
"IMAGE"
],
[
180,
77,
0,
69,
1,
"IMAGE"
],
[
181,
77,
0,
76,
1,
"IMAGE"
],
[
182,
78,
0,
79,
0,
"HMIMAGEPIPELINE"
],
[
183,
67,
0,
79,
1,
"HEAD_POSE"
],
[
184,
75,
0,
79,
2,
"EXPRESSION"
],
[
185,
69,
0,
79,
3,
"HEAD_POSE"
],
[
186,
76,
0,
79,
4,
"EXPRESSION"
],
[
187,
79,
0,
80,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.8264462809917358,
"offset": [
493.27712102477454,
-69.45337656976547
]
},
"class_type": "GetHeadPose",
"_meta": {
"title": "Get Head Pose"
}
"frontendVersion": "1.19.2"
},
"69": {
"inputs": {
"crop": true,
"face_toolkits": [
"2",
0
],
"images": [
"77",
0
],
"landmarks": [
"61",
0
]
},
"class_type": "GetHeadPose",
"_meta": {
"title": "Get Head Pose"
}
},
"75": {
"inputs": {
"face_toolkits": [
"2",
0
],
"images": [
"58",
0
],
"landmarks": [
"60",
0
]
},
"class_type": "GetExpression2",
"_meta": {
"title": "Get Face Expression V2"
}
},
"76": {
"inputs": {
"face_toolkits": [
"2",
0
],
"images": [
"77",
0
],
"landmarks": [
"61",
0
]
},
"class_type": "GetExpression2",
"_meta": {
"title": "Get Face Expression V2"
}
},
"77": {
"inputs": {
"image": "yao.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "加载图像"
}
},
"78": {
"inputs": {
"checkpoint": "[preset]DisneyPixarCartoonB",
"lora": "None",
"vae": "same as checkpoint",
"version": "v5",
"stylize": "x1",
"deployment": "huggingface",
"lora_scale": 1
},
"class_type": "HMImagePipelineLoader",
"_meta": {
"title": "Load HelloMemeImage Pipeline"
}
},
"79": {
"inputs": {
"trans_ratio": 0,
"prompt": "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"negative_prompt": "",
"steps": 25,
"seed": 815482659530374,
"guidance_scale": 2,
"gpu_id": 0,
"hm_image_pipeline": [
"78",
0
],
"ref_head_pose": [
"67",
0
],
"ref_expression": [
"75",
0
],
"drive_head_pose": [
"69",
0
],
"drive_expression": [
"76",
0
]
},
"class_type": "HMPipelineImage",
"_meta": {
"title": "HelloMeme Image Pipeline"
}
},
"80": {
"inputs": {
"images": [
"79",
0
]
},
"class_type": "PreviewImage",
"_meta": {
"title": "预览图像"
}
}
"version": 0.4
}
example_workflows/video_generation.json
+42 -41
View File
@@ -404,44 +404,6 @@
true
]
},
{
"id": 57,
"type": "HMVideoPipelineLoader",
"pos": [
798.4796752929688,
193.7666015625
],
"size": [
352.79998779296875,
202
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"slot_index": 0,
"links": [
165
]
}
],
"properties": {
"Node name for S&R": "HMVideoPipelineLoader"
},
"widgets_values": [
"[preset]RealisticVisionV60B1",
"None",
"same as checkpoint",
"v5",
"x1",
"huggingface",
1
]
},
{
"id": 48,
"type": "VHS_LoadVideo",
@@ -451,10 +413,10 @@
],
"size": [
247.455078125,
503.455078125
262
],
"flags": {},
"order": 2,
"order": 1,
"mode": 0,
"inputs": [
{
@@ -541,7 +503,7 @@
314
],
"flags": {},
"order": 3,
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
@@ -615,6 +577,45 @@
"widgets_values": [
true
]
},
{
"id": 57,
"type": "HMVideoPipelineLoader",
"pos": [
798.4796752929688,
193.7666015625
],
"size": [
352.79998779296875,
226
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"slot_index": 0,
"links": [
165
]
}
],
"properties": {
"Node name for S&R": "HMVideoPipelineLoader"
},
"widgets_values": [
"[preset]RealisticVisionV60B1",
"None",
"same as checkpoint",
"v5",
"x1",
"huggingface",
1,
"fp32"
]
}
],
"links": [
meme.py
+21 -10
View File
@@ -123,10 +123,11 @@ class HMImagePipelineLoader:
"checkpoint": (checkpoint_files, ),
"lora": (lora_files, ),
"vae": (vae_files, ),
"version": (['v1', 'v2', 'v3', 'v4', 'v5'], ),
"version": (['v5', 'v4', 'v3', 'v2', 'v1'], ),
"stylize": (['x1', 'x2'], ),
"deployment": (['huggingface', 'modelscope'], ),
"lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}),
"dtype": (['fp32', 'fp16'], ),
}
}
RETURN_TYPES = ("HMIMAGEPIPELINE", )
@@ -134,8 +135,8 @@ class HMImagePipelineLoader:
FUNCTION = "load_pipeline"
CATEGORY = "hellomeme"
def load_pipeline(self, checkpoint=None, lora=None, vae=None,
version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0):
dtype = torch.float16
version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0, dtype='fp32'):
dtype = torch.float32 if dtype == 'fp32' else torch.float16
if deployment == 'modelscope':
from modelscope import snapshot_download
sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5')
@@ -167,10 +168,11 @@ class HMVideoPipelineLoader:
"checkpoint": (checkpoint_files, ),
"lora": (lora_files, ),
"vae": (vae_files, ),
"version": (['v1', 'v2', 'v3', 'v4', 'v5'], ),
"version": (['v5', 'v4', 'v3', 'v2', 'v1'], ),
"stylize": (['x1', 'x2'], ),
"deployment": (['huggingface', 'modelscope'], ),
"lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}),
"dtype": (['fp32', 'fp16'], ),
}
}
@@ -180,8 +182,8 @@ class HMVideoPipelineLoader:
CATEGORY = "hellomeme"
def load_pipeline(self, checkpoint=None, lora=None, vae=None,
version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0):
dtype = torch.float16
version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0, dtype='fp32'):
dtype = torch.float32 if dtype == 'fp32' else torch.float16
if deployment == 'modelscope':
from modelscope import snapshot_download
sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5')
@@ -220,7 +222,7 @@ class HMFaceToolkitsLoader:
CATEGORY = "hellomeme"
def load_face_toolkits(self, gpu_id, deployment='huggingface'):
dtype = torch.float16
dtype = torch.float32
face_toolkits = load_face_toolkits(dtype=dtype, gpu_id=gpu_id, modelscope=deployment=='modelscope')
return (face_toolkits, )
@@ -356,6 +358,7 @@ class HMPipelineImage:
gpu_id=0
):
device = get_torch_device(gpu_id)
dtype = hm_image_pipeline.dtype
image_np = cv2.cvtColor(ref_head_pose['frame'][0], cv2.COLOR_BGR2RGB)
image_pil = Image.fromarray(image_np)
@@ -364,8 +367,10 @@ class HMPipelineImage:
drive_rot, drive_trans = drive_head_pose['rot'], drive_head_pose['trans']
condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans, 512, trans_ratio)
drive_params = dict(condition=condition.unsqueeze(0).to(dtype=torch.float16, device='cpu'))
drive_params = dict(condition=condition.unsqueeze(0))
drive_params.update(drive_expression)
for k, v in drive_params.items():
drive_params[k] = v.to(dtype=dtype, device='cpu')
generator = torch.Generator().manual_seed(seed)
@@ -427,6 +432,7 @@ class HMPipelineVideo:
gpu_id=0
):
device = get_torch_device(gpu_id)
dtype = hm_video_pipeline.dtype
image_np = cv2.cvtColor(ref_head_pose['frame'][0], cv2.COLOR_BGR2RGB)
image_pil = Image.fromarray(image_np)
@@ -437,11 +443,16 @@ class HMPipelineVideo:
drive_rot, drive_trans = drive_head_pose['rot'], drive_head_pose['trans']
condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans[0], 512, trans_ratio)
ref_condition = gen_control_heatmaps(ref_rot, ref_trans, ref_trans[0], 512, 0.0)
drive_params = dict(condition=condition.unsqueeze(0).to(dtype=torch.float16, device='cpu'))
ref_params = dict(condition=ref_condition.unsqueeze(0).to(dtype=torch.float16, device='cpu'))
drive_params = dict(condition=condition.unsqueeze(0))
ref_params = dict(condition=ref_condition.unsqueeze(0))
drive_params.update(drive_expression)
ref_params.update(ref_expression)
for k, v in drive_params.items():
drive_params[k] = v.to(dtype=dtype)
for k, v in ref_params.items():
ref_params[k] = v.to(dtype=dtype, device='cpu')
res_frames, latents = hm_video_pipeline(
prompt=[prompt],
strength=1.0,
pyproject.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "comfyui_hellomeme"
description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface"
version = "1.3.5"
version = "1.3.6"
license = {file = "LICENSE"}
[project.urls]