From 1142e98655d77dd77d1588c9e524096c60426c7f Mon Sep 17 00:00:00 2001 From: songkey-win Date: Tue, 3 Jun 2025 13:41:37 +0800 Subject: [PATCH] fp32 supported --- example_workflows/image_generation.json | 896 +++++++++++++++++++----- example_workflows/video_generation.json | 83 +-- meme.py | 31 +- pyproject.toml | 2 +- 4 files changed, 767 insertions(+), 245 deletions(-) diff --git a/example_workflows/image_generation.json b/example_workflows/image_generation.json index 5cd7ef4..2925fe3 100644 --- a/example_workflows/image_generation.json +++ b/example_workflows/image_generation.json @@ -1,206 +1,716 @@ { - "2": { - "inputs": { - "gpu_id": 0, - "deployment": "huggingface" - }, - "class_type": "HMFaceToolkitsLoader", - "_meta": { - "title": "Load Face Toolkits" - } - }, - "58": { - "inputs": { - "image": "chillout.jpg" - }, - "class_type": "LoadImage", - "_meta": { - "title": "加载图像" - } - }, - "60": { - "inputs": { - "face_toolkits": [ - "2", - 0 + "id": "a8e8ebf3-8dc8-4d76-ad73-dfe0b945cc77", + "revision": 0, + "last_node_id": 80, + "last_link_id": 187, + "nodes": [ + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": [ + 45.41996383666992, + 769.9866333007812 ], - "images": [ - "58", + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "slot_index": 0, + "links": [ + 122, + 129, + 147, + 149, + 173, + 174 + ] + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0, + "huggingface" + ] + }, + { + "id": 61, + "type": "GetFaceLandmarks", + "pos": [ + 519.7196655273438, + 938.032958984375 + ], + "size": [ + 170.62362670898438, + 46 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 129 + }, + { + "name": "images", + "type": "IMAGE", + "link": 179 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 145, + 177 + ] + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 60, + "type": "GetFaceLandmarks", + "pos": [ + 518.9132690429688, + 612.9461669921875 + ], + "size": [ + 170.62362670898438, + 46 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 122 + }, + { + "name": "images", + "type": "IMAGE", + "link": 121 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 143, + 178 + ] + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 77, + "type": "LoadImage", + "pos": [ + 61.306365966796875, + 1023.740478515625 + ], + "size": [ + 270, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 179, + 180, + 181 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "yao.jpg", + "image" + ] + }, + { + "id": 67, + "type": "GetHeadPose", + "pos": [ + 814.1528930664062, + 495.9793701171875 + ], + "size": [ + 270, + 98 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 147 + }, + { + "name": "images", + "type": "IMAGE", + "link": 151 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 143 + } + ], + "outputs": [ + { + "name": "head_pose", + "type": "HEAD_POSE", + "links": [ + 183 + ] + } + ], + "properties": { + "Node name for S&R": "GetHeadPose" + }, + "widgets_values": [ + true + ] + }, + { + "id": 75, + "type": "GetExpression2", + "pos": [ + 814.1359252929688, + 689.6795654296875 + ], + "size": [ + 191.63729858398438, + 66 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 173 + }, + { + "name": "images", + "type": "IMAGE", + "link": 175 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 178 + } + ], + "outputs": [ + { + "name": "expression", + "type": "EXPRESSION", + "links": [ + 184 + ] + } + ], + "properties": { + "Node name for S&R": "GetExpression2" + }, + "widgets_values": [] + }, + { + "id": 69, + "type": "GetHeadPose", + "pos": [ + 819.799560546875, + 860.5927734375 + ], + "size": [ + 270, + 98 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 149 + }, + { + "name": "images", + "type": "IMAGE", + "link": 180 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 145 + } + ], + "outputs": [ + { + "name": "head_pose", + "type": "HEAD_POSE", + "links": [ + 185 + ] + } + ], + "properties": { + "Node name for S&R": "GetHeadPose" + }, + "widgets_values": [ + true + ] + }, + { + "id": 76, + "type": "GetExpression2", + "pos": [ + 833.6573486328125, + 1042.0396728515625 + ], + "size": [ + 191.63729858398438, + 66 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 174 + }, + { + "name": "images", + "type": "IMAGE", + "link": 181 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 177 + } + ], + "outputs": [ + { + "name": "expression", + "type": "EXPRESSION", + "links": [ + 186 + ] + } + ], + "properties": { + "Node name for S&R": "GetExpression2" + }, + "widgets_values": [] + }, + { + "id": 79, + "type": "HMPipelineImage", + "pos": [ + 1334.226318359375, + 613.9535522460938 + ], + "size": [ + 270, + 306 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "link": 182 + }, + { + "name": "ref_head_pose", + "type": "HEAD_POSE", + "link": 183 + }, + { + "name": "ref_expression", + "type": "EXPRESSION", + "link": 184 + }, + { + "name": "drive_head_pose", + "type": "HEAD_POSE", + "link": 185 + }, + { + "name": "drive_expression", + "type": "EXPRESSION", + "link": 186 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 187 + ] + }, + { + "name": "LATENT", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "HMPipelineImage" + }, + "widgets_values": [ + 0, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 815482659530374, + "randomize", + 2, 0 ] }, - "class_type": "GetFaceLandmarks", - "_meta": { - "title": "Get Face Landmarks" - } - }, - "61": { - "inputs": { - "face_toolkits": [ - "2", - 0 + { + "id": 58, + "type": "LoadImage", + "pos": [ + 37.4466552734375, + 260.9000244140625 ], - "images": [ - "77", - 0 + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 121, + 151, + 175 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "chillout.jpg", + "image" ] }, - "class_type": "GetFaceLandmarks", - "_meta": { - "title": "Get Face Landmarks" + { + "id": 80, + "type": "PreviewImage", + "pos": [ + 1703.6793212890625, + 613.9529418945312 + ], + "size": [ + 210, + 258 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 187 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 78, + "type": "HMImagePipelineLoader", + "pos": [ + 797.7930297851562, + 188.84007263183594 + ], + "size": [ + 298.3472595214844, + 226 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "links": [ + 182 + ] + } + ], + "properties": { + "Node name for S&R": "HMImagePipelineLoader" + }, + "widgets_values": [ + "[preset]DisneyPixarCartoonB", + "None", + "same as checkpoint", + "v5", + "x1", + "huggingface", + 1, + "fp32" + ] } - }, - "67": { - "inputs": { - "crop": true, - "face_toolkits": [ - "2", - 0 - ], - "images": [ - "58", - 0 - ], - "landmarks": [ - "60", - 0 + ], + "links": [ + [ + 121, + 58, + 0, + 60, + 1, + "IMAGE" + ], + [ + 122, + 2, + 0, + 60, + 0, + "FACE_TOOLKITS" + ], + [ + 129, + 2, + 0, + 61, + 0, + "FACE_TOOLKITS" + ], + [ + 143, + 60, + 0, + 67, + 2, + "FACELANDMARKS222" + ], + [ + 145, + 61, + 0, + 69, + 2, + "FACELANDMARKS222" + ], + [ + 147, + 2, + 0, + 67, + 0, + "FACE_TOOLKITS" + ], + [ + 149, + 2, + 0, + 69, + 0, + "FACE_TOOLKITS" + ], + [ + 151, + 58, + 0, + 67, + 1, + "IMAGE" + ], + [ + 173, + 2, + 0, + 75, + 0, + "FACE_TOOLKITS" + ], + [ + 174, + 2, + 0, + 76, + 0, + "FACE_TOOLKITS" + ], + [ + 175, + 58, + 0, + 75, + 1, + "IMAGE" + ], + [ + 177, + 61, + 0, + 76, + 2, + "FACELANDMARKS222" + ], + [ + 178, + 60, + 0, + 75, + 2, + "FACELANDMARKS222" + ], + [ + 179, + 77, + 0, + 61, + 1, + "IMAGE" + ], + [ + 180, + 77, + 0, + 69, + 1, + "IMAGE" + ], + [ + 181, + 77, + 0, + 76, + 1, + "IMAGE" + ], + [ + 182, + 78, + 0, + 79, + 0, + "HMIMAGEPIPELINE" + ], + [ + 183, + 67, + 0, + 79, + 1, + "HEAD_POSE" + ], + [ + 184, + 75, + 0, + 79, + 2, + "EXPRESSION" + ], + [ + 185, + 69, + 0, + 79, + 3, + "HEAD_POSE" + ], + [ + 186, + 76, + 0, + 79, + 4, + "EXPRESSION" + ], + [ + 187, + 79, + 0, + 80, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917358, + "offset": [ + 493.27712102477454, + -69.45337656976547 ] }, - "class_type": "GetHeadPose", - "_meta": { - "title": "Get Head Pose" - } + "frontendVersion": "1.19.2" }, - "69": { - "inputs": { - "crop": true, - "face_toolkits": [ - "2", - 0 - ], - "images": [ - "77", - 0 - ], - "landmarks": [ - "61", - 0 - ] - }, - "class_type": "GetHeadPose", - "_meta": { - "title": "Get Head Pose" - } - }, - "75": { - "inputs": { - "face_toolkits": [ - "2", - 0 - ], - "images": [ - "58", - 0 - ], - "landmarks": [ - "60", - 0 - ] - }, - "class_type": "GetExpression2", - "_meta": { - "title": "Get Face Expression V2" - } - }, - "76": { - "inputs": { - "face_toolkits": [ - "2", - 0 - ], - "images": [ - "77", - 0 - ], - "landmarks": [ - "61", - 0 - ] - }, - "class_type": "GetExpression2", - "_meta": { - "title": "Get Face Expression V2" - } - }, - "77": { - "inputs": { - "image": "yao.jpg" - }, - "class_type": "LoadImage", - "_meta": { - "title": "加载图像" - } - }, - "78": { - "inputs": { - "checkpoint": "[preset]DisneyPixarCartoonB", - "lora": "None", - "vae": "same as checkpoint", - "version": "v5", - "stylize": "x1", - "deployment": "huggingface", - "lora_scale": 1 - }, - "class_type": "HMImagePipelineLoader", - "_meta": { - "title": "Load HelloMemeImage Pipeline" - } - }, - "79": { - "inputs": { - "trans_ratio": 0, - "prompt": "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "negative_prompt": "", - "steps": 25, - "seed": 815482659530374, - "guidance_scale": 2, - "gpu_id": 0, - "hm_image_pipeline": [ - "78", - 0 - ], - "ref_head_pose": [ - "67", - 0 - ], - "ref_expression": [ - "75", - 0 - ], - "drive_head_pose": [ - "69", - 0 - ], - "drive_expression": [ - "76", - 0 - ] - }, - "class_type": "HMPipelineImage", - "_meta": { - "title": "HelloMeme Image Pipeline" - } - }, - "80": { - "inputs": { - "images": [ - "79", - 0 - ] - }, - "class_type": "PreviewImage", - "_meta": { - "title": "预览图像" - } - } + "version": 0.4 } \ No newline at end of file diff --git a/example_workflows/video_generation.json b/example_workflows/video_generation.json index 4a90c4b..be18b14 100644 --- a/example_workflows/video_generation.json +++ b/example_workflows/video_generation.json @@ -404,44 +404,6 @@ true ] }, - { - "id": 57, - "type": "HMVideoPipelineLoader", - "pos": [ - 798.4796752929688, - 193.7666015625 - ], - "size": [ - 352.79998779296875, - 202 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "slot_index": 0, - "links": [ - 165 - ] - } - ], - "properties": { - "Node name for S&R": "HMVideoPipelineLoader" - }, - "widgets_values": [ - "[preset]RealisticVisionV60B1", - "None", - "same as checkpoint", - "v5", - "x1", - "huggingface", - 1 - ] - }, { "id": 48, "type": "VHS_LoadVideo", @@ -451,10 +413,10 @@ ], "size": [ 247.455078125, - 503.455078125 + 262 ], "flags": {}, - "order": 2, + "order": 1, "mode": 0, "inputs": [ { @@ -541,7 +503,7 @@ 314 ], "flags": {}, - "order": 3, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -615,6 +577,45 @@ "widgets_values": [ true ] + }, + { + "id": 57, + "type": "HMVideoPipelineLoader", + "pos": [ + 798.4796752929688, + 193.7666015625 + ], + "size": [ + 352.79998779296875, + 226 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "slot_index": 0, + "links": [ + 165 + ] + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "[preset]RealisticVisionV60B1", + "None", + "same as checkpoint", + "v5", + "x1", + "huggingface", + 1, + "fp32" + ] } ], "links": [ diff --git a/meme.py b/meme.py index 81335fc..faeb666 100644 --- a/meme.py +++ b/meme.py @@ -123,10 +123,11 @@ class HMImagePipelineLoader: "checkpoint": (checkpoint_files, ), "lora": (lora_files, ), "vae": (vae_files, ), - "version": (['v1', 'v2', 'v3', 'v4', 'v5'], ), + "version": (['v5', 'v4', 'v3', 'v2', 'v1'], ), "stylize": (['x1', 'x2'], ), "deployment": (['huggingface', 'modelscope'], ), "lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), + "dtype": (['fp32', 'fp16'], ), } } RETURN_TYPES = ("HMIMAGEPIPELINE", ) @@ -134,8 +135,8 @@ class HMImagePipelineLoader: FUNCTION = "load_pipeline" CATEGORY = "hellomeme" def load_pipeline(self, checkpoint=None, lora=None, vae=None, - version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0): - dtype = torch.float16 + version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0, dtype='fp32'): + dtype = torch.float32 if dtype == 'fp32' else torch.float16 if deployment == 'modelscope': from modelscope import snapshot_download sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5') @@ -167,10 +168,11 @@ class HMVideoPipelineLoader: "checkpoint": (checkpoint_files, ), "lora": (lora_files, ), "vae": (vae_files, ), - "version": (['v1', 'v2', 'v3', 'v4', 'v5'], ), + "version": (['v5', 'v4', 'v3', 'v2', 'v1'], ), "stylize": (['x1', 'x2'], ), "deployment": (['huggingface', 'modelscope'], ), "lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), + "dtype": (['fp32', 'fp16'], ), } } @@ -180,8 +182,8 @@ class HMVideoPipelineLoader: CATEGORY = "hellomeme" def load_pipeline(self, checkpoint=None, lora=None, vae=None, - version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0): - dtype = torch.float16 + version='v2', stylize='x1', deployment='huggingface', lora_scale=1.0, dtype='fp32'): + dtype = torch.float32 if dtype == 'fp32' else torch.float16 if deployment == 'modelscope': from modelscope import snapshot_download sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5') @@ -220,7 +222,7 @@ class HMFaceToolkitsLoader: CATEGORY = "hellomeme" def load_face_toolkits(self, gpu_id, deployment='huggingface'): - dtype = torch.float16 + dtype = torch.float32 face_toolkits = load_face_toolkits(dtype=dtype, gpu_id=gpu_id, modelscope=deployment=='modelscope') return (face_toolkits, ) @@ -356,6 +358,7 @@ class HMPipelineImage: gpu_id=0 ): device = get_torch_device(gpu_id) + dtype = hm_image_pipeline.dtype image_np = cv2.cvtColor(ref_head_pose['frame'][0], cv2.COLOR_BGR2RGB) image_pil = Image.fromarray(image_np) @@ -364,8 +367,10 @@ class HMPipelineImage: drive_rot, drive_trans = drive_head_pose['rot'], drive_head_pose['trans'] condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans, 512, trans_ratio) - drive_params = dict(condition=condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')) + drive_params = dict(condition=condition.unsqueeze(0)) drive_params.update(drive_expression) + for k, v in drive_params.items(): + drive_params[k] = v.to(dtype=dtype, device='cpu') generator = torch.Generator().manual_seed(seed) @@ -427,6 +432,7 @@ class HMPipelineVideo: gpu_id=0 ): device = get_torch_device(gpu_id) + dtype = hm_video_pipeline.dtype image_np = cv2.cvtColor(ref_head_pose['frame'][0], cv2.COLOR_BGR2RGB) image_pil = Image.fromarray(image_np) @@ -437,11 +443,16 @@ class HMPipelineVideo: drive_rot, drive_trans = drive_head_pose['rot'], drive_head_pose['trans'] condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans[0], 512, trans_ratio) ref_condition = gen_control_heatmaps(ref_rot, ref_trans, ref_trans[0], 512, 0.0) - drive_params = dict(condition=condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')) - ref_params = dict(condition=ref_condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')) + drive_params = dict(condition=condition.unsqueeze(0)) + ref_params = dict(condition=ref_condition.unsqueeze(0)) drive_params.update(drive_expression) ref_params.update(ref_expression) + for k, v in drive_params.items(): + drive_params[k] = v.to(dtype=dtype) + for k, v in ref_params.items(): + ref_params[k] = v.to(dtype=dtype, device='cpu') + res_frames, latents = hm_video_pipeline( prompt=[prompt], strength=1.0, diff --git a/pyproject.toml b/pyproject.toml index 89a66c0..7c61fba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_hellomeme" description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface" -version = "1.3.5" +version = "1.3.6" license = {file = "LICENSE"} [project.urls]