nodes/HelloVision-ComfyUI_HelloMeme
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v5b and matryoshka-liked style transfer

Browse Source
This commit is contained in:
songkey-win
2025-06-12 18:49:11 +08:00
parent eccc304676
commit 458be6d5bf
11 changed files with 522 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6
View File
@@ -77,6 +77,12 @@ Keyword of ComfyUI Manager : `hellomeme-api`
<img src="example_workflows/hellomeme_image_workflow.jpg" alt="image_generation_interface">
</p>
### Style Transfer Interface
<p align="center">
<img src="example_workflows/hellomeme_style_workflow.jpg" alt="image_generation_interface">
</p>
### Video Generation Interface
<p align="center">
example_workflows/hellomeme_style_workflow.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 313 KiB

example_workflows/image_style_transfer.json
+437 -129
View File
@@ -1,47 +1,9 @@
{
"id": "a8e8ebf3-8dc8-4d76-ad73-dfe0b945cc77",
"revision": 0,
"last_node_id": 80,
"last_link_id": 190,
"last_node_id": 86,
"last_link_id": 205,
"nodes": [
{
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": [
45.41996383666992,
769.9866333007812
],
"size": [
315,
82
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"slot_index": 0,
"links": [
122,
129,
147,
149,
173,
174
]
}
],
"properties": {
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": [
0,
"huggingface"
]
},
{
"id": 61,
"type": "GetFaceLandmarks",
@@ -79,6 +41,8 @@
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetFaceLandmarks"
},
"widgets_values": []
@@ -120,6 +84,8 @@
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetFaceLandmarks"
},
"widgets_values": []
@@ -165,6 +131,8 @@
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetHeadPose"
},
"widgets_values": [
@@ -172,49 +140,46 @@
]
},
{
"id": 75,
"type": "GetExpression2",
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": [
814.1359252929688,
689.6795654296875
45.41996383666992,
769.9866333007812
],
"size": [
191.63729858398438,
66
315,
82
],
"flags": {},
"order": 6,
"order": 0,
"mode": 0,
"inputs": [
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 173
},
{
"name": "images",
"type": "IMAGE",
"link": 175
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 178
}
],
"outputs": [
{
"name": "expression",
"type": "EXPRESSION",
"slot_index": 0,
"links": [
184
122,
129,
147,
149,
173,
174,
193,
194
]
}
],
"properties": {
"Node name for S&R": "GetExpression2"
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": []
"widgets_values": [
0,
"huggingface"
]
},
{
"id": 69,
@@ -252,11 +217,14 @@
"name": "head_pose",
"type": "HEAD_POSE",
"links": [
185
185,
202
]
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetHeadPose"
},
"widgets_values": [
@@ -299,53 +267,18 @@
"name": "expression",
"type": "EXPRESSION",
"links": [
186
186,
203
]
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetExpression2"
},
"widgets_values": []
},
{
"id": 78,
"type": "HMImagePipelineLoader",
"pos": [
797.7930297851562,
188.84007263183594
],
"size": [
298.3472595214844,
226
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"links": [
182
]
}
],
"properties": {
"Node name for S&R": "HMImagePipelineLoader"
},
"widgets_values": [
"[preset]DisneyPixarCartoonB",
"[preset]BabyFaceV1",
"same as checkpoint",
"v5",
"x1",
"huggingface",
1,
"fp32"
]
},
{
"id": 79,
"type": "HMPipelineImage",
@@ -392,7 +325,9 @@
"name": "IMAGE",
"type": "IMAGE",
"links": [
187
187,
191,
192
]
},
{
@@ -402,6 +337,8 @@
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "HMPipelineImage"
},
"widgets_values": [
@@ -409,12 +346,279 @@
"",
"",
25,
485743939425196,
778376016259914,
"randomize",
1.5,
1.2000000000000002,
0
]
},
{
"id": 80,
"type": "PreviewImage",
"pos": [
1706.34228515625,
92.85418701171875
],
"size": [
426.67877197265625,
454.14093017578125
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 187
}
],
"outputs": [],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.39",
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 81,
"type": "GetFaceLandmarks",
"pos": [
1656.5469970703125,
756.1448364257812
],
"size": [
170.62362670898438,
46
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 193
},
{
"name": "images",
"type": "IMAGE",
"link": 191
}
],
"outputs": [
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"links": [
195
]
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetFaceLandmarks"
}
},
{
"id": 75,
"type": "GetExpression2",
"pos": [
814.1359252929688,
689.6795654296875
],
"size": [
191.63729858398438,
66
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 173
},
{
"name": "images",
"type": "IMAGE",
"link": 175
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 178
}
],
"outputs": [
{
"name": "expression",
"type": "EXPRESSION",
"links": [
184,
205
]
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetExpression2"
},
"widgets_values": []
},
{
"id": 82,
"type": "GetHeadPose",
"pos": [
1887.2037353515625,
718.1502075195312
],
"size": [
270,
98
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 194
},
{
"name": "images",
"type": "IMAGE",
"link": 192
},
{
"name": "landmarks",
"type": "FACELANDMARKS222",
"link": 195
}
],
"outputs": [
{
"name": "head_pose",
"type": "HEAD_POSE",
"links": [
200
]
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "GetHeadPose"
},
"widgets_values": [
false
]
},
{
"id": 85,
"type": "HMPipelineImage",
"pos": [
2279.40625,
704.7041015625
],
"size": [
270,
306
],
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"link": 199
},
{
"name": "ref_head_pose",
"type": "HEAD_POSE",
"link": 200
},
{
"name": "ref_expression",
"type": "EXPRESSION",
"link": 205
},
{
"name": "drive_head_pose",
"type": "HEAD_POSE",
"link": 202
},
{
"name": "drive_expression",
"type": "EXPRESSION",
"link": 203
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
204
]
},
{
"name": "LATENT",
"type": "LATENT",
"links": null
}
],
"properties": {
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "HMPipelineImage"
},
"widgets_values": [
0,
"",
"",
25,
661833773290484,
"randomize",
1.2000000000000002,
0
]
},
{
"id": 86,
"type": "PreviewImage",
"pos": [
2605.176025390625,
703.9863891601562
],
"size": [
535.2344360351562,
631.5162963867188
],
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 204
}
],
"outputs": [],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.39",
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 58,
"type": "LoadImage",
@@ -427,7 +631,7 @@
314
],
"flags": {},
"order": 2,
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
@@ -451,39 +655,55 @@
}
],
"properties": {
"cnr_id": "comfy-core",
"ver": "0.3.39",
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"chillout.jpg",
"hinton1.jpg",
"image"
]
},
{
"id": 80,
"type": "PreviewImage",
"id": 78,
"type": "HMImagePipelineLoader",
"pos": [
1703.6793212890625,
613.1461791992188
797.7930297851562,
188.84007263183594
],
"size": [
210,
258
298.3472595214844,
226
],
"flags": {},
"order": 10,
"order": 2,
"mode": 0,
"inputs": [
"inputs": [],
"outputs": [
{
"name": "images",
"type": "IMAGE",
"link": 187
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"links": [
182,
199
]
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
"cnr_id": "comfyui_hellomeme",
"ver": "56e8052c9b449621b029226988b0ed7d2dc96b4c",
"Node name for S&R": "HMImagePipelineLoader"
},
"widgets_values": []
"widgets_values": [
"[preset]toonyou_beta6",
"[preset]Drawing",
"same as checkpoint",
"v5b",
"x1",
"huggingface",
1,
"fp32"
]
}
],
"links": [
@@ -662,6 +882,94 @@
76,
1,
"IMAGE"
],
[
191,
79,
0,
81,
1,
"IMAGE"
],
[
192,
79,
0,
82,
1,
"IMAGE"
],
[
193,
2,
0,
81,
0,
"FACE_TOOLKITS"
],
[
194,
2,
0,
82,
0,
"FACE_TOOLKITS"
],
[
195,
81,
0,
82,
2,
"FACELANDMARKS222"
],
[
199,
78,
0,
85,
0,
"HMIMAGEPIPELINE"
],
[
200,
82,
0,
85,
1,
"HEAD_POSE"
],
[
202,
69,
0,
85,
3,
"HEAD_POSE"
],
[
203,
76,
0,
85,
4,
"EXPRESSION"
],
[
204,
85,
0,
86,
0,
"IMAGE"
],
[
205,
75,
0,
85,
2,
"EXPRESSION"
]
],
"groups": [],
@@ -670,8 +978,8 @@
"ds": {
"scale": 0.8264462809917358,
"offset": [
496.5037384564152,
-71.873413496035
148.58950276642736,
102.78195962059476
]
},
"frontendVersion": "1.19.2"
hellomeme/model_config.json
+2 -1
View File
@@ -22,5 +22,6 @@
"[preset]Drawing": ["songkey/loras_sd_1_5", "Drawing.safetensors"]
}
},
"prompt": "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture"
"prompt": "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"prompt_new": "best quality, highly detailed, masterpiece, person, portrait, headshot, looking at the viewer, sharp focus."
}
hellomeme/models/__init__.py
+1 -1
View File
@@ -12,7 +12,7 @@ from .hm_denoising_motion import HMDenoisingMotion
from .hm_control import (HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2,
HMV3ControlNet, HMControlNetBase, HM5ControlNetBase,
HM4SD15ControlProj, HM5SD15ControlProj)
from .hm_adapters import (HMReferenceAdapter, HM3ReferenceAdapter, HM5ReferenceAdapter,
from .hm_adapters import (HMReferenceAdapter, HM3ReferenceAdapter, HM5ReferenceAdapter, HM5bReferenceAdapter,
HM3MotionAdapter, HM5MotionAdapter, HMPipeline)
from .hm_denoising_3d import HMDenoising3D
from .hm3_denoising_3d import HM3Denoising3D
hellomeme/models/hm_adapters.py
+19
View File
@@ -41,6 +41,7 @@ from diffusers.models.lora import adjust_lora_scale_text_encoder
from .hm_blocks import (SKReferenceAttentionV3,
SKReferenceAttentionV5,
STKReferenceModule,
SKReferenceAttention,
SKMotionModule,
SKMotionModuleV5)
@@ -136,6 +137,24 @@ class HM3ReferenceAdapter(ModelMixin, ConfigMixin):
)
class HM5bReferenceAdapter(ModelMixin, ConfigMixin):
@register_to_config
def __init__(self, block_down_channels: Tuple[int] = (320, 640, 1280, 1280),
block_up_channels: Tuple[int] = (1280, 1280, 1280, 640),
num_attention_heads: int = 8,
use_3d: bool = False):
super().__init__()
self.reference_modules_up = nn.ModuleList([])
for i, in_channels in enumerate(block_up_channels):
self.reference_modules_up.append(
STKReferenceModule(
in_channels=in_channels,
num_attention_heads=num_attention_heads,
)
)
class HM5ReferenceAdapter(ModelMixin, ConfigMixin):
@register_to_config
def __init__(self, block_down_channels: Tuple[int] = (320, 640, 1280, 1280),
hellomeme/models/hm_blocks.py
+36 -2
View File
@@ -191,6 +191,38 @@ class SKReferenceAttentionV5(nn.Module):
return hidden_states + self.proj(res2.contiguous())
class STKReferenceModule(nn.Module):
def __init__(
self,
in_channels: int,
num_attention_heads: int = 1,
norm_elementwise_affine: bool = True,
norm_eps: float = 1e-5,
):
super().__init__()
self.attn_ref = STKAttentionV5(in_channels=in_channels,
mid_channels=in_channels,
num_attention_heads=num_attention_heads,
time_embed_dim=None,
norm_elementwise_affine=norm_elementwise_affine,
norm_eps=norm_eps,
temporal_attn=False,)
self.attn = STKAttentionV5(in_channels=in_channels,
mid_channels=in_channels,
num_attention_heads=num_attention_heads,
time_embed_dim=None,
norm_elementwise_affine=norm_elementwise_affine,
norm_eps=norm_eps,
temporal_attn=False,)
def forward(self, hidden_states, ref_stats, num_frames):
if ref_stats.shape[0] != hidden_states.shape[0]:
ref_stats = ref_stats.repeat_interleave(num_frames, dim=0)
hidden_states = rearrange(hidden_states, "(b f) c h w -> b f h w c", f=num_frames)
ref_stats = rearrange(ref_stats, "(b f) c h w -> b f h w c", f=num_frames)
ref_stats = self.attn_ref(ref_stats, residual=False)
hidden_states = self.attn(hidden_states, ref_stats, residual=True)
hidden_states = rearrange(hidden_states, "b f h w c -> (b f) c h w")
return hidden_states
class SmallUnet(nn.Module):
def __init__(self, in_channels: int = 3,
@@ -380,7 +412,7 @@ class STKAttentionV5(nn.Module):
self.norm = nn.LayerNorm(mid_channels, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
self.proj = zero_module(nn.Conv2d(mid_channels, in_channels, kernel_size=3, padding=1))
def forward(self, hidden_states, ref_stats=None, emb=None):
def forward(self, hidden_states, ref_stats=None, emb=None, residual=True):
b, f, h, w, _ = hidden_states.shape
hidden_states_in = rearrange(hidden_states, "b f h w c -> (b f) c h w")
@@ -433,7 +465,9 @@ class STKAttentionV5(nn.Module):
else:
rest = rearrange(resy, "b f h w c -> (b f) c h w")
return hidden_states + rearrange(self.proj(rest), "(b f) c h w -> b f h w c", f=f)
rest = rearrange(self.proj(rest), "(b f) c h w -> b f h w c", f=f)
return hidden_states + rest if residual else rest
class STKCrossAttentionV5(nn.Module):
def __init__(
hellomeme/pipelines/pipline_hm5_image.py
+14 -8
View File
@@ -20,10 +20,8 @@ from diffusers.utils.torch_utils import randn_tensor
from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput
from diffusers import DPMSolverMultistepScheduler
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents
from ..models import (HM3Denoising3D,
HMPipeline, HM5ReferenceAdapter,
HM5ControlNetBase,
HM5SD15ControlProj)
from ..models import (HM3Denoising3D, HMPipeline, HM5ReferenceAdapter,
HM5bReferenceAdapter, HM5ControlNetBase, HM5SD15ControlProj)
class HM5ImagePipeline(HMPipeline):
def caryomitosis(self, **kwargs):
@@ -46,20 +44,28 @@ class HM5ImagePipeline(HMPipeline):
self.safety_checker.cpu()
def insert_hm_modules(self, version='v5', dtype=torch.float16, modelscope=False):
self.version = version
if modelscope:
from modelscope import snapshot_download
hm_reference_dir = snapshot_download('songkey/hm5_reference')
if version == 'v5b':
hm_reference_dir = snapshot_download('songkey/hm5b_reference')
else:
hm_reference_dir = snapshot_download('songkey/hm5_reference')
hm_control_dir = snapshot_download('songkey/hm5_control_base')
hm_control_proj_dir = snapshot_download('songkey/hm5_control_proj')
else:
hm_reference_dir = 'songkey/hm5_reference'
if version == 'v5b':
hm_reference_dir = 'songkey/hm5b_reference'
else:
hm_reference_dir = 'songkey/hm5_reference'
hm_control_dir = 'songkey/hm5_control_base'
hm_control_proj_dir = 'songkey/hm5_control_proj'
if isinstance(self.unet, HM3Denoising3D):
hm_adapter = HM5ReferenceAdapter.from_pretrained(hm_reference_dir)
if version == 'v5b':
hm_adapter = HM5bReferenceAdapter.from_pretrained(hm_reference_dir)
else:
hm_adapter = HM5ReferenceAdapter.from_pretrained(hm_reference_dir)
self.unet.insert_reference_adapter(hm_adapter)
self.unet.to(device='cpu', dtype=dtype).eval()
hellomeme/pipelines/pipline_hm_image.py
+1
View File
@@ -45,6 +45,7 @@ class HMImagePipeline(HMPipeline):
self.safety_checker.cpu()
def insert_hm_modules(self, version, dtype, modelscope=False):
self.version = version
if modelscope:
from modelscope import snapshot_download
hm_reference_dir = snapshot_download('songkey/hm_reference')
meme.py
+5 -3
View File
@@ -59,6 +59,7 @@ with open(config_path, 'r') as f:
MODEL_CONFIG = json.load(f)
DEFAULT_PROMPT = MODEL_CONFIG['prompt']
DEFAULT_PROMPT_NEW = MODEL_CONFIG['prompt_new']
def get_models_files():
checkpoint_files = folder_paths.get_filename_list("checkpoints")
@@ -123,7 +124,7 @@ class HMImagePipelineLoader:
"checkpoint": (checkpoint_files, ),
"lora": (lora_files, ),
"vae": (vae_files, ),
"version": (['v5', 'v4', 'v3', 'v2', 'v1'], ),
"version": (['v5b', 'v5', 'v4', 'v3', 'v2', 'v1'], ),
"stylize": (['x1', 'x2'], ),
"deployment": (['huggingface', 'modelscope'], ),
"lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}),
@@ -144,7 +145,7 @@ class HMImagePipelineLoader:
sd1_5_dir = "songkey/stable-diffusion-v1-5"
if version == 'v3' or version == 'v4':
pipeline = HM3ImagePipeline.from_pretrained(sd1_5_dir)
elif version == 'v5':
elif version == 'v5' or version == 'v5b':
pipeline = HM5ImagePipeline.from_pretrained(sd1_5_dir)
else:
pipeline = HMImagePipeline.from_pretrained(sd1_5_dir)
@@ -360,7 +361,8 @@ class HMPipelineImage:
device = get_torch_device(gpu_id)
dtype = hm_image_pipeline.dtype
prompt = DEFAULT_PROMPT if prompt == '' else prompt + ", " + DEFAULT_PROMPT
PROMPT = DEFAULT_PROMPT_NEW if hm_image_pipeline.version == 'v5b' else DEFAULT_PROMPT
prompt = PROMPT if prompt == '' else prompt + ", " + PROMPT
image_np = cv2.cvtColor(ref_head_pose['frame'][0], cv2.COLOR_BGR2RGB)
image_pil = Image.fromarray(image_np)
pyproject.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "comfyui_hellomeme"
description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface"
version = "1.3.12"
version = "1.3.13"
license = {file = "LICENSE"}
[project.urls]