nodes/HelloVision-ComfyUI_HelloMeme
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update interface

Browse Source
This commit is contained in:
songkey-win
2024-11-12 12:50:40 +08:00
parent caeb3972fd
commit d90961e2e3
8 changed files with 959 additions and 955 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hellomeme/tools/hello_3dmm.py
+54 -54
View File
@@ -73,68 +73,68 @@ class Hello3DMMPred(object):
focal_length=self.used_focal_len)
return angles, crop_global_transl[np.newaxis, :]
def compute_rotation_matrix(self, angles):
n_b = angles.shape[0]
sinx = np.sin(angles[:, 0])
siny = np.sin(angles[:, 1])
sinz = np.sin(angles[:, 2])
cosx = np.cos(angles[:, 0])
cosy = np.cos(angles[:, 1])
cosz = np.cos(angles[:, 2])
rotXYZ = np.eye(3).reshape(1, 3, 3).repeat(n_b*3, 0).reshape(3, n_b, 3, 3)
rotXYZ[0, :, 1, 1] = cosx
rotXYZ[0, :, 1, 2] = -sinx
rotXYZ[0, :, 2, 1] = sinx
rotXYZ[0, :, 2, 2] = cosx
rotXYZ[1, :, 0, 0] = cosy
rotXYZ[1, :, 0, 2] = siny
rotXYZ[1, :, 2, 0] = -siny
rotXYZ[1, :, 2, 2] = cosy
rotXYZ[2, :, 0, 0] = cosz
rotXYZ[2, :, 0, 1] = -sinz
rotXYZ[2, :, 1, 0] = sinz
rotXYZ[2, :, 1, 1] = cosz
rotation = np.matmul(np.matmul(rotXYZ[2], rotXYZ[1]), rotXYZ[0])
return rotation.transpose(0, 2, 1)
def compute_rotation_matrix(angles):
n_b = angles.shape[0]
sinx = np.sin(angles[:, 0])
siny = np.sin(angles[:, 1])
sinz = np.sin(angles[:, 2])
cosx = np.cos(angles[:, 0])
cosy = np.cos(angles[:, 1])
cosz = np.cos(angles[:, 2])
rotXYZ = np.eye(3).reshape(1, 3, 3).repeat(n_b*3, 0).reshape(3, n_b, 3, 3)
rotXYZ[0, :, 1, 1] = cosx
rotXYZ[0, :, 1, 2] = -sinx
rotXYZ[0, :, 2, 1] = sinx
rotXYZ[0, :, 2, 2] = cosx
rotXYZ[1, :, 0, 0] = cosy
rotXYZ[1, :, 0, 2] = siny
rotXYZ[1, :, 2, 0] = -siny
rotXYZ[1, :, 2, 2] = cosy
rotXYZ[2, :, 0, 0] = cosz
rotXYZ[2, :, 0, 1] = -sinz
rotXYZ[2, :, 1, 0] = sinz
rotXYZ[2, :, 1, 1] = cosz
rotation = np.matmul(np.matmul(rotXYZ[2], rotXYZ[1]), rotXYZ[0])
return rotation.transpose(0, 2, 1)
def rigid_transform(self, vs, rot, trans):
vs_r = np.matmul(vs, rot)
vs_t = vs_r + trans.reshape(-1, 1, 3)
return vs_t
def rigid_transform(vs, rot, trans):
vs_r = np.matmul(vs, rot)
vs_t = vs_r + trans.reshape(-1, 1, 3)
return vs_t
def perspective_projection_points(self, points, image_w, image_h, used_focal_len):
batch_size = points.shape[0]
K = np.zeros([batch_size, 3, 3])
K[:, 0, 0] = used_focal_len
K[:, 1, 1] = used_focal_len
K[:, 2, 2] = 1.
K[:, 0, 2] = image_w * 0.5
K[:, 1, 2] = image_h * 0.5
def perspective_projection_points(points, image_w, image_h, used_focal_len):
batch_size = points.shape[0]
K = np.zeros([batch_size, 3, 3])
K[:, 0, 0] = used_focal_len
K[:, 1, 1] = used_focal_len
K[:, 2, 2] = 1.
K[:, 0, 2] = image_w * 0.5
K[:, 1, 2] = image_h * 0.5
reverse_z = np.array([[1, 0, 0], [0, 1, 0], [0, 0, -1]])[np.newaxis, :, :].repeat(batch_size, 0)
reverse_z = np.array([[1, 0, 0], [0, 1, 0], [0, 0, -1]])[np.newaxis, :, :].repeat(batch_size, 0)
# Transform points
aug_projection = np.matmul(points, reverse_z)
aug_projection = np.matmul(aug_projection, K.transpose((0, 2, 1)))
# Transform points
aug_projection = np.matmul(points, reverse_z)
aug_projection = np.matmul(aug_projection, K.transpose((0, 2, 1)))
# Apply perspective distortion
projected_points = aug_projection[:, :, :2] / aug_projection[:, :, 2:]
return projected_points
# Apply perspective distortion
projected_points = aug_projection[:, :, :2] / aug_projection[:, :, 2:]
return projected_points
def get_project_points_rect(self, angle, trans, image_w, image_h):
vs = np.array(
[[-1, -1, 0], [-1, 1, 0], [1, 1, 0], [1, -1, 0]],
) * 0.05
vs = vs[np.newaxis, :, :]
def get_project_points_rect(angle, trans, image_w, image_h, used_focal_len=4828.598903609935):
vs = np.array(
[[-1, -1, 0], [-1, 1, 0], [1, 1, 0], [1, -1, 0]],
) * 0.05
vs = vs[np.newaxis, :, :]
rotation = self.compute_rotation_matrix(angle)
translation = trans.copy()
translation[0, 2] *= 0.05
rotation = compute_rotation_matrix(angle)
translation = trans.copy()
translation[0, 2] *= 0.05
vs_t = self.rigid_transform(vs, rotation, translation)
vs_t = rigid_transform(vs, rotation, translation)
project_points = self.perspective_projection_points(vs_t, image_w, image_h, self.used_focal_len*0.05)
project_points = np.stack([project_points[:, :, 0], image_h - project_points[:, :, 1]], axis=2)
project_points = perspective_projection_points(vs_t, image_w, image_h, used_focal_len*0.05)
project_points = np.stack([project_points[:, :, 0], image_h - project_points[:, :, 1]], axis=2)
return project_points[0]
return project_points[0]
hellomeme/utils.py
+6 -7
View File
@@ -17,6 +17,7 @@ from PIL import Image
import subprocess
from einops import rearrange
from .tools.utils import transform_points
from .tools.hello_3dmm import get_project_points_rect
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_unet_checkpoint
from safetensors import safe_open
@@ -196,7 +197,7 @@ def get_face_params(h3dmm, harkit_bs, frames, landmarks, save_size=(512, 512), a
drive_coeff = torch.from_numpy(np.stack(arkit_bs_list, axis=0))
return face_parts, drive_coeff, rot_list, trans_list
def face_params_to_tensor(clip_encoder, h3dmm, face_parts, drive_rot, drive_trans, ref_trans, save_size=512, trans_ratio=0.0):
def face_params_to_tensor(clip_encoder, face_parts):
face_parts_list = []
for tmp_face_parts_list in face_parts:
tmp_face_parts_list = [clip_preprocess_from_bgr(x).unsqueeze(1) for x in tmp_face_parts_list]
@@ -216,19 +217,17 @@ def face_params_to_tensor(clip_encoder, h3dmm, face_parts, drive_rot, drive_tran
face_parts_embedding = rearrange(face_parts_embedding,
"(f p) c -> f p c",
f=face_parts.size(2))
return face_parts_embedding
def gen_control_heatmaps(drive_rot, drive_trans, ref_trans, save_size=512, trans_ratio=0.0):
control_list = []
control_show_list = []
for rot, trans in zip(drive_rot, drive_trans):
rect = h3dmm.get_project_points_rect(rot, ref_trans + (trans - drive_trans[0]) * trans_ratio, save_size, save_size)
rect = get_project_points_rect(rot, ref_trans + (trans - drive_trans[0]) * trans_ratio, save_size, save_size)
control_heatmap = draw_skl_by_rect(save_size, rect)
control_show_list.append(control_heatmap.copy())
control_heatmap = image_preprocess(control_heatmap, (save_size, save_size))
control_list.append(control_heatmap)
control_heatmaps = torch.stack(control_list, dim=1)
return face_parts_embedding, control_heatmaps
return torch.stack(control_list, dim=1)
def det_landmarks(face_aligner, frame_list, save_size=(512, 512), reset=False):
rect_list = []
meme.py
+34 -25
View File
@@ -2,12 +2,10 @@ import os
import os.path as osp
import random
import folder_paths
import torch
import numpy as np
import cv2
import sys
import copy
from PIL import Image
@@ -31,11 +29,13 @@ if missing:
subprocess.check_call([python, '-m', 'pip', 'install', *missing], stdout=subprocess.DEVNULL)
from .hellomeme.utils import (face_params_to_tensor,
get_drive_params,
crop_and_resize,
get_face_params,
load_data_list,
load_unet_from_safetensors)
gen_control_heatmaps,
get_drive_params,
crop_and_resize,
get_face_params,
load_data_list,
load_unet_from_safetensors
)
from .hellomeme.tools import Hello3DMMPred, HelloARKitBSPred, HelloFaceAlignment, HelloCameraDemo
from .hellomeme import HMImagePipeline, HMVideoPipeline, HMVideoSimplePipeline
@@ -305,15 +305,13 @@ class GetImageDriveParams:
save_size=(512, 512),
align=False)
face_parts_embedding, control_heatmaps = face_params_to_tensor(face_toolkits['image_encoder'], face_toolkits['h3dmm'],
drive_face_parts,
drive_rot, drive_trans, ref_rt['trans'],
save_size=512, trans_ratio=0.0)
face_parts_embedding = face_params_to_tensor(face_toolkits['image_encoder'], drive_face_parts)
drive_params = dict(
face_parts=face_parts_embedding.unsqueeze(0).to(dtype=dtype),
drive_coeff=drive_coeff.unsqueeze(0).to(dtype=dtype),
condition=control_heatmaps.unsqueeze(0).to(dtype=dtype),
face_parts=face_parts_embedding.unsqueeze(0).to(dtype=dtype, device='cpu'),
drive_coeff=drive_coeff.unsqueeze(0).to(dtype=dtype, device='cpu'),
drive_rot=drive_rot,
drive_trans=drive_trans,
)
return (drive_params, )
@@ -324,6 +322,8 @@ class HMPipelineImage:
"required": {
"hm_image_pipeline": ("HMIMAGEPIPELINE",),
"ref_image": ("IMAGE",),
"ref_rt": ("REFRT",),
"trans_ratio": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"drive_params": ("DRIVE_PARAMS",),
"prompt": ("STRING", {"default": '(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture'}),
"negative_prompt": ("STRING", {"default": ''}),
@@ -337,11 +337,17 @@ class HMPipelineImage:
FUNCTION = "sample"
CATEGORY = "hellomeme"
def sample(self, hm_image_pipeline, ref_image, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0):
def sample(self, hm_image_pipeline, ref_image, ref_rt, trans_ratio, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0):
image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8)
image_np = cv2.resize(image_np, (512, 512))
image_pil = Image.fromarray(image_np)
drive_rot = drive_params['drive_rot']
drive_trans = drive_params['drive_trans']
condition = gen_control_heatmaps(drive_rot, drive_trans, ref_rt['trans'], 512, trans_ratio)
drive_params['condition'] = condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')
if seed < 0:
generator = torch.Generator().manual_seed(random.randint(0, 100000))
else:
@@ -367,15 +373,13 @@ class GetVideoDriveParams:
"required": {
"face_toolkits": ("FACE_TOOLKITS",),
"images": ("IMAGE",),
"ref_rt": ("REFRT",),
"trans_ratio": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
}
}
RETURN_TYPES = ("DRIVE_PARAMS",)
RETURN_NAMES = ("drive_params",)
FUNCTION = "get_face_params"
CATEGORY = "hellomeme"
def get_face_params(self, face_toolkits, images, ref_rt, trans_ratio):
def get_face_params(self, face_toolkits, images):
dtype = torch.float16
frame_list = [cv2.cvtColor((frame * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) for frame in images]
@@ -388,16 +392,13 @@ class GetVideoDriveParams:
save_size=512,
align=True)
face_toolkits['face_aligner'].reset_track()
face_parts_embedding, control_heatmaps = face_params_to_tensor(
face_toolkits['image_encoder'], face_toolkits['h3dmm'],
drive_face_parts,
drive_rot, drive_trans, ref_rt['trans'],
save_size=512, trans_ratio=trans_ratio)
face_parts_embedding = face_params_to_tensor(face_toolkits['image_encoder'], drive_face_parts)
drive_params = dict(
face_parts=face_parts_embedding.unsqueeze(0).to(dtype=dtype, device='cpu'),
drive_coeff=drive_coeff.unsqueeze(0).to(dtype=dtype, device='cpu'),
condition=control_heatmaps.unsqueeze(0).to(dtype=dtype, device='cpu'),
drive_rot=drive_rot,
drive_trans=drive_trans,
)
return (drive_params, )
@@ -408,6 +409,8 @@ class HMPipelineVideo:
{
"hm_video_pipeline": ("HMVIDEOPIPELINE",),
"ref_image": ("IMAGE",),
"ref_rt": ("REFRT",),
"trans_ratio": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"drive_params": ("DRIVE_PARAMS",),
"prompt": ("STRING", {"default": '(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture'}),
"negative_prompt": ("STRING", {"default": ''}),
@@ -421,7 +424,7 @@ class HMPipelineVideo:
FUNCTION = "sample"
CATEGORY = "hellomeme"
def sample(self, hm_video_pipeline, ref_image, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0):
def sample(self, hm_video_pipeline, ref_image, ref_rt, trans_ratio, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0):
image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8)
image_np = cv2.resize(image_np, (512, 512))
image_pil = Image.fromarray(image_np)
@@ -430,6 +433,12 @@ class HMPipelineVideo:
else:
generator = torch.Generator().manual_seed(seed)
drive_rot = drive_params['drive_rot']
drive_trans = drive_params['drive_trans']
condition = gen_control_heatmaps(drive_rot, drive_trans, ref_rt['trans'], 512, trans_ratio)
drive_params['condition'] = condition.unsqueeze(0).to(dtype=torch.float16, device='cpu')
res_frames = hm_video_pipeline(
prompt=[prompt],
strength=1.0,
workflows/hellomeme_image_example.jpg
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 313 KiB

After

Width:  |  Height:  |  Size: 221 KiB

workflows/hellomeme_image_workflow.json
+375 -377
View File
@@ -1,54 +1,20 @@
{
"last_node_id": 31,
"last_link_id": 40,
"last_node_id": 37,
"last_link_id": 55,
"nodes": [
{
"id": 28,
"type": "HMImagePipelineLoader",
"pos": {
"0": 111,
"1": 102
},
"size": {
"0": 352.79998779296875,
"1": 106
},
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"links": [
34
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMImagePipelineLoader"
},
"widgets_values": [
"None",
"None",
0
]
},
{
"id": 25,
"id": 8,
"type": "LoadImage",
"pos": {
"0": 120,
"1": 796
"0": 61,
"1": 290
},
"size": {
"0": 315,
"1": 314
},
"flags": {},
"order": 1,
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
@@ -70,31 +36,336 @@
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"yao.jpg",
"majicmix2.jpg",
"image"
]
},
{
"id": 8,
"type": "LoadImage",
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": {
"0": 120,
"1": 414
"0": 61,
"1": 697
},
"size": {
"0": 315,
"1": 314
"1": 58
},
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"links": [
13,
32,
42
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": [
0
]
},
{
"id": 26,
"type": "PreviewImage",
"pos": {
"0": 795,
"1": 121
},
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 34
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 11,
"type": "GetReferenceImageRT",
"pos": {
"0": 749,
"1": 546
},
"size": {
"0": 241.79998779296875,
"1": 46
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 13
},
{
"name": "image",
"type": "IMAGE",
"link": 37
}
],
"outputs": [
{
"name": "ref_rt",
"type": "REFRT",
"links": [
50
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetReferenceImageRT"
},
"widgets_values": []
},
{
"id": 31,
"type": "GetVideoDriveParams",
"pos": {
"0": 447,
"1": 745
},
"size": {
"0": 330,
"1": 46
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 42
},
{
"name": "images",
"type": "IMAGE",
"link": 54
}
],
"outputs": [
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"links": [
51
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetVideoDriveParams"
},
"widgets_values": []
},
{
"id": 25,
"type": "CropReferenceImage",
"pos": {
"0": 447,
"1": 498
},
"size": {
"0": 252,
"1": 46
},
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 31
},
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 32
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
34,
37,
52
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CropReferenceImage"
},
"widgets_values": []
},
{
"id": 34,
"type": "HMPipelineImage",
"pos": {
"0": 1119.219970703125,
"1": 376.4541015625
},
"size": {
"0": 315,
"1": 262
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"link": 53
},
{
"name": "ref_image",
"type": "IMAGE",
"link": 52
},
{
"name": "ref_rt",
"type": "REFRT",
"link": 50
},
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"link": 51
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
55
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMPipelineImage"
},
"widgets_values": [
0,
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
77251,
"randomize",
2
]
},
{
"id": 37,
"type": "PreviewImage",
"pos": {
"0": 1501,
"1": 377
},
"size": [
210,
246
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 55
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 35,
"type": "HMImagePipelineLoader",
"pos": {
"0": 58,
"1": 113
},
"size": {
"0": 352.79998779296875,
"1": 106
},
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"links": [
53
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMImagePipelineLoader"
},
"widgets_values": [
"None",
"None",
0
]
},
{
"id": 36,
"type": "LoadImage",
"pos": {
"0": 60,
"1": 862
},
"size": [
315,
314
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
36
54
],
"slot_index": 0
},
@@ -108,282 +379,9 @@
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"majicmix2.jpg",
"yao.jpg",
"image"
]
},
{
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": {
"0": 124,
"1": 276
},
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"links": [
13,
27,
37
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": [
0
]
},
{
"id": 30,
"type": "CropReferenceImage",
"pos": {
"0": 506,
"1": 235
},
"size": {
"0": 252,
"1": 46
},
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 36
},
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 37
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
38,
39,
40
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CropReferenceImage"
},
"widgets_values": []
},
{
"id": 31,
"type": "PreviewImage",
"pos": {
"0": 858,
"1": 37
},
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 40
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 11,
"type": "GetReferenceImageRT",
"pos": {
"0": 807,
"1": 370
},
"size": {
"0": 241.79998779296875,
"1": 46
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 13
},
{
"name": "image",
"type": "IMAGE",
"link": 38
}
],
"outputs": [
{
"name": "ref_rt",
"type": "REFRT",
"links": [
26
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetReferenceImageRT"
},
"widgets_values": []
},
{
"id": 29,
"type": "PreviewImage",
"pos": {
"0": 1843,
"1": 360
},
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 35
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 26,
"type": "HMPipelineImage",
"pos": {
"0": 1461,
"1": 378
},
"size": {
"0": 315,
"1": 218
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "hm_image_pipeline",
"type": "HMIMAGEPIPELINE",
"link": 34
},
{
"name": "ref_image",
"type": "IMAGE",
"link": 39
},
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"link": 32
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
35
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMPipelineImage"
},
"widgets_values": [
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
52143,
"randomize",
2
]
},
{
"id": 22,
"type": "GetVideoDriveParams",
"pos": {
"0": 1082,
"1": 494
},
"size": {
"0": 330,
"1": 98
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 27
},
{
"name": "images",
"type": "IMAGE",
"link": 31
},
{
"name": "ref_rt",
"type": "REFRT",
"link": 26
}
],
"outputs": [
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"links": [
32
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetVideoDriveParams"
},
"widgets_values": [
0
]
}
],
"links": [
@@ -395,91 +393,91 @@
0,
"FACE_TOOLKITS"
],
[
26,
11,
0,
22,
2,
"REFRT"
],
[
27,
2,
0,
22,
0,
"FACE_TOOLKITS"
],
[
31,
8,
0,
25,
0,
22,
1,
"IMAGE"
],
[
32,
22,
0,
26,
2,
"DRIVE_PARAMS"
0,
25,
1,
"FACE_TOOLKITS"
],
[
34,
28,
25,
0,
26,
0,
"HMIMAGEPIPELINE"
],
[
35,
26,
0,
29,
0,
"IMAGE"
],
[
36,
8,
0,
30,
0,
"IMAGE"
],
[
37,
2,
0,
30,
1,
"FACE_TOOLKITS"
],
[
38,
30,
25,
0,
11,
1,
"IMAGE"
],
[
39,
30,
42,
2,
0,
26,
31,
0,
"FACE_TOOLKITS"
],
[
50,
11,
0,
34,
2,
"REFRT"
],
[
51,
31,
0,
34,
3,
"DRIVE_PARAMS"
],
[
52,
25,
0,
34,
1,
"IMAGE"
],
[
40,
30,
53,
35,
0,
34,
0,
"HMIMAGEPIPELINE"
],
[
54,
36,
0,
31,
1,
"IMAGE"
],
[
55,
34,
0,
37,
0,
"IMAGE"
]
@@ -488,10 +486,10 @@
"config": {},
"extra": {
"ds": {
"scale": 0.8264462809917354,
"scale": 1,
"offset": [
509.5058815371377,
98.92725456392243
265.78003215780683,
-30.4541044667626
]
}
},
workflows/hellomeme_video_example.jpg
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 241 KiB

After

Width:  |  Height:  |  Size: 271 KiB

workflows/hellomeme_video_simple_workflow.json
+229 -230
View File
@@ -1,17 +1,17 @@
{
"last_node_id": 31,
"last_link_id": 40,
"last_node_id": 33,
"last_link_id": 49,
"nodes": [
{
"id": 2,
"type": "HMFaceToolkitsLoader",
"id": 8,
"type": "LoadImage",
"pos": {
"0": 106,
"1": 301
"0": 61,
"1": 290
},
"size": {
"0": 315,
"1": 58
"1": 314
},
"flags": {},
"order": 0,
@@ -19,29 +19,33 @@
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"name": "IMAGE",
"type": "IMAGE",
"links": [
13,
27,
32
31
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "HMFaceToolkitsLoader"
"Node name for S&R": "LoadImage"
},
"widgets_values": [
0
"majicmix2.jpg",
"image"
]
},
{
"id": 20,
"type": "VHS_LoadVideoFFmpeg",
"pos": {
"0": 95,
"1": 822
"0": 66,
"1": 835
},
"size": {
"0": 303.46875,
@@ -69,7 +73,7 @@
"name": "IMAGE",
"type": "IMAGE",
"links": [
28
43
],
"slot_index": 0
},
@@ -120,15 +124,15 @@
}
},
{
"id": 8,
"type": "LoadImage",
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": {
"0": 92,
"1": 441
"0": 61,
"1": 697
},
"size": {
"0": 315,
"1": 314
"1": 58
},
"flags": {},
"order": 2,
@@ -136,60 +140,29 @@
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"links": [
31
13,
32,
42
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": [
"majicmix2.jpg",
"image"
0
]
},
{
"id": 26,
"type": "PreviewImage",
"pos": {
"0": 949,
"1": 36
},
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 34
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 25,
"type": "CropReferenceImage",
"pos": {
"0": 480,
"1": 279
"0": 447,
"1": 498
},
"size": {
"0": 252,
@@ -215,9 +188,9 @@
"name": "IMAGE",
"type": "IMAGE",
"links": [
33,
34,
37
37,
46
],
"slot_index": 0
}
@@ -227,19 +200,46 @@
},
"widgets_values": []
},
{
"id": 26,
"type": "PreviewImage",
"pos": {
"0": 795,
"1": 121
},
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 34
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 11,
"type": "GetReferenceImageRT",
"pos": {
"0": 769,
"1": 415
"0": 749,
"1": 546
},
"size": {
"0": 241.79998779296875,
"1": 46
},
"flags": {},
"order": 6,
"order": 7,
"mode": 0,
"inputs": [
{
@@ -258,7 +258,7 @@
"name": "ref_rt",
"type": "REFRT",
"links": [
26
47
],
"slot_index": 0
}
@@ -268,12 +268,112 @@
},
"widgets_values": []
},
{
"id": 32,
"type": "HMPipelineVideo",
"pos": {
"0": 1106,
"1": 463
},
"size": {
"0": 315,
"1": 262
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"link": 49
},
{
"name": "ref_image",
"type": "IMAGE",
"link": 46
},
{
"name": "ref_rt",
"type": "REFRT",
"link": 47
},
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"link": 48
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
44
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMPipelineVideo"
},
"widgets_values": [
0,
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
28524,
"randomize",
2
]
},
{
"id": 31,
"type": "GetVideoDriveParams",
"pos": {
"0": 447,
"1": 745
},
"size": {
"0": 330,
"1": 46
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 42
},
{
"name": "images",
"type": "IMAGE",
"link": 43
}
],
"outputs": [
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"links": [
48
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetVideoDriveParams"
},
"widgets_values": []
},
{
"id": 21,
"type": "VHS_VideoCombine",
"pos": {
"0": 1856,
"1": 546
"0": 1502,
"1": 613
},
"size": [
214.7587890625,
@@ -286,7 +386,7 @@
{
"name": "images",
"type": "IMAGE",
"link": 24
"link": 44
},
{
"name": "audio",
@@ -331,7 +431,7 @@
"hidden": false,
"paused": false,
"params": {
"filename": "HelloMeme_00065-audio.mp4",
"filename": "HelloMeme_00068-audio.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4",
@@ -342,112 +442,11 @@
}
},
{
"id": 15,
"type": "HMPipelineVideo",
"pos": {
"0": 1443,
"1": 271
},
"size": {
"0": 315,
"1": 218
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"link": 40
},
{
"name": "ref_image",
"type": "IMAGE",
"link": 33
},
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"link": 29
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
24
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMPipelineVideo"
},
"widgets_values": [
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
77866,
"randomize",
2
]
},
{
"id": 22,
"type": "GetVideoDriveParams",
"pos": {
"0": 1025,
"1": 557
},
"size": {
"0": 330,
"1": 98
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 27
},
{
"name": "images",
"type": "IMAGE",
"link": 28
},
{
"name": "ref_rt",
"type": "REFRT",
"link": 26
}
],
"outputs": [
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"links": [
29
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetVideoDriveParams"
},
"widgets_values": [
0.2
]
},
{
"id": 31,
"id": 33,
"type": "HMVideoSimplePipelineLoader",
"pos": {
"0": 91,
"1": 85
"0": 59,
"1": 118
},
"size": {
"0": 428.4000244140625,
@@ -462,7 +461,7 @@
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"links": [
40
49
],
"slot_index": 0
}
@@ -486,14 +485,6 @@
0,
"FACE_TOOLKITS"
],
[
24,
15,
0,
21,
0,
"IMAGE"
],
[
25,
20,
@@ -502,38 +493,6 @@
1,
"AUDIO"
],
[
26,
11,
0,
22,
2,
"REFRT"
],
[
27,
2,
0,
22,
0,
"FACE_TOOLKITS"
],
[
28,
20,
0,
22,
1,
"IMAGE"
],
[
29,
22,
0,
15,
2,
"DRIVE_PARAMS"
],
[
31,
8,
@@ -550,14 +509,6 @@
1,
"FACE_TOOLKITS"
],
[
33,
25,
0,
15,
1,
"IMAGE"
],
[
34,
25,
@@ -575,10 +526,58 @@
"IMAGE"
],
[
40,
42,
2,
0,
31,
0,
15,
"FACE_TOOLKITS"
],
[
43,
20,
0,
31,
1,
"IMAGE"
],
[
44,
32,
0,
21,
0,
"IMAGE"
],
[
46,
25,
0,
32,
1,
"IMAGE"
],
[
47,
11,
0,
32,
2,
"REFRT"
],
[
48,
31,
0,
32,
3,
"DRIVE_PARAMS"
],
[
49,
33,
0,
32,
0,
"HMVIDEOPIPELINE"
]
@@ -587,10 +586,10 @@
"config": {},
"extra": {
"ds": {
"scale": 0.8264462809917354,
"scale": 1,
"offset": [
509.5058815371377,
98.92725456392243
265.78003215780683,
-30.4541044667626
]
}
},
workflows/hellomeme_video_workflow.json
+261 -262
View File
@@ -1,17 +1,17 @@
{
"last_node_id": 30,
"last_link_id": 39,
"last_node_id": 32,
"last_link_id": 48,
"nodes": [
{
"id": 2,
"type": "HMFaceToolkitsLoader",
"id": 30,
"type": "HMVideoPipelineLoader",
"pos": {
"0": 106,
"1": 301
"0": 55,
"1": 112
},
"size": {
"0": 315,
"1": 58
"0": 352.79998779296875,
"1": 106
},
"flags": {},
"order": 0,
@@ -19,36 +19,74 @@
"inputs": [],
"outputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"links": [
13,
27,
32
45
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMFaceToolkitsLoader"
"Node name for S&R": "HMVideoPipelineLoader"
},
"widgets_values": [
"None",
"None",
0
]
},
{
"id": 8,
"type": "LoadImage",
"pos": {
"0": 61,
"1": 290
},
"size": {
"0": 315,
"1": 314
},
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
31
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"majicmix2.jpg",
"image"
]
},
{
"id": 20,
"type": "VHS_LoadVideoFFmpeg",
"pos": {
"0": 95,
"1": 822
"0": 66,
"1": 835
},
"size": {
"0": 303.46875,
"1": 238
},
"flags": {},
"order": 1,
"order": 2,
"mode": 0,
"inputs": [
{
@@ -69,7 +107,7 @@
"name": "IMAGE",
"type": "IMAGE",
"links": [
28
43
],
"slot_index": 0
},
@@ -120,76 +158,45 @@
}
},
{
"id": 8,
"type": "LoadImage",
"id": 2,
"type": "HMFaceToolkitsLoader",
"pos": {
"0": 92,
"1": 441
"0": 61,
"1": 697
},
"size": {
"0": 315,
"1": 314
"1": 58
},
"flags": {},
"order": 2,
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"links": [
31
13,
32,
42
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
"Node name for S&R": "HMFaceToolkitsLoader"
},
"widgets_values": [
"majicmix2.jpg",
"image"
0
]
},
{
"id": 26,
"type": "PreviewImage",
"pos": {
"0": 949,
"1": 36
},
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 34
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 25,
"type": "CropReferenceImage",
"pos": {
"0": 480,
"1": 279
"0": 447,
"1": 498
},
"size": {
"0": 252,
@@ -215,9 +222,9 @@
"name": "IMAGE",
"type": "IMAGE",
"links": [
33,
34,
37
37,
46
],
"slot_index": 0
}
@@ -227,19 +234,46 @@
},
"widgets_values": []
},
{
"id": 26,
"type": "PreviewImage",
"pos": {
"0": 795,
"1": 121
},
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 34
}
],
"outputs": [],
"properties": {
"Node name for S&R": "PreviewImage"
},
"widgets_values": []
},
{
"id": 11,
"type": "GetReferenceImageRT",
"pos": {
"0": 769,
"1": 415
"0": 749,
"1": 546
},
"size": {
"0": 241.79998779296875,
"1": 46
},
"flags": {},
"order": 6,
"order": 7,
"mode": 0,
"inputs": [
{
@@ -258,7 +292,7 @@
"name": "ref_rt",
"type": "REFRT",
"links": [
26
47
],
"slot_index": 0
}
@@ -268,12 +302,112 @@
},
"widgets_values": []
},
{
"id": 32,
"type": "HMPipelineVideo",
"pos": {
"0": 1106,
"1": 463
},
"size": {
"0": 315,
"1": 262
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"link": 45
},
{
"name": "ref_image",
"type": "IMAGE",
"link": 46
},
{
"name": "ref_rt",
"type": "REFRT",
"link": 47
},
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"link": 48
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
44
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMPipelineVideo"
},
"widgets_values": [
0,
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
89993,
"randomize",
2
]
},
{
"id": 31,
"type": "GetVideoDriveParams",
"pos": {
"0": 447,
"1": 745
},
"size": {
"0": 330,
"1": 46
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 42
},
{
"name": "images",
"type": "IMAGE",
"link": 43
}
],
"outputs": [
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"links": [
48
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetVideoDriveParams"
},
"widgets_values": []
},
{
"id": 21,
"type": "VHS_VideoCombine",
"pos": {
"0": 1856,
"1": 546
"0": 1502,
"1": 613
},
"size": [
214.7587890625,
@@ -286,7 +420,7 @@
{
"name": "images",
"type": "IMAGE",
"link": 24
"link": 44
},
{
"name": "audio",
@@ -331,7 +465,7 @@
"hidden": false,
"paused": false,
"params": {
"filename": "HelloMeme_00062-audio.mp4",
"filename": "HelloMeme_00067-audio.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4",
@@ -340,141 +474,6 @@
"muted": false
}
}
},
{
"id": 15,
"type": "HMPipelineVideo",
"pos": {
"0": 1443,
"1": 271
},
"size": {
"0": 315,
"1": 218
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"link": 39
},
{
"name": "ref_image",
"type": "IMAGE",
"link": 33
},
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"link": 29
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
24
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMPipelineVideo"
},
"widgets_values": [
"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture",
"",
25,
81209,
"randomize",
2
]
},
{
"id": 30,
"type": "HMVideoPipelineLoader",
"pos": {
"0": 115,
"1": 100
},
"size": {
"0": 352.79998779296875,
"1": 106
},
"flags": {},
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "hm_video_pipeline",
"type": "HMVIDEOPIPELINE",
"links": [
39
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "HMVideoPipelineLoader"
},
"widgets_values": [
"None",
"None",
0
]
},
{
"id": 22,
"type": "GetVideoDriveParams",
"pos": {
"0": 1025,
"1": 557
},
"size": {
"0": 330,
"1": 98
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "face_toolkits",
"type": "FACE_TOOLKITS",
"link": 27
},
{
"name": "images",
"type": "IMAGE",
"link": 28
},
{
"name": "ref_rt",
"type": "REFRT",
"link": 26
}
],
"outputs": [
{
"name": "drive_params",
"type": "DRIVE_PARAMS",
"links": [
29
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "GetVideoDriveParams"
},
"widgets_values": [
0.2
]
}
],
"links": [
@@ -486,14 +485,6 @@
0,
"FACE_TOOLKITS"
],
[
24,
15,
0,
21,
0,
"IMAGE"
],
[
25,
20,
@@ -502,38 +493,6 @@
1,
"AUDIO"
],
[
26,
11,
0,
22,
2,
"REFRT"
],
[
27,
2,
0,
22,
0,
"FACE_TOOLKITS"
],
[
28,
20,
0,
22,
1,
"IMAGE"
],
[
29,
22,
0,
15,
2,
"DRIVE_PARAMS"
],
[
31,
8,
@@ -550,14 +509,6 @@
1,
"FACE_TOOLKITS"
],
[
33,
25,
0,
15,
1,
"IMAGE"
],
[
34,
25,
@@ -575,22 +526,70 @@
"IMAGE"
],
[
39,
42,
2,
0,
31,
0,
"FACE_TOOLKITS"
],
[
43,
20,
0,
31,
1,
"IMAGE"
],
[
44,
32,
0,
21,
0,
"IMAGE"
],
[
45,
30,
0,
15,
32,
0,
"HMVIDEOPIPELINE"
],
[
46,
25,
0,
32,
1,
"IMAGE"
],
[
47,
11,
0,
32,
2,
"REFRT"
],
[
48,
31,
0,
32,
3,
"DRIVE_PARAMS"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.8264462809917354,
"scale": 1,
"offset": [
509.5058815371377,
97.31394584810211
265.78003215780683,
-30.4541044667626
]
}
},