diff --git a/README.md b/README.md index 70becf8..95994f5 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ ## 🔆 New Features/Updates - +- ✅ `11/11/2024` Optimized VRAM usage and added `HMVideoSimplePipeline` (`workflows/hellomeme_video_simple_workflow.json`), which doesn’t use Animatediff and can run on machines with less than 12G VRAM. - ✅ `11/6/2024` The face proportion in the reference image significantly affects the generation quality. We have encapsulated the **recommended image cropping method** used during training into a `CropReferenceImage` Node. Refer to the workflows in the `ComfyUI_HelloMeme/workflows directory`: `hellomeme_video_cropref_workflow.json` and `hellomeme_image_cropref_workflow.json`. diff --git a/hellomeme/__init__.py b/hellomeme/__init__.py index 8c06cdc..d4eded7 100644 --- a/hellomeme/__init__.py +++ b/hellomeme/__init__.py @@ -8,5 +8,5 @@ @Desc : """ -from .pipelines import HMVideoPipeline, HMImagePipeline +from .pipelines import HMVideoPipeline, HMImagePipeline, HMVideoSimplePipeline from .models import HMDenoising3D, HMDenoisingMotion, HMControlNet, HMReferenceAdapter \ No newline at end of file diff --git a/hellomeme/pipelines/__init__.py b/hellomeme/pipelines/__init__.py index a5d36f8..c9ef35e 100644 --- a/hellomeme/pipelines/__init__.py +++ b/hellomeme/pipelines/__init__.py @@ -8,5 +8,6 @@ @Desc : """ -from .pipline_hm_video import HMVideoPipeline +from .pipline_hm_video_simple import HMVideoSimplePipeline from .pipline_hm_image import HMImagePipeline +from .pipline_hm_video import HMVideoPipeline diff --git a/hellomeme/pipelines/pipline_hm_video.py b/hellomeme/pipelines/pipline_hm_video.py index 869eebf..4418251 100644 --- a/hellomeme/pipelines/pipline_hm_video.py +++ b/hellomeme/pipelines/pipline_hm_video.py @@ -8,7 +8,7 @@ @Desc : adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py """ - +import copy from typing import Any, Callable, Dict, List, Optional, Union import torch @@ -19,42 +19,46 @@ from diffusers.utils.torch_utils import randn_tensor from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents from diffusers import StableDiffusionImg2ImgPipeline -from diffusers import EulerDiscreteScheduler from diffusers import MotionAdapter from ..models import HMDenoising3D, HMDenoisingMotion, HMControlNet from ..models import HMReferenceAdapter +from ..utils import dicts_to_device, cat_dicts class HMVideoPipeline(StableDiffusionImg2ImgPipeline): def caryomitosis(self,): - if not hasattr(self, "unet_ref"): - self.unet_ref = HMDenoising3D.from_unet2d(self.unet) - if not isinstance(self.unet, HMDenoisingMotion): - adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff", torch_dtype=torch.float16) - unet = HMDenoisingMotion.from_unet2d(unet=self.unet, motion_adapter=adapter, load_weights=True) - # todo: 不够优雅 - del self.unet - self.unet = unet + if hasattr(self, "unet_ref"): + del self.unet_ref + self.unet_ref = HMDenoising3D.from_unet2d(self.unet) + + self.num_frames = 16 + adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff", torch_dtype=torch.float16) + unet = HMDenoisingMotion.from_unet2d(unet=self.unet, motion_adapter=adapter, load_weights=True) + # todo: 不够优雅 + del self.unet + self.unet = unet def insert_hm_modules(self, dtype, device): + hm_adapter = HMReferenceAdapter.from_pretrained('songkey/hm_reference') if isinstance(self.unet, HMDenoisingMotion): - hm_adapter = HMReferenceAdapter.from_pretrained('songkey/hm_reference') self.unet.insert_reference_adapter(hm_adapter) - self.unet_ref.insert_reference_adapter(hm_adapter) - self.unet_ref = self.unet_ref.to(device=device, dtype=dtype).eval() self.unet = self.unet.to(device=device, dtype=dtype).eval() - if not hasattr(self, "mp_control"): - self.mp_control = HMControlNet.from_pretrained('songkey/hm_control') - self.mp_control = self.mp_control.to(device=device, dtype=dtype).eval() + if hasattr(self, "unet_ref"): + self.unet_ref.insert_reference_adapter(hm_adapter) + self.unet_ref = self.unet_ref.to(device=device, dtype=dtype).eval() + + if hasattr(self, "mp_control"): + del self.mp_control + self.mp_control = HMControlNet.from_pretrained('songkey/hm_control') + self.mp_control = self.mp_control.to(device=device, dtype=dtype).eval() @torch.no_grad() def __call__( self, prompt: Union[str, List[str]] = None, image: PipelineImageInput = None, - num_frames: int = 16, chunk_overlap: int = 4, drive_params: Dict[str, Any] = None, strength: float = 0.8, @@ -168,21 +172,6 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) raw_video_len = drive_params['condition'].size(2) - chunk_size = num_frames - chunk_overlap = min(max(0, chunk_overlap), chunk_size // 2) - chunk_stride = chunk_size - chunk_overlap - - drive_idx_list = list(range(raw_video_len)) - - steped_video_len = len(drive_idx_list) - post_pad_num = chunk_stride - (steped_video_len - chunk_overlap) % chunk_stride - - if post_pad_num > 0: - drive_idx_list = drive_idx_list + [drive_idx_list[-1]] * post_pad_num - paded_video_len = len(drive_idx_list) - - drive_idx_chunks = [drive_idx_list[i:i + chunk_size] for i in - range(0, paded_video_len - chunk_stride + 1, chunk_stride)] # 6. Prepare latent variables ref_latents = [ @@ -203,7 +192,6 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): else None ) - # if hasattr(self, "unet_ref"): ref_latents = ref_latents.unsqueeze(2) cached_res = self.unet_ref( torch.cat([torch.zeros_like(ref_latents), ref_latents], dim=0) if @@ -212,8 +200,6 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): encoder_hidden_states=prompt_embeds, return_dict=False, )[1] - # else: - # cached_res = None # 7.2 Optionally get Guidance Scale Embedding timestep_cond = None @@ -244,12 +230,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): if self.do_classifier_free_guidance: control_latent = cat_dicts([control_latent_neg, control_latent], dim=0) - scheduler = EulerDiscreteScheduler( - num_train_timesteps=1000, - beta_start=0.00085, - beta_end=0.012, - beta_schedule="scaled_linear", - ) + scheduler = copy.deepcopy(self.scheduler) tmp_timesteps, _ = retrieve_timesteps( scheduler, 8, device, None, sigmas @@ -283,9 +264,27 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): pre_latents.append(pred_latent.cpu()) control_latents.append(dicts_to_device([control_latent], device='cpu')[0]) + progress_bar.set_description(f"GEN stage1") progress_bar.update() + pre_latents = torch.cat(pre_latents, dim=2) + chunk_size = self.num_frames + chunk_overlap = min(max(0, chunk_overlap), chunk_size // 2) + chunk_stride = chunk_size - chunk_overlap + + drive_idx_list = list(range(raw_video_len)) + + steped_video_len = len(drive_idx_list) + post_pad_num = chunk_stride - (steped_video_len - chunk_overlap) % chunk_stride + + if post_pad_num > 0: + drive_idx_list = drive_idx_list + [drive_idx_list[-1]] * post_pad_num + paded_video_len = len(drive_idx_list) + + drive_idx_chunks = [drive_idx_list[i:i + chunk_size] for i in + range(0, paded_video_len - chunk_stride + 1, chunk_stride)] + if post_pad_num > 0: pre_latents = torch.cat([pre_latents, (pre_latents[:, :, -2:-1]).clone().repeat_interleave(post_pad_num, 2)], dim=2) @@ -362,6 +361,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): # call the callback, if provided if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.set_description(f"GEN stage2") progress_bar.update() if callback is not None and i % callback_steps == 0: step_idx = i // getattr(self.scheduler, "order", 1) @@ -372,19 +372,4 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): res_frames = [self.vae.decode(latents[:,:,i,:,:].to(device=device), return_dict=False, generator=generator)[0] for i in range(raw_video_len)] res_frames = [self.image_processor.postprocess(frame, output_type=output_type) for frame in res_frames] - return res_frames - -def cat_dicts(dicts, dim=0): - res = {} - for k in dicts[0].keys(): - res[k] = torch.cat([d[k].clone() for d in dicts], dim=dim) - return res - -def dicts_to_device(dicts, device): - ret = [] - for d in dicts: - tmpd = {} - for k in d.keys(): - tmpd[k] = d[k].clone().to(device) - ret.append(tmpd) - return ret \ No newline at end of file + return res_frames \ No newline at end of file diff --git a/hellomeme/pipelines/pipline_hm_video_simple.py b/hellomeme/pipelines/pipline_hm_video_simple.py new file mode 100644 index 0000000..ebd313a --- /dev/null +++ b/hellomeme/pipelines/pipline_hm_video_simple.py @@ -0,0 +1,297 @@ +# coding: utf-8 + +""" +@File : hm_pipline_image.py +@Author : Songkey +@Email : songkey@pku.edu.cn +@Date : 8/29/2024 +@Desc : +adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +""" +import copy +import time +from typing import Any, Callable, Dict, List, Optional, Union +import torch + +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import PipelineImageInput +from diffusers.utils import deprecate +from diffusers.utils.torch_utils import randn_tensor + +from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents +from diffusers import StableDiffusionImg2ImgPipeline + +from ..models import HMDenoising3D, HMControlNet +from ..models import HMReferenceAdapter +from ..utils import dicts_to_device, cat_dicts + +class HMVideoSimplePipeline(StableDiffusionImg2ImgPipeline): + def caryomitosis(self,): + if hasattr(self, "unet_ref"): + del self.unet_ref + self.unet_ref = HMDenoising3D.from_unet2d(self.unet) + unet = HMDenoising3D.from_unet2d(self.unet) + # todo: 不够优雅 + del self.unet + self.unet = unet + + def insert_hm_modules(self, dtype, device): + hm_adapter = HMReferenceAdapter.from_pretrained('songkey/hm_reference') + if isinstance(self.unet, HMDenoising3D): + self.unet.insert_reference_adapter(hm_adapter) + self.unet = self.unet.to(device=device, dtype=dtype).eval() + + if hasattr(self, "unet_ref"): + self.unet_ref = self.unet_ref.to(device=device, dtype=dtype).eval() + + if hasattr(self, "mp_control"): + del self.mp_control + self.mp_control = HMControlNet.from_pretrained('songkey/hm_control') + self.mp_control = self.mp_control.to(device=device, dtype=dtype).eval() + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + drive_params: Dict[str, Any] = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + timesteps: List[int] = None, + sigmas: List[float] = None, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: int = None, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + num_images_per_prompt = 1 + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + strength, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Preprocess image + image = self.image_processor.preprocess(image).to(device=device, dtype=prompt_embeds.dtype) + + # 6. Prepare latent variables + ref_latents = [ + retrieve_latents(self.vae.encode(image[i: i + 1].to(device=device)), generator=generator) + for i in range(batch_size) + ] + ref_latents = torch.cat(ref_latents, dim=0) + ref_latents = self.vae.config.scaling_factor * ref_latents + c, h, w = ref_latents.shape[1:] + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + ref_latents = ref_latents.unsqueeze(2) + cached_res = self.unet_ref( + torch.cat([torch.zeros_like(ref_latents), ref_latents], dim=0) if + self.do_classifier_free_guidance else ref_latents, + 0, + encoder_hidden_states=prompt_embeds, + return_dict=False, + )[1] + + # 7.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=prompt_embeds.dtype) + + raw_video_len = drive_params['condition'].size(2) + + pre_latents = [] + control_latents = [] + control_latent_neg = None + base_noise = randn_tensor([batch_size, c, 1, h, w], dtype=prompt_embeds.dtype, generator=generator).to(device=device) + with self.progress_bar(total=raw_video_len) as progress_bar: + for idx in range(raw_video_len): + condition = drive_params['condition'][:, :, idx:idx+1].clone().to(device=device) + drive_coeff = drive_params['drive_coeff'][:, idx:idx+1].clone().to(device=device) + face_parts = drive_params['face_parts'][:, idx:idx+1].clone().to(device=device) + + if control_latent_neg is None and self.do_classifier_free_guidance: + control_latent_neg = self.mp_control(condition=torch.ones_like(condition)*-1, + drive_coeff=torch.zeros_like(drive_coeff), + face_parts=torch.zeros_like(face_parts)) + + control_latent = self.mp_control(condition=condition, + drive_coeff=drive_coeff, + face_parts=face_parts) + if self.do_classifier_free_guidance: + control_latent = cat_dicts([control_latent_neg, control_latent], dim=0) + + scheduler = copy.deepcopy(self.scheduler) + tmp_timesteps, _ = retrieve_timesteps(scheduler, 8, device, None, sigmas) + if idx == 0: + pred_latent = scheduler.add_noise(ref_latents, base_noise, tmp_timesteps[:1]) + else: + pred_latent = scheduler.add_noise(pred_latent, base_noise, tmp_timesteps[:1]) + + for i, t in enumerate(tmp_timesteps): + latent_model_input = torch.cat([pred_latent, pred_latent], dim=0) if \ + self.do_classifier_free_guidance else pred_latent + latent_model_input = scheduler.scale_model_input(latent_model_input, t) + noise_pred_chunk = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + reference_hidden_states=cached_res, + control_hidden_states=control_latent, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred_chunk.chunk(2) + noise_pred_chunk = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + pred_latent = scheduler.step(noise_pred_chunk, t, pred_latent, + **extra_step_kwargs, return_dict=False)[0] + pre_latents.append(pred_latent.cpu()) + control_latents.append(dicts_to_device([control_latent], device='cpu')[0]) + + progress_bar.set_description(f"GEN stage1") + progress_bar.update() + + res_frames = [] + with self.progress_bar(total=raw_video_len) as progress_bar: + for idx in range(raw_video_len): + + scheduler = copy.deepcopy(self.scheduler) + tmp_timesteps, _ = retrieve_timesteps(scheduler, num_inference_steps, device, None, sigmas) + + if idx == 0: + res_latent = scheduler.add_noise(pre_latents[idx].to(device=device), + base_noise, tmp_timesteps[:1]) + else: + res_latent = scheduler.add_noise(pre_latents[idx].to(device=device) * 0.8 + res_latent * 0.2, + base_noise, tmp_timesteps[:1]) + + for i, t in enumerate(tmp_timesteps): + latent_model_input = torch.cat([res_latent, res_latent], dim=0) if \ + self.do_classifier_free_guidance else res_latent + latent_model_input = scheduler.scale_model_input(latent_model_input, t) + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + reference_hidden_states=cached_res, + control_hidden_states=dicts_to_device([control_latents[idx]], device=device)[0], + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * ( + noise_pred_text - noise_pred_uncond) + res_latent = scheduler.step(noise_pred, t, res_latent, + **extra_step_kwargs, return_dict=False)[0] + + res_frame_tensor = self.vae.decode(res_latent[:,:,0,:,:].to(device=device) / self.vae.config.scaling_factor, + return_dict=False, generator=generator)[0] + res_frame = self.image_processor.postprocess(res_frame_tensor, output_type=output_type) + + res_frames.append(res_frame) + + progress_bar.set_description(f"GEN stage2") + progress_bar.update() + return res_frames \ No newline at end of file diff --git a/hellomeme/utils.py b/hellomeme/utils.py index d3e65cb..b691cf1 100644 --- a/hellomeme/utils.py +++ b/hellomeme/utils.py @@ -11,6 +11,7 @@ import os import cv2 import numpy as np +from tqdm import tqdm import torch from PIL import Image import subprocess @@ -20,6 +21,27 @@ from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_u from safetensors import safe_open +def merge_dicts(dictl, dictr, wl=0.5): + res = {} + for k in dictl.keys(): + res[k] = dictl[k] * wl + dictr[k] * (1-wl) + return res + +def cat_dicts(dicts, dim=0): + res = {} + for k in dicts[0].keys(): + res[k] = torch.cat([d[k].clone() for d in dicts], dim=dim) + return res + +def dicts_to_device(dicts, device): + ret = [] + for d in dicts: + tmpd = {} + for k in d.keys(): + tmpd[k] = d[k].clone().to(device) + ret.append(tmpd) + return ret + def load_safetensors(model_path): tensors = {} with safe_open(model_path, framework="pt", device=0) as f: @@ -161,11 +183,16 @@ def get_face_params(h3dmm, harkit_bs, frames, landmarks, save_size=(512, 512), a rot_list = [] trans_list = [] arkit_bs_list = [] - for landmark, frame in zip(new_landmarks, new_frames): - drive_rot, drive_trans = h3dmm.forward_params(frame, landmark) - rot_list.append(drive_rot) - trans_list.append(drive_trans) - arkit_bs_list.append(harkit_bs.forward(frame, landmark)) + + with tqdm(total=len(new_landmarks)) as pbar: + for landmark, frame in zip(new_landmarks, new_frames): + drive_rot, drive_trans = h3dmm.forward_params(frame, landmark) + rot_list.append(drive_rot) + trans_list.append(drive_trans) + arkit_bs_list.append(harkit_bs.forward(frame, landmark)) + + pbar.set_description('RT & ARKIT') + pbar.update() drive_coeff = torch.from_numpy(np.stack(arkit_bs_list, axis=0)) return face_parts, drive_coeff, rot_list, trans_list @@ -177,10 +204,18 @@ def face_params_to_tensor(clip_encoder, h3dmm, face_parts, drive_rot, drive_tran face_parts = torch.stack(face_parts_list, dim=2) face_parts_tensor = rearrange(face_parts, "c f p h w -> (f p) c h w") - face_parts_embedding = clip_encoder(face_parts_tensor.to(device=clip_encoder.device, dtype=clip_encoder.dtype)).image_embeds + face_parts_embedding_list = [] + with tqdm(total=face_parts_tensor.size(0)) as pbar: + for i in range(0, face_parts_tensor.size(0)): + face_parts_embedding = clip_encoder(face_parts_tensor[i:i+1].to(device=clip_encoder.device, dtype=clip_encoder.dtype)).image_embeds + face_parts_embedding_list.append(face_parts_embedding.cpu()) + pbar.set_description('CLIP IMAGE ENCODER') + pbar.update() + + face_parts_embedding = torch.cat(face_parts_embedding_list, dim=0) face_parts_embedding = rearrange(face_parts_embedding, "(f p) c -> f p c", - f=face_parts.size(2)).cpu() + f=face_parts.size(2)) control_list = [] control_show_list = [] @@ -198,26 +233,32 @@ def face_params_to_tensor(clip_encoder, h3dmm, face_parts, drive_rot, drive_tran def det_landmarks(face_aligner, frame_list, save_size=(512, 512), reset=False): rect_list = [] new_frame_list = [] - for frame in frame_list: - frame = cv2.resize(frame, save_size) - faces = face_aligner.forward(frame, reset=reset) - if len(faces) > 0: - face = sorted(faces, key=lambda x: (x['face_rect'][2] - x['face_rect'][0]) * ( - x['face_rect'][3] - x['face_rect'][1]))[-1] - rect_list.append(face['face_rect']) - new_frame_list.append(frame) + with tqdm(total=len(frame_list)) as pbar: + for frame in frame_list: + frame = cv2.resize(frame, save_size) + faces = face_aligner.forward(frame, reset=reset) + if len(faces) > 0: + face = sorted(faces, key=lambda x: (x['face_rect'][2] - x['face_rect'][0]) * ( + x['face_rect'][3] - x['face_rect'][1]))[-1] + rect_list.append(face['face_rect']) + new_frame_list.append(frame) + pbar.set_description('DET stage1') + pbar.update() face_aligner.reset_track() save_frame_list = [] save_landmark_list = [] - for frame, rect in zip(new_frame_list, rect_list): - faces = face_aligner.forward(frame, pre_rect=rect, reset=reset) - if len(faces) > 0: - face = sorted(faces, key=lambda x: (x['face_rect'][2] - x['face_rect'][0]) * ( - x['face_rect'][3] - x['face_rect'][1]))[-1] - landmarks = face['pre_kpt_222'] - save_frame_list.append(frame) - save_landmark_list.append(landmarks) + with tqdm(total=len(new_frame_list)) as pbar: + for frame, rect in zip(new_frame_list, rect_list): + faces = face_aligner.forward(frame, pre_rect=rect, reset=reset) + if len(faces) > 0: + face = sorted(faces, key=lambda x: (x['face_rect'][2] - x['face_rect'][0]) * ( + x['face_rect'][3] - x['face_rect'][1]))[-1] + landmarks = face['pre_kpt_222'] + save_frame_list.append(frame) + save_landmark_list.append(landmarks) + pbar.set_description('DET stage2') + pbar.update() save_landmark_list = np.stack(save_landmark_list, axis=0).astype(np.float16) face_aligner.reset_track() diff --git a/meme.py b/meme.py index f615383..22a24ca 100644 --- a/meme.py +++ b/meme.py @@ -38,7 +38,7 @@ from .hellomeme.utils import (face_params_to_tensor, load_unet_from_safetensors) from .hellomeme.tools import Hello3DMMPred, HelloARKitBSPred, HelloFaceAlignment, HelloCameraDemo -from .hellomeme import HMImagePipeline, HMVideoPipeline +from .hellomeme import HMImagePipeline, HMVideoPipeline, HMVideoSimplePipeline from transformers import CLIPVisionModelWithProjection class HMImagePipelineLoader: @@ -138,6 +138,56 @@ class HMVideoPipelineLoader: return (pipeline,) +class HMVideoSimplePipelineLoader: + @classmethod + def INPUT_TYPES(s): + checkpoint_files = sorted(load_data_list(osp.join(cur_dir, '../../models/checkpoints'), '.pt;.pth;.ckpt;.safetensors')) + lora_files = sorted(load_data_list(osp.join(cur_dir, '../../models/loras'), '.safetensors')) + + return { + "optional": { + "checkpoint_path": (['None'] + checkpoint_files, ), + "lora_path": (['None'] + lora_files, ), + "gpu_id": ("INT", {"default": 0}), + } + } + + RETURN_TYPES = ("HMVIDEOPIPELINE",) + RETURN_NAMES = ("hm_video_pipeline",) + FUNCTION = "load_pipeline" + CATEGORY = "hellomeme" + + def load_pipeline(self, checkpoint_path=None, lora_path=None, gpu_id=0): + dtype = torch.float16 + if gpu_id >= 0: + device = torch.device("cuda:{}".format(gpu_id)) + else: + device = torch.device("cpu") + pipeline = HMVideoSimplePipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5").to(dtype=dtype, + device=device) + pipeline.caryomitosis() + + ### load customized checkpoint or lora here: + ## checkpoints + + if checkpoint_path and osp.isfile(checkpoint_path): + if checkpoint_path.endswith('.safetensors'): + state_dict = load_unet_from_safetensors(checkpoint_path, pipeline.unet_ref.config) + pipeline.unet.load_state_dict(state_dict, strict=False) + elif osp.splitext(checkpoint_path)[-1] in ['.pt', '.pth', '.ckpt']: + state_dict = torch.load(checkpoint_path) + pipeline.unet.load_state_dict(state_dict, strict=False) + else: + print("Invalid checkpoint path", checkpoint_path) + + ### lora + if lora_path and osp.isfile(lora_path): + pipeline.load_lora_weights(osp.dirname(lora_path), weight_name=osp.basename(lora_path), adapter_name="lora") + + pipeline.insert_hm_modules(dtype=dtype, device=device) + + return (pipeline,) + class HMFaceToolkitsLoader: @classmethod def INPUT_TYPES(s): @@ -180,7 +230,7 @@ class GetReferenceImageRT: def get_reference_image_rt(self, face_toolkits, image): image_np = cv2.cvtColor((image[0] * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) image_np = cv2.resize(image_np, (512, 512)) - print(image_np.shape) + # print(image_np.shape) face_toolkits['face_aligner'].reset_track() faces = face_toolkits['face_aligner'].forward(image_np) assert len(faces) > 0 @@ -208,7 +258,7 @@ class CropReferenceImage: def crop_reference_image(self, image, face_toolkits): image_np = cv2.cvtColor((image[0] * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) image_np = cv2.resize(image_np, (512, 512)) - print(image_np.shape) + # print(image_np.shape) face_toolkits['face_aligner'].reset_track() faces = face_toolkits['face_aligner'].forward(image_np) assert len(faces) > 0 @@ -228,7 +278,6 @@ class GetImageDriveParams: "face_toolkits": ("FACE_TOOLKITS",), "image": ("IMAGE",), "ref_rt": ("REFRT",), - "gpu_id": ("INT", {"default": 0}) } } @@ -236,9 +285,8 @@ class GetImageDriveParams: RETURN_NAMES = ("drive_params",) FUNCTION = "get_face_params" CATEGORY = "hellomeme" - def get_face_params(self, face_toolkits, image, ref_rt, gpu_id): + def get_face_params(self, face_toolkits, image, ref_rt): dtype = torch.float16 - device = torch.device(f'cuda:{gpu_id}') image_np = cv2.cvtColor((image[0] * 255).cpu().numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) @@ -263,9 +311,9 @@ class GetImageDriveParams: save_size=512, trans_ratio=0.0) drive_params = dict( - face_parts=face_parts_embedding.unsqueeze(0).to(dtype=dtype, device=device), - drive_coeff=drive_coeff.unsqueeze(0).to(dtype=dtype, device=device), - condition=control_heatmaps.unsqueeze(0).to(dtype=dtype, device=device), + face_parts=face_parts_embedding.unsqueeze(0).to(dtype=dtype), + drive_coeff=drive_coeff.unsqueeze(0).to(dtype=dtype), + condition=control_heatmaps.unsqueeze(0).to(dtype=dtype), ) return (drive_params, ) @@ -274,8 +322,8 @@ class HMPipelineImage: def INPUT_TYPES(s): return { "required": { - "pipeline": ("HMIMAGEPIPELINE",), - "image": ("IMAGE",), + "hm_image_pipeline": ("HMIMAGEPIPELINE",), + "ref_image": ("IMAGE",), "drive_params": ("DRIVE_PARAMS",), "prompt": ("STRING", {"default": '(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture'}), "negative_prompt": ("STRING", {"default": ''}), @@ -289,8 +337,8 @@ class HMPipelineImage: FUNCTION = "sample" CATEGORY = "hellomeme" - def sample(self, pipeline, image, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0): - image_np = (image[0] * 255).cpu().numpy().astype(np.uint8) + def sample(self, hm_image_pipeline, ref_image, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0): + image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8) image_np = cv2.resize(image_np, (512, 512)) image_pil = Image.fromarray(image_np) @@ -299,7 +347,7 @@ class HMPipelineImage: else: generator = torch.Generator().manual_seed(seed) - result_img = pipeline( + result_img = hm_image_pipeline( prompt=[prompt], strength=1.0, image=image_pil, @@ -358,8 +406,8 @@ class HMPipelineVideo: def INPUT_TYPES(s): return {"required": { - "pipeline": ("HMVIDEOPIPELINE",), - "image": ("IMAGE",), + "hm_video_pipeline": ("HMVIDEOPIPELINE",), + "ref_image": ("IMAGE",), "drive_params": ("DRIVE_PARAMS",), "prompt": ("STRING", {"default": '(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture'}), "negative_prompt": ("STRING", {"default": ''}), @@ -373,8 +421,8 @@ class HMPipelineVideo: FUNCTION = "sample" CATEGORY = "hellomeme" - def sample(self, pipeline, image, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0): - image_np = (image[0] * 255).cpu().numpy().astype(np.uint8) + def sample(self, hm_video_pipeline, ref_image, drive_params, prompt, negative_prompt, steps=25, seed=-1, guidance_scale=2.0): + image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8) image_np = cv2.resize(image_np, (512, 512)) image_pil = Image.fromarray(image_np) if seed < 0: @@ -382,7 +430,7 @@ class HMPipelineVideo: else: generator = torch.Generator().manual_seed(seed) - res_frames = pipeline( + res_frames = hm_video_pipeline( prompt=[prompt], strength=1.0, image=image_pil, @@ -399,6 +447,7 @@ class HMPipelineVideo: NODE_CLASS_MAPPINGS = { "HMImagePipelineLoader": HMImagePipelineLoader, "HMVideoPipelineLoader": HMVideoPipelineLoader, + "HMVideoSimplePipelineLoader": HMVideoSimplePipelineLoader, "HMFaceToolkitsLoader": HMFaceToolkitsLoader, "GetReferenceImageRT": GetReferenceImageRT, "GetImageDriveParams": GetImageDriveParams, @@ -411,6 +460,7 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "HMImagePipelineLoader": "Load HelloMemeImage Pipeline", "HMVideoPipelineLoader": "Load HelloMemeVideo Pipeline", + "HMVideoSimplePipelineLoader": "Load HelloMemeVideoSimple Pipeline", "HMFaceToolkitsLoader": "Load Face Toolkits", "GetReferenceImageT": "Get Reference Image Translation", "GetImageDriveParams": "Get Drive Image Parameters", diff --git a/workflows/hellomeme_image_cropref_workflow.json b/workflows/hellomeme_image_cropref_workflow.json deleted file mode 100644 index 6cf394b..0000000 --- a/workflows/hellomeme_image_cropref_workflow.json +++ /dev/null @@ -1,497 +0,0 @@ -{ - "last_node_id": 31, - "last_link_id": 40, - "nodes": [ - { - "id": 28, - "type": "HMImagePipelineLoader", - "pos": { - "0": 111, - "1": 102 - }, - "size": { - "0": 352.79998779296875, - "1": 106 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "HMIMAGEPIPELINE", - "type": "HMIMAGEPIPELINE", - "links": [ - 34 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMImagePipelineLoader" - }, - "widgets_values": [ - "None", - "None", - 0 - ] - }, - { - "id": 29, - "type": "PreviewImage", - "pos": { - "0": 1825, - "1": 420 - }, - "size": [ - 210, - 246 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 35 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 26, - "type": "HMPipelineImage", - "pos": { - "0": 1407.3828125, - "1": 432.0332336425781 - }, - "size": { - "0": 315, - "1": 218 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "pipeline", - "type": "HMIMAGEPIPELINE", - "link": 34 - }, - { - "name": "image", - "type": "IMAGE", - "link": 39 - }, - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "link": 32 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 35 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMPipelineImage" - }, - "widgets_values": [ - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 467, - "randomize", - 2 - ] - }, - { - "id": 25, - "type": "LoadImage", - "pos": { - "0": 120, - "1": 796 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 31 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "yao.jpg", - "image" - ] - }, - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 120, - "1": 414 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 36 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, - { - "id": 2, - "type": "HMFaceToolkitsLoader", - "pos": { - "0": 124, - "1": 276 - }, - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "FACE_TOOLKITS", - "type": "FACE_TOOLKITS", - "links": [ - 13, - 27, - 37 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMFaceToolkitsLoader" - }, - "widgets_values": [ - 0 - ] - }, - { - "id": 11, - "type": "GetReferenceImageRT", - "pos": { - "0": 639, - "1": 512 - }, - "size": { - "0": 241.79998779296875, - "1": 46 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 13 - }, - { - "name": "image", - "type": "IMAGE", - "link": 38 - } - ], - "outputs": [ - { - "name": "REFRT", - "type": "REFRT", - "links": [ - 26 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetReferenceImageRT" - }, - "widgets_values": [] - }, - { - "id": 22, - "type": "GetVideoDriveParams", - "pos": { - "0": 960, - "1": 552 - }, - "size": { - "0": 330, - "1": 98 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 27 - }, - { - "name": "images", - "type": "IMAGE", - "link": 31 - }, - { - "name": "ref_rt", - "type": "REFRT", - "link": 26 - } - ], - "outputs": [ - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "links": [ - 32 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetVideoDriveParams" - }, - "widgets_values": [ - 0 - ] - }, - { - "id": 31, - "type": "PreviewImage", - "pos": { - "0": 934, - "1": 134 - }, - "size": [ - 210, - 246 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 40 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, - { - "id": 30, - "type": "CropReferenceImage", - "pos": { - "0": 488, - "1": 306 - }, - "size": { - "0": 252, - "1": 46 - }, - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 36 - }, - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 37 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 38, - 39, - 40 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CropReferenceImage" - } - } - ], - "links": [ - [ - 13, - 2, - 0, - 11, - 0, - "FACE_TOOLKITS" - ], - [ - 26, - 11, - 0, - 22, - 2, - "REFRT" - ], - [ - 27, - 2, - 0, - 22, - 0, - "FACE_TOOLKITS" - ], - [ - 31, - 25, - 0, - 22, - 1, - "IMAGE" - ], - [ - 32, - 22, - 0, - 26, - 2, - "DRIVE_PARAMS" - ], - [ - 34, - 28, - 0, - 26, - 0, - "HMIMAGEPIPELINE" - ], - [ - 35, - 26, - 0, - 29, - 0, - "IMAGE" - ], - [ - 36, - 8, - 0, - 30, - 0, - "IMAGE" - ], - [ - 37, - 2, - 0, - 30, - 1, - "FACE_TOOLKITS" - ], - [ - 38, - 30, - 0, - 11, - 1, - "IMAGE" - ], - [ - 39, - 30, - 0, - 26, - 1, - "IMAGE" - ], - [ - 40, - 30, - 0, - 31, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": { - "ds": { - "scale": 0.9090909090909091, - "offset": [ - 328.0506633321691, - 3.5667581298829703 - ] - } - }, - "version": 0.4 -} \ No newline at end of file diff --git a/workflows/hellomeme_image_workflow.json b/workflows/hellomeme_image_workflow.json index 2560c4c..484b26d 100644 --- a/workflows/hellomeme_image_workflow.json +++ b/workflows/hellomeme_image_workflow.json @@ -1,47 +1,116 @@ { - "last_node_id": 29, - "last_link_id": 35, + "last_node_id": 31, + "last_link_id": 40, "nodes": [ { - "id": 11, - "type": "GetReferenceImageRT", + "id": 28, + "type": "HMImagePipelineLoader", "pos": { - "0": 536, - "1": 516 + "0": 111, + "1": 102 }, "size": { - "0": 241.79998779296875, - "1": 46 + "0": 352.79998779296875, + "1": 106 }, "flags": {}, - "order": 4, + "order": 0, "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 13 - }, - { - "name": "image", - "type": "IMAGE", - "link": 12 - } - ], + "inputs": [], "outputs": [ { - "name": "REFRT", - "type": "REFRT", + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", "links": [ - 26 + 34 ], "slot_index": 0 } ], "properties": { - "Node name for S&R": "GetReferenceImageRT" + "Node name for S&R": "HMImagePipelineLoader" }, - "widgets_values": [] + "widgets_values": [ + "None", + "None", + 0 + ] + }, + { + "id": 25, + "type": "LoadImage", + "pos": { + "0": 120, + "1": 796 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 31 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "yao.jpg", + "image" + ] + }, + { + "id": 8, + "type": "LoadImage", + "pos": { + "0": 120, + "1": 414 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 36 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] }, { "id": 2, @@ -55,16 +124,17 @@ "1": 58 }, "flags": {}, - "order": 0, + "order": 3, "mode": 0, "inputs": [], "outputs": [ { - "name": "FACE_TOOLKITS", + "name": "face_toolkits", "type": "FACE_TOOLKITS", "links": [ 13, - 27 + 27, + 37 ], "slot_index": 0 } @@ -76,19 +146,210 @@ 0 ] }, + { + "id": 30, + "type": "CropReferenceImage", + "pos": { + "0": 506, + "1": 235 + }, + "size": { + "0": 252, + "1": 46 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 36 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 37 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 38, + 39, + 40 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropReferenceImage" + }, + "widgets_values": [] + }, + { + "id": 31, + "type": "PreviewImage", + "pos": { + "0": 858, + "1": 37 + }, + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 40 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 11, + "type": "GetReferenceImageRT", + "pos": { + "0": 807, + "1": 370 + }, + "size": { + "0": 241.79998779296875, + "1": 46 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 13 + }, + { + "name": "image", + "type": "IMAGE", + "link": 38 + } + ], + "outputs": [ + { + "name": "ref_rt", + "type": "REFRT", + "links": [ + 26 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetReferenceImageRT" + }, + "widgets_values": [] + }, + { + "id": 29, + "type": "PreviewImage", + "pos": { + "0": 1843, + "1": 360 + }, + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 35 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 26, + "type": "HMPipelineImage", + "pos": { + "0": 1461, + "1": 378 + }, + "size": { + "0": 315, + "1": 218 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "link": 34 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 39 + }, + { + "name": "drive_params", + "type": "DRIVE_PARAMS", + "link": 32 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 35 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMPipelineImage" + }, + "widgets_values": [ + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 52143, + "randomize", + 2 + ] + }, { "id": 22, "type": "GetVideoDriveParams", "pos": { - "0": 932, - "1": 503 + "0": 1082, + "1": 494 }, "size": { "0": 330, "1": 98 }, "flags": {}, - "order": 5, + "order": 7, "mode": 0, "inputs": [ { @@ -123,208 +384,9 @@ "widgets_values": [ 0 ] - }, - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 120, - "1": 414 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 12, - 33 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, - { - "id": 28, - "type": "HMImagePipelineLoader", - "pos": { - "0": 111, - "1": 102 - }, - "size": { - "0": 352.79998779296875, - "1": 106 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "HMIMAGEPIPELINE", - "type": "HMIMAGEPIPELINE", - "links": [ - 34 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMImagePipelineLoader" - }, - "widgets_values": [ - "None", - "None", - 0 - ] - }, - { - "id": 29, - "type": "PreviewImage", - "pos": { - "0": 1825, - "1": 420 - }, - "size": [ - 210, - 246 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 35 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 26, - "type": "HMPipelineImage", - "pos": { - "0": 1407.3828125, - "1": 432.0332336425781 - }, - "size": { - "0": 315, - "1": 218 - }, - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "pipeline", - "type": "HMIMAGEPIPELINE", - "link": 34 - }, - { - "name": "image", - "type": "IMAGE", - "link": 33 - }, - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "link": 32 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 35 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMPipelineImage" - }, - "widgets_values": [ - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 87153, - "randomize", - 2 - ] - }, - { - "id": 25, - "type": "LoadImage", - "pos": { - "0": 120, - "1": 796 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 31 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "yao.jpg", - "image" - ] } ], "links": [ - [ - 12, - 8, - 0, - 11, - 1, - "IMAGE" - ], [ 13, 2, @@ -365,14 +427,6 @@ 2, "DRIVE_PARAMS" ], - [ - 33, - 8, - 0, - 26, - 1, - "IMAGE" - ], [ 34, 28, @@ -388,16 +442,56 @@ 29, 0, "IMAGE" + ], + [ + 36, + 8, + 0, + 30, + 0, + "IMAGE" + ], + [ + 37, + 2, + 0, + 30, + 1, + "FACE_TOOLKITS" + ], + [ + 38, + 30, + 0, + 11, + 1, + "IMAGE" + ], + [ + 39, + 30, + 0, + 26, + 1, + "IMAGE" + ], + [ + 40, + 30, + 0, + 31, + 0, + "IMAGE" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.9090909090909091, + "scale": 0.8264462809917354, "offset": [ - 328.0506633321691, - 3.5667581298829703 + 509.5058815371377, + 98.92725456392243 ] } }, diff --git a/workflows/hellomeme_video_cropref_workflow.json b/workflows/hellomeme_video_simple_workflow.json similarity index 91% rename from workflows/hellomeme_video_cropref_workflow.json rename to workflows/hellomeme_video_simple_workflow.json index 354fff1..aad9716 100644 --- a/workflows/hellomeme_video_cropref_workflow.json +++ b/workflows/hellomeme_video_simple_workflow.json @@ -1,93 +1,20 @@ { - "last_node_id": 26, - "last_link_id": 34, + "last_node_id": 31, + "last_link_id": 40, "nodes": [ - { - "id": 14, - "type": "HMVideoPipelineLoader", - "pos": { - "0": 116, - "1": 105 - }, - "size": { - "0": 352.79998779296875, - "1": 106 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "links": [ - 16 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMVideoPipelineLoader" - }, - "widgets_values": [ - "None", - "None", - 0 - ] - }, - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 120, - "1": 414 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 12, - 31 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, { "id": 2, "type": "HMFaceToolkitsLoader", "pos": { - "0": 124, - "1": 276 + "0": 106, + "1": 301 }, "size": { "0": 315, "1": 58 }, "flags": {}, - "order": 2, + "order": 0, "mode": 0, "inputs": [], "outputs": [ @@ -113,15 +40,15 @@ "id": 20, "type": "VHS_LoadVideoFFmpeg", "pos": { - "0": 120, - "1": 841 + "0": 95, + "1": 822 }, "size": { "0": 303.46875, "1": 238 }, "flags": {}, - "order": 3, + "order": 1, "mode": 0, "inputs": [ { @@ -192,19 +119,84 @@ } } }, + { + "id": 8, + "type": "LoadImage", + "pos": { + "0": 92, + "1": 441 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 31 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 26, + "type": "PreviewImage", + "pos": { + "0": 949, + "1": 36 + }, + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 34 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, { "id": 25, "type": "CropReferenceImage", "pos": { - "0": 617, - "1": 287 + "0": 480, + "1": 279 }, "size": { "0": 252, "1": 46 }, "flags": {}, - "order": 5, + "order": 4, "mode": 0, "inputs": [ { @@ -224,7 +216,8 @@ "type": "IMAGE", "links": [ 33, - 34 + 34, + 37 ], "slot_index": 0 } @@ -234,72 +227,19 @@ }, "widgets_values": [] }, - { - "id": 15, - "type": "HMPipelineVideo", - "pos": { - "0": 1366, - "1": 390 - }, - "size": { - "0": 315, - "1": 218 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "pipeline", - "type": "HMVIDEOPIPELINE", - "link": 16 - }, - { - "name": "image", - "type": "IMAGE", - "link": 33 - }, - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "link": 29 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 24 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMPipelineVideo" - }, - "widgets_values": [ - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 75328, - "randomize", - 2 - ] - }, { "id": 11, "type": "GetReferenceImageRT", "pos": { - "0": 565, - "1": 529 + "0": 769, + "1": 415 }, "size": { "0": 241.79998779296875, "1": 46 }, "flags": {}, - "order": 4, + "order": 6, "mode": 0, "inputs": [ { @@ -310,7 +250,7 @@ { "name": "image", "type": "IMAGE", - "link": 12 + "link": 37 } ], "outputs": [ @@ -328,87 +268,12 @@ }, "widgets_values": [] }, - { - "id": 22, - "type": "GetVideoDriveParams", - "pos": { - "0": 951, - "1": 562 - }, - "size": { - "0": 330, - "1": 98 - }, - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 27 - }, - { - "name": "images", - "type": "IMAGE", - "link": 28 - }, - { - "name": "ref_rt", - "type": "REFRT", - "link": 26 - } - ], - "outputs": [ - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "links": [ - 29 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetVideoDriveParams" - }, - "widgets_values": [ - 0 - ] - }, - { - "id": 26, - "type": "PreviewImage", - "pos": { - "0": 995, - "1": 115 - }, - "size": { - "0": 210, - "1": 246 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 34 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, { "id": 21, "type": "VHS_VideoCombine", "pos": { - "0": 1863, - "1": 613 + "0": 1856, + "1": 546 }, "size": [ 214.7587890625, @@ -466,7 +331,7 @@ "hidden": false, "paused": false, "params": { - "filename": "HelloMeme_00014-audio.mp4", + "filename": "HelloMeme_00065-audio.mp4", "subfolder": "", "type": "output", "format": "video/h264-mp4", @@ -475,17 +340,144 @@ "muted": false } } + }, + { + "id": 15, + "type": "HMPipelineVideo", + "pos": { + "0": 1443, + "1": 271 + }, + "size": { + "0": 315, + "1": 218 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "link": 40 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 33 + }, + { + "name": "drive_params", + "type": "DRIVE_PARAMS", + "link": 29 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 24 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMPipelineVideo" + }, + "widgets_values": [ + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 77866, + "randomize", + 2 + ] + }, + { + "id": 22, + "type": "GetVideoDriveParams", + "pos": { + "0": 1025, + "1": 557 + }, + "size": { + "0": 330, + "1": 98 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 27 + }, + { + "name": "images", + "type": "IMAGE", + "link": 28 + }, + { + "name": "ref_rt", + "type": "REFRT", + "link": 26 + } + ], + "outputs": [ + { + "name": "drive_params", + "type": "DRIVE_PARAMS", + "links": [ + 29 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetVideoDriveParams" + }, + "widgets_values": [ + 0.2 + ] + }, + { + "id": 31, + "type": "HMVideoSimplePipelineLoader", + "pos": { + "0": 91, + "1": 85 + }, + "size": { + "0": 428.4000244140625, + "1": 106 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 40 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoSimplePipelineLoader" + }, + "widgets_values": [ + "None", + "None", + 0 + ] } ], "links": [ - [ - 12, - 8, - 0, - 11, - 1, - "IMAGE" - ], [ 13, 2, @@ -494,14 +486,6 @@ 0, "FACE_TOOLKITS" ], - [ - 16, - 14, - 0, - 15, - 0, - "HMVIDEOPIPELINE" - ], [ 24, 15, @@ -581,16 +565,32 @@ 26, 0, "IMAGE" + ], + [ + 37, + 25, + 0, + 11, + 1, + "IMAGE" + ], + [ + 40, + 31, + 0, + 15, + 0, + "HMVIDEOPIPELINE" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.9090909090909091, + "scale": 0.8264462809917354, "offset": [ - 328.0506633321691, - 3.5667581298829703 + 509.5058815371377, + 98.92725456392243 ] } }, diff --git a/workflows/hellomeme_video_workflow.json b/workflows/hellomeme_video_workflow.json index 2ea7cb3..3629d0e 100644 --- a/workflows/hellomeme_video_workflow.json +++ b/workflows/hellomeme_video_workflow.json @@ -1,66 +1,13 @@ { - "last_node_id": 24, - "last_link_id": 30, + "last_node_id": 30, + "last_link_id": 39, "nodes": [ - { - "id": 15, - "type": "HMPipelineVideo", - "pos": { - "0": 1408, - "1": 390 - }, - "size": { - "0": 315, - "1": 218 - }, - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "pipeline", - "type": "HMVIDEOPIPELINE", - "link": 16 - }, - { - "name": "image", - "type": "IMAGE", - "link": 17 - }, - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "link": 29 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 24 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMPipelineVideo" - }, - "widgets_values": [ - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 97398, - "randomize", - 2 - ] - }, { "id": 2, "type": "HMFaceToolkitsLoader", "pos": { - "0": 124, - "1": 276 + "0": 106, + "1": 301 }, "size": { "0": 315, @@ -76,7 +23,8 @@ "type": "FACE_TOOLKITS", "links": [ 13, - 27 + 27, + 32 ], "slot_index": 0 } @@ -88,254 +36,19 @@ 0 ] }, - { - "id": 11, - "type": "GetReferenceImageRT", - "pos": { - "0": 536, - "1": 516 - }, - "size": { - "0": 241.79998779296875, - "1": 46 - }, - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 13 - }, - { - "name": "image", - "type": "IMAGE", - "link": 12 - } - ], - "outputs": [ - { - "name": "ref_rt", - "type": "REFRT", - "links": [ - 26 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetReferenceImageRT" - }, - "widgets_values": [] - }, - { - "id": 21, - "type": "VHS_VideoCombine", - "pos": { - "0": 1927, - "1": 604 - }, - "size": [ - 214.7587890625, - 518.7587890625 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 24 - }, - { - "name": "audio", - "type": "AUDIO", - "link": 25, - "shape": 7 - }, - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null, - "shape": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": null, - "shape": 7 - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": [] - } - ], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 15, - "loop_count": 0, - "filename_prefix": "HelloMeme", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 19, - "save_metadata": true, - "pingpong": false, - "save_output": true, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "HelloMeme_00016-audio.mp4", - "subfolder": "", - "type": "output", - "format": "video/h264-mp4", - "frame_rate": 15 - }, - "muted": false - } - } - }, - { - "id": 22, - "type": "GetVideoDriveParams", - "pos": { - "0": 932, - "1": 503 - }, - "size": { - "0": 330, - "1": 98 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 27 - }, - { - "name": "images", - "type": "IMAGE", - "link": 28 - }, - { - "name": "ref_rt", - "type": "REFRT", - "link": 26 - } - ], - "outputs": [ - { - "name": "drive_params", - "type": "DRIVE_PARAMS", - "links": [ - 29 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetVideoDriveParams" - }, - "widgets_values": [ - 0 - ] - }, - { - "id": 14, - "type": "HMVideoPipelineLoader", - "pos": { - "0": 116, - "1": 105 - }, - "size": { - "0": 352.79998779296875, - "1": 106 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "links": [ - 16 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMVideoPipelineLoader" - }, - "widgets_values": [ - "None", - "None", - 0 - ] - }, - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 120, - "1": 414 - }, - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 12, - 17 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, { "id": 20, "type": "VHS_LoadVideoFFmpeg", "pos": { - "0": 139, - "1": 793 + "0": 95, + "1": 822 + }, + "size": { + "0": 303.46875, + "1": 238 }, - "size": [ - 303.46875, - 535.46875 - ], "flags": {}, - "order": 3, + "order": 1, "mode": 0, "inputs": [ { @@ -405,17 +118,366 @@ "muted": false } } + }, + { + "id": 8, + "type": "LoadImage", + "pos": { + "0": 92, + "1": 441 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 31 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 26, + "type": "PreviewImage", + "pos": { + "0": 949, + "1": 36 + }, + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 34 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 25, + "type": "CropReferenceImage", + "pos": { + "0": 480, + "1": 279 + }, + "size": { + "0": 252, + "1": 46 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 31 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 32 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 33, + 34, + 37 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropReferenceImage" + }, + "widgets_values": [] + }, + { + "id": 11, + "type": "GetReferenceImageRT", + "pos": { + "0": 769, + "1": 415 + }, + "size": { + "0": 241.79998779296875, + "1": 46 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 13 + }, + { + "name": "image", + "type": "IMAGE", + "link": 37 + } + ], + "outputs": [ + { + "name": "ref_rt", + "type": "REFRT", + "links": [ + 26 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetReferenceImageRT" + }, + "widgets_values": [] + }, + { + "id": 21, + "type": "VHS_VideoCombine", + "pos": { + "0": 1856, + "1": 546 + }, + "size": [ + 214.7587890625, + 518.7587890625 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 24 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 25, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": [] + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "HelloMeme", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "HelloMeme_00062-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + }, + "muted": false + } + } + }, + { + "id": 15, + "type": "HMPipelineVideo", + "pos": { + "0": 1443, + "1": 271 + }, + "size": { + "0": 315, + "1": 218 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "link": 39 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 33 + }, + { + "name": "drive_params", + "type": "DRIVE_PARAMS", + "link": 29 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 24 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMPipelineVideo" + }, + "widgets_values": [ + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 81209, + "randomize", + 2 + ] + }, + { + "id": 30, + "type": "HMVideoPipelineLoader", + "pos": { + "0": 115, + "1": 100 + }, + "size": { + "0": 352.79998779296875, + "1": 106 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 39 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "None", + "None", + 0 + ] + }, + { + "id": 22, + "type": "GetVideoDriveParams", + "pos": { + "0": 1025, + "1": 557 + }, + "size": { + "0": 330, + "1": 98 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 27 + }, + { + "name": "images", + "type": "IMAGE", + "link": 28 + }, + { + "name": "ref_rt", + "type": "REFRT", + "link": 26 + } + ], + "outputs": [ + { + "name": "drive_params", + "type": "DRIVE_PARAMS", + "links": [ + 29 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetVideoDriveParams" + }, + "widgets_values": [ + 0.2 + ] } ], "links": [ - [ - 12, - 8, - 0, - 11, - 1, - "IMAGE" - ], [ 13, 2, @@ -424,22 +486,6 @@ 0, "FACE_TOOLKITS" ], - [ - 16, - 14, - 0, - 15, - 0, - "HMVIDEOPIPELINE" - ], - [ - 17, - 8, - 0, - 15, - 1, - "IMAGE" - ], [ 24, 15, @@ -487,16 +533,64 @@ 15, 2, "DRIVE_PARAMS" + ], + [ + 31, + 8, + 0, + 25, + 0, + "IMAGE" + ], + [ + 32, + 2, + 0, + 25, + 1, + "FACE_TOOLKITS" + ], + [ + 33, + 25, + 0, + 15, + 1, + "IMAGE" + ], + [ + 34, + 25, + 0, + 26, + 0, + "IMAGE" + ], + [ + 37, + 25, + 0, + 11, + 1, + "IMAGE" + ], + [ + 39, + 30, + 0, + 15, + 0, + "HMVIDEOPIPELINE" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.9090909090909091, + "scale": 0.8264462809917354, "offset": [ - 328.0506633321691, - 3.5667581298829703 + 509.5058815371377, + 97.31394584810211 ] } },