diff --git a/README.md b/README.md index 5545d79..51bc236 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,11 @@ ## πŸ”† New Features/Updates -- βœ… `11/14/2024` Added the `HMControlNet2` module, which uses the `PD-FGC` motion module to extract facial expression information (`drive_exp2`); restructured the ComfyUI interface; and optimized VRAM usage. +- βœ… `11/29/2024` a.Optimize the algorithm; b.Add VAE selection functionality; c.Introduce a super-resolution feature. + [![YouTube Demo](https://img.youtube.com/vi/fM5nyn6q02Y/0.jpg)](https://www.youtube.com/watch?v=fM5nyn6q02Y) + +- βœ… `11/14/2024` Added the `HMControlNet2` module, which uses the `PD-FGC` motion module to extract facial expression information (`drive_exp2`); restructured the ComfyUI interface; and optimized VRAM usage. [![YouTube Demo](https://img.youtube.com/vi/ZvoMHyRm310/0.jpg)](https://www.youtube.com/watch?v=ZvoMHyRm310) @@ -43,6 +46,19 @@ This repository is the official implementation of the [`HelloMeme`](https://arxi > > **Loras** under: `ComfyUI/models/checkpoints` +### workflows + +| workflow file | Video Generation | Image Generation | HMControlNet | HMControlNet2 | Super-Resolution | +|---------------|------------------|------------------|-----------|---------------|-----------------| +| hellomeme_image_workflow.json | | βœ… | βœ… | | | +| hellomeme_video_workflow.json | βœ… | | βœ… | | | +| hellomeme_image_sr_workflow.json | | βœ… | βœ… | | βœ… | +| hellomeme_video_sr_workflow.json | βœ… | | βœ… | | βœ… | +| hellomeme_image_v2_workflow.json | | βœ… | | βœ… | | +| hellomeme_video_v2_workflow.json | βœ… | | | βœ… | | +| hellomeme_image_v2_sr_workflow.json | | βœ… | | βœ… | βœ… | +| hellomeme_video_v2_sr_workflow.json | βœ… | | | βœ… | βœ… | + ### Image Generation Interface

diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..65055a1 Binary files /dev/null and b/environment.yml differ diff --git a/hellomeme/models/hm_blocks.py b/hellomeme/models/hm_blocks.py index 31e2175..f46bb98 100644 --- a/hellomeme/models/hm_blocks.py +++ b/hellomeme/models/hm_blocks.py @@ -136,7 +136,9 @@ class SKReferenceAttention(nn.Module): def forward(self, hidden_states, ref_stats, num_frames): h, w = hidden_states.shape[-2:] - ref_stats = ref_stats.repeat_interleave(num_frames, dim=0) + + if ref_stats.shape[0] != hidden_states.shape[0]: + ref_stats = ref_stats.repeat_interleave(num_frames, dim=0) cat_stats = torch.cat([hidden_states, ref_stats], dim=-1) cat_stats = rearrange(cat_stats.contiguous(), "b c h w -> (b h) w c") diff --git a/hellomeme/pipelines/pipline_hm_image.py b/hellomeme/pipelines/pipline_hm_image.py index d2cf3fc..3b9df7b 100644 --- a/hellomeme/pipelines/pipline_hm_image.py +++ b/hellomeme/pipelines/pipline_hm_image.py @@ -9,8 +9,8 @@ adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py """ +import copy from typing import Any, Callable, Dict, List, Optional, Union -from einops import rearrange import torch from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback @@ -29,28 +29,36 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): def caryomitosis(self, **kwargs): if not hasattr(self, "unet_ref"): self.unet_ref = HMDenoising3D.from_unet2d(self.unet) + self.unet_ref.cpu() if not isinstance(self.unet, HMDenoising3D): unet = HMDenoising3D.from_unet2d(unet=self.unet) # todo: δΈε€ŸδΌ˜ι›… del self.unet self.unet = unet + self.unet.cpu() + self.vae_decode = copy.deepcopy(self.vae) - def insert_hm_modules(self, dtype, device): + def insert_hm_modules(self, dtype): if isinstance(self.unet, HMDenoising3D): hm_adapter = HMReferenceAdapter.from_pretrained('songkey/hm_reference') self.unet.insert_reference_adapter(hm_adapter) - self.unet_ref = self.unet_ref.to(device=device, dtype=dtype).eval() - self.unet = self.unet.to(device=device, dtype=dtype).eval() + self.unet_ref.insert_reference_adapter(hm_adapter) + self.unet_ref.to(device='cpu', dtype=dtype).eval() + self.unet.to(device='cpu', dtype=dtype).eval() if hasattr(self, "mp_control"): del self.mp_control self.mp_control = HMControlNet.from_pretrained('songkey/hm_control') - self.mp_control = self.mp_control.to(device=device, dtype=dtype).eval() + self.mp_control.to(device='cpu', dtype=dtype).eval() if hasattr(self, "mp_control2"): del self.mp_control2 self.mp_control2 = HMControlNet2.from_pretrained('songkey/hm_control2') - self.mp_control2 = self.mp_control2.to(device=device, dtype=dtype).eval() + self.mp_control2.to(device='cpu', dtype=dtype).eval() + + self.vae.to(device='cpu', dtype=dtype).eval() + self.vae_decode.to(device='cpu', dtype=dtype).eval() + self.text_encoder.to(device='cpu', dtype=dtype).eval() @torch.no_grad() def __call__( @@ -71,6 +79,7 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): ip_adapter_image: Optional[PipelineImageInput] = None, ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, output_type: Optional[str] = "pil", + device: Optional[str] = "cpu", return_dict: bool = True, cross_attention_kwargs: Optional[Dict[str, Any]] = None, clip_skip: int = None, @@ -126,13 +135,14 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): else: batch_size = prompt_embeds.shape[0] - device = self.device + # device = self.device # 3. Encode input prompt text_encoder_lora_scale = ( self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None ) - print('device:', device) + + self.text_encoder.to(device=device) prompt_embeds, negative_prompt_embeds = self.encode_prompt( prompt, device, @@ -144,6 +154,8 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): lora_scale=text_encoder_lora_scale, clip_skip=self.clip_skip, ) + self.text_encoder.cpu() + # For classifier free guidance, we need to do two forward passes. # Here we concatenate the unconditional and text embeddings into a single batch # to avoid doing two forward passes @@ -169,10 +181,14 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + # 6. Prepare reference latents + self.vae.to(device=device) ref_latents = [ retrieve_latents(self.vae.encode(image[i: i + 1].to(device=device)), generator=generator) for i in range(batch_size) ] + self.vae.cpu() + ref_latents = torch.cat(ref_latents, dim=0) ref_latents = self.vae.config.scaling_factor * ref_latents @@ -219,6 +235,7 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): latent_model_input.unsqueeze(2), 0, encoder_hidden_states=prompt_embeds, + # control_hidden_states=control_latents, return_dict=False, )[1] self.unet_ref.cpu() @@ -237,6 +254,7 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): # 8. Denoising loop num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order self._num_timesteps = len(timesteps) + self.unet.to(device=device) with self.progress_bar(total=num_inference_steps) as progress_bar: for i, t in enumerate(timesteps): if self.interrupt: @@ -284,12 +302,16 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): step_idx = i // getattr(self.scheduler, "order", 1) callback(step_idx, t, latents) + self.unet.cpu() + + self.vae_decode.to(device=device) if not output_type == "latent": - image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + image = self.vae_decode.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ 0 ] else: image = latents + self.vae_decode.cpu() do_denormalize = [True] * image.shape[0] @@ -298,4 +320,4 @@ class HMImagePipeline(StableDiffusionImg2ImgPipeline): # Offload all models self.maybe_free_model_hooks() - return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=None) \ No newline at end of file + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=None), latents.detach().cpu() / self.vae.config.scaling_factor diff --git a/hellomeme/pipelines/pipline_hm_video.py b/hellomeme/pipelines/pipline_hm_video.py index 5fe337c..e9aaeff 100644 --- a/hellomeme/pipelines/pipline_hm_video.py +++ b/hellomeme/pipelines/pipline_hm_video.py @@ -8,6 +8,7 @@ @Desc : adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py """ + import copy from typing import Any, Callable, Dict, List, Optional, Union import torch @@ -22,45 +23,53 @@ from diffusers import StableDiffusionImg2ImgPipeline, MotionAdapter, EulerDiscre from ..models import HMDenoising3D, HMDenoisingMotion, HMControlNet, HMControlNet2 from ..models import HMReferenceAdapter -from ..utils import dicts_to_device, cat_dicts - +from ..utils import dicts_to_device, cat_dicts, cat_dicts_ref class HMVideoPipeline(StableDiffusionImg2ImgPipeline): def caryomitosis(self, patch_frames=12, **kwargs): if hasattr(self, "unet_ref"): del self.unet_ref self.unet_ref = HMDenoising3D.from_unet2d(self.unet) + self.unet_ref.cpu() + self.unet_pre = HMDenoising3D.from_unet2d(self.unet) + self.unet_pre.cpu() - if patch_frames >= 14: - self.num_frames = 16 - adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff", torch_dtype=torch.float16) - else: - self.num_frames = 12 - adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff_frame12", torch_dtype=torch.float16) + self.num_frames = 12 + adapter = MotionAdapter.from_pretrained("songkey/hm_animatediff_frame12", torch_dtype=torch.float16) unet = HMDenoisingMotion.from_unet2d(unet=self.unet, motion_adapter=adapter, load_weights=True) # todo: δΈε€ŸδΌ˜ι›… del self.unet self.unet = unet - def insert_hm_modules(self, dtype, device): + self.vae_decode = copy.deepcopy(self.vae) + + def insert_hm_modules(self, dtype): hm_adapter = HMReferenceAdapter.from_pretrained('songkey/hm_reference') if isinstance(self.unet, HMDenoisingMotion): self.unet.insert_reference_adapter(hm_adapter) - self.unet = self.unet.to(device=device, dtype=dtype).eval() + self.unet.to(device='cpu', dtype=dtype).eval() if hasattr(self, "unet_ref"): self.unet_ref.insert_reference_adapter(hm_adapter) - self.unet_ref = self.unet_ref.to(device=device, dtype=dtype).eval() + self.unet_ref.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "unet_pre"): + self.unet_pre.insert_reference_adapter(hm_adapter) + self.unet_pre.to(device='cpu', dtype=dtype).eval() if hasattr(self, "mp_control"): del self.mp_control self.mp_control = HMControlNet.from_pretrained('songkey/hm_control') - self.mp_control = self.mp_control.to(device=device, dtype=dtype).eval() + self.mp_control.to(device='cpu', dtype=dtype).eval() if hasattr(self, "mp_control2"): del self.mp_control2 self.mp_control2 = HMControlNet2.from_pretrained('songkey/hm_control2') - self.mp_control2 = self.mp_control2.to(device=device, dtype=dtype).eval() + self.mp_control2.to(device='cpu', dtype=dtype).eval() + + self.vae.to(device='cpu', dtype=dtype).eval() + self.vae_decode.to(device='cpu', dtype=dtype).eval() + self.text_encoder.to(device='cpu', dtype=dtype).eval() @torch.no_grad() def __call__( @@ -82,6 +91,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): ip_adapter_image: Optional[PipelineImageInput] = None, ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, output_type: Optional[str] = "pil", + device: Optional[str] = "cpu", return_dict: bool = True, cross_attention_kwargs: Optional[Dict[str, Any]] = None, clip_skip: int = None, @@ -137,12 +147,13 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): else: batch_size = prompt_embeds.shape[0] - device = self.device + # device = self.device # 3. Encode input prompt text_encoder_lora_scale = ( self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None ) + self.text_encoder.to(device=device) prompt_embeds, negative_prompt_embeds = self.encode_prompt( prompt, device, @@ -154,6 +165,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): lora_scale=text_encoder_lora_scale, clip_skip=self.clip_skip, ) + self.text_encoder.cpu() # For classifier free guidance, we need to do two forward passes. # Here we concatenate the unconditional and text embeddings into a single batch # to avoid doing two forward passes @@ -172,20 +184,16 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): # 4. Preprocess image image = self.image_processor.preprocess(image).to(device=device, dtype=prompt_embeds.dtype) - # 5. set timesteps - timesteps, num_inference_steps = retrieve_timesteps( - self.scheduler, num_inference_steps, device, timesteps, sigmas - ) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) - latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) - raw_video_len = drive_params['condition'].size(2) # 6. Prepare latent variables + self.vae.to(device=device) ref_latents = [ retrieve_latents(self.vae.encode(image[i: i + 1].to(device=device)), generator=generator) for i in range(batch_size) ] + self.vae.cpu() + ref_latents = torch.cat(ref_latents, dim=0) ref_latents = self.vae.config.scaling_factor * ref_latents c, h, w = ref_latents.shape[1:] @@ -201,14 +209,6 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): ) ref_latents = ref_latents.unsqueeze(2) - self.unet_ref.to(device=device) - cached_res = self.unet_ref( - torch.cat([ref_latents, ref_latents], dim=0) if - self.do_classifier_free_guidance else ref_latents, - 0, - encoder_hidden_states=prompt_embeds, - return_dict=False, - )[1] # 7.2 Optionally get Guidance Scale Embedding timestep_cond = None @@ -218,12 +218,23 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim ).to(device=device, dtype=prompt_embeds.dtype) + self.unet_ref.to(device=device) + cached_res = self.unet_ref( + torch.cat([torch.zeros_like(ref_latents), ref_latents], dim=0) if + self.do_classifier_free_guidance else ref_latents, + 0, + encoder_hidden_states=prompt_embeds, + return_dict=False, + )[1] + self.unet_ref.cpu() + pre_latents = [] control_latents = [] control_latent1_neg, control_latent2_neg = None, None base_noise = randn_tensor([batch_size, c, 1, h, w], dtype=prompt_embeds.dtype, generator=generator).to(device=device) self.mp_control.to(device=device) self.mp_control2.to(device=device) + self.unet_pre.to(device=device) with self.progress_bar(total=raw_video_len) as progress_bar: for idx in range(raw_video_len): condition = drive_params['condition'][:, :, idx:idx+1].clone().to(device=device) @@ -266,9 +277,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): beta_schedule="scaled_linear", ) - tmp_timesteps, _ = retrieve_timesteps( - scheduler, 8, device, None, sigmas - ) + tmp_timesteps, _ = retrieve_timesteps(scheduler, 8, device, None, sigmas) if idx == 0: pred_latent = scheduler.add_noise(ref_latents, base_noise, tmp_timesteps[:1]) else: @@ -278,7 +287,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): latent_model_input = torch.cat([pred_latent, pred_latent], dim=0) if \ self.do_classifier_free_guidance else pred_latent latent_model_input = scheduler.scale_model_input(latent_model_input, t) - noise_pred_chunk = self.unet_ref( + noise_pred_chunk = self.unet_pre( latent_model_input, t, encoder_hidden_states=prompt_embeds, @@ -303,6 +312,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): pre_latents = torch.cat(pre_latents, dim=2) self.mp_control.cpu() self.mp_control2.cpu() + self.unet_pre.cpu() self.unet_ref.cpu() chunk_size = self.num_frames @@ -328,20 +338,25 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): drive_idx_chunks[-1] = list(range(drive_idx_chunks[-1][0], paded_video_len)) - # 5. set timesteps - timesteps, num_inference_steps = retrieve_timesteps( - self.scheduler, num_inference_steps, device, None, sigmas + scheduler = EulerDiscreteScheduler( + num_train_timesteps=1000, + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", ) - timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + + # 5. set timesteps + timesteps, num_inference_steps = retrieve_timesteps(scheduler, num_inference_steps, device, None, sigmas) latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) # 8. Denoising loop - num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + num_warmup_steps = len(timesteps) - num_inference_steps * scheduler.order self._num_timesteps = len(timesteps) base_noise = randn_tensor([batch_size, c, paded_video_len, h, w], dtype=prompt_embeds.dtype, generator=generator) - latents = self.scheduler.add_noise(pre_latents, base_noise, latent_timestep) + latents = scheduler.add_noise(pre_latents, base_noise, latent_timestep) + self.unet.to(device=device) with self.progress_bar(total=num_inference_steps) as progress_bar: for i, t in enumerate(timesteps): noise_pred = torch.zeros([batch_size * 2 if self.do_classifier_free_guidance @@ -351,12 +366,13 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): latent_model_input_all = torch.cat([latents.clone(), latents.clone()], dim=0) if \ self.do_classifier_free_guidance else latents.clone() - latent_model_input_all = self.scheduler.scale_model_input(latent_model_input_all, t) + latent_model_input_all = scheduler.scale_model_input(latent_model_input_all, t) for cidx, chunk in enumerate(drive_idx_chunks): if self.interrupt: continue control_latent = cat_dicts([control_latents[ix] for ix in chunk], dim=2) + # cached_res = cat_dicts_ref([cached_latents[ix] for ix in chunk]) latent_model_input = latent_model_input_all[:, :, chunk].to(device=device) # predict the noise residual @@ -382,7 +398,7 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) # compute the previous noisy sample x_t -> x_t-1 - latents = self.scheduler.step(noise_pred.cpu(), t, latents, + latents = scheduler.step(noise_pred.cpu(), t, latents, **extra_step_kwargs, return_dict=False)[0] if callback_on_step_end is not None: @@ -396,22 +412,24 @@ class HMVideoPipeline(StableDiffusionImg2ImgPipeline): negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) # call the callback, if provided - if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0): progress_bar.set_description(f"GEN stage2") progress_bar.update() if callback is not None and i % callback_steps == 0: - step_idx = i // getattr(self.scheduler, "order", 1) + step_idx = i // getattr(scheduler, "order", 1) callback(step_idx, t, latents) + self.unet.cpu() + latents_res = latents / self.vae.config.scaling_factor - latents = latents / self.vae.config.scaling_factor - + self.vae_decode.to(device=device) res_frames = [] with self.progress_bar(total=raw_video_len) as progress_bar: for i in range(raw_video_len): - ret_tensor = self.vae.decode(latents[:, :, i, :, :].to(device=device), + ret_tensor = self.vae_decode.decode(latents_res[:, :, i, :, :].to(device=device), return_dict=False, generator=generator)[0] ret_image = self.image_processor.postprocess(ret_tensor, output_type=output_type) res_frames.append(ret_image) progress_bar.set_description(f"VAE DECODE") progress_bar.update() - return res_frames \ No newline at end of file + self.vae_decode.cpu() + return res_frames, latents_res diff --git a/hellomeme/utils.py b/hellomeme/utils.py index e42c983..05b5be2 100644 --- a/hellomeme/utils.py +++ b/hellomeme/utils.py @@ -18,10 +18,13 @@ import subprocess from einops import rearrange from .tools.utils import transform_points from .tools.hello_3dmm import get_project_points_rect -from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_unet_checkpoint from safetensors import safe_open +def get_torch_device(gpu_id=-1): + if gpu_id < 0: return torch.device("cpu") + gpu_id = min(max(-1, gpu_id), torch.cuda.device_count()) + return torch.device(f"cuda:{gpu_id}") def merge_dicts(dictl, dictr, wl=0.5): res = {} @@ -36,6 +39,11 @@ def cat_dicts(dicts, dim=0): res[k] = torch.cat([d[k].clone() for d in dicts], dim=dim) return res +def cat_dicts_ref(dicts): + res = {} + for k in dicts[0].keys(): + res[k] = rearrange(torch.cat([d[k].clone().unsqueeze(1) for d in dicts], dim=1), "b f c h w -> (b f) c h w ") + return res def dicts_to_device(dicts, device): ret = [] @@ -55,11 +63,6 @@ def load_safetensors(model_path): return tensors -def load_unet_from_safetensors(safetensors_path, unet_config): - original_stats = load_safetensors(safetensors_path) - return convert_ldm_unet_checkpoint(original_stats, unet_config) - - def image_preprocess(np_bgr, size, dtype=torch.float32): img_np = cv2.resize(np_bgr, size) return np_bgr_to_tensor(img_np, dtype) diff --git a/meme.py b/meme.py index 8518e3f..05f716b 100644 --- a/meme.py +++ b/meme.py @@ -1,16 +1,18 @@ import os.path as osp import random -import torch import numpy as np import cv2 import sys - from PIL import Image import subprocess +import torch +from einops import rearrange + import importlib.metadata +import folder_paths cur_dir = osp.dirname(osp.abspath(__file__)) @@ -33,61 +35,99 @@ from .hellomeme.utils import (get_drive_expression, get_drive_pose, crop_and_resize, det_landmarks, - load_data_list, - load_unet_from_safetensors + get_torch_device, + load_safetensors, ) - +from diffusers.pipelines.stable_diffusion.convert_from_ckpt import (convert_ldm_unet_checkpoint, + convert_ldm_vae_checkpoint) from .hellomeme.tools import Hello3DMMPred, HelloARKitBSPred, HelloFaceAlignment, HelloCameraDemo, FanEncoder from .hellomeme import HMImagePipeline, HMVideoPipeline from transformers import CLIPVisionModelWithProjection DEFAULT_PROMPT = '(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture' +def get_models_files(): + checkpoint_files = folder_paths.get_filename_list("checkpoints") + + vae_files = folder_paths.get_filename_list("vae") + vae_files = ["[vae] " + x for x in vae_files] + \ + ["[checkpoint] " + x for x in checkpoint_files] + + lora_files = folder_paths.get_filename_list("loras") + + return ['SD1.5'] + checkpoint_files, ['same as checkpoint', 'SD1.5 default vae'] + vae_files, ['None'] + lora_files + +def append_pipline_weights(pipeline, checkpoint=None, lora=None, vae=None): + ### load customized checkpoint or lora here: + ## checkpoints + + raw_stats = None + if checkpoint and not checkpoint.startswith('SD1.5'): + checkpoint_path = folder_paths.get_full_path_or_raise("checkpoints", checkpoint) + if osp.exists(checkpoint_path): + print("Loading checkpoint from", checkpoint_path) + if checkpoint_path.endswith('.safetensors'): + raw_stats = load_safetensors(checkpoint_path) + else: + raw_stats = torch.load(checkpoint_path) + + if raw_stats: + state_dict = convert_ldm_unet_checkpoint(raw_stats, pipeline.unet_ref.config) + if hasattr(pipeline, 'unet_pre'): + pipeline.unet_pre.load_state_dict(state_dict, strict=False) + pipeline.unet.load_state_dict(state_dict, strict=False) + + if vae and not vae.startswith('SD1.5 default vae'): + raw_vae_stats = raw_stats + real_vae_path = '' + if vae.startswith("[checkpoint] "): + real_vae_path = folder_paths.get_full_path_or_raise("checkpoints", vae.replace("[checkpoint] ", "")) + elif vae.startswith("[vae] "): + real_vae_path = folder_paths.get_full_path_or_raise("vae", vae.replace("[vae] ", "")) + if osp.isfile(real_vae_path): + print("Loading vae from", real_vae_path) + if real_vae_path.endswith('.safetensors'): + raw_vae_stats = load_safetensors(real_vae_path) + else: + raw_vae_stats = torch.load(real_vae_path) + + if raw_vae_stats: + vae_state_dict = convert_ldm_vae_checkpoint(raw_vae_stats, pipeline.vae_decode.config) + if hasattr(pipeline, 'vae_decode'): + pipeline.vae_decode.load_state_dict(vae_state_dict, strict=True) + + ### lora + if lora and not lora.startswith('None'): + lora_path = folder_paths.get_full_path_or_raise("loras", lora) + if osp.exists(lora_path): + print("Loading lora from", lora_path) + pipeline.load_lora_weights(osp.dirname(lora_path), weight_name=osp.basename(lora_path), adapter_name="lora") class HMImagePipelineLoader: @classmethod def INPUT_TYPES(s): - checkpoint_files = sorted(load_data_list(osp.join(cur_dir, '../../models/checkpoints'), '.pt;.pth;.ckpt;.safetensors')) - lora_files = sorted(load_data_list(osp.join(cur_dir, '../../models/loras'), '.safetensors')) + checkpoint_files, vae_files, lora_files = get_models_files() return { "optional": { - "checkpoint_path": (['SD1.5'] + checkpoint_files, ), - "lora_path": (['None'] + lora_files, ), - "gpu_id": ("INT", {"default": 0}), + "checkpoint": (checkpoint_files, ), + "lora": (lora_files, ), + "vae": (vae_files, ), } } RETURN_TYPES = ("HMIMAGEPIPELINE", ) RETURN_NAMES = ("hm_image_pipeline", ) FUNCTION = "load_pipeline" CATEGORY = "hellomeme" - def load_pipeline(self, checkpoint_path=None, lora_path=None, gpu_id=0): + def load_pipeline(self, checkpoint=None, lora=None, vae=None): dtype = torch.float16 - if gpu_id >= 0: - device = torch.device("cuda:{}".format(gpu_id)) - else: - device = torch.device("cpu") - pipeline = HMImagePipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5").to(dtype=dtype, device=device) + pipeline = HMImagePipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5") + pipeline.to(dtype=dtype) pipeline.caryomitosis() - ### load customized checkpoint or lora here: - ## checkpoints + append_pipline_weights(pipeline, checkpoint=checkpoint, lora=lora, vae=vae) - if checkpoint_path and osp.isfile(checkpoint_path): - if checkpoint_path.endswith('.safetensors'): - state_dict = load_unet_from_safetensors(checkpoint_path, pipeline.unet_ref.config) - pipeline.unet.load_state_dict(state_dict, strict=False) - elif osp.splitext(checkpoint_path)[-1] in ['.pt', '.pth', '.ckpt']: - state_dict = torch.load(checkpoint_path) - pipeline.unet.load_state_dict(state_dict, strict=False) - else: - print("Invalid checkpoint path", checkpoint_path) - - ### lora - if lora_path and osp.isfile(lora_path): - pipeline.load_lora_weights(osp.dirname(lora_path), weight_name=osp.basename(lora_path), adapter_name="lora") - - pipeline.insert_hm_modules(dtype=dtype, device=device) + pipeline.insert_hm_modules(dtype=dtype) return (pipeline, ) @@ -95,15 +135,14 @@ class HMImagePipelineLoader: class HMVideoPipelineLoader: @classmethod def INPUT_TYPES(s): - checkpoint_files = sorted(load_data_list(osp.join(cur_dir, '../../models/checkpoints'), '.pt;.pth;.ckpt;.safetensors')) - lora_files = sorted(load_data_list(osp.join(cur_dir, '../../models/loras'), '.safetensors')) + checkpoint_files, vae_files, lora_files = get_models_files() return { "optional": { - "checkpoint_path": (['SD1.5'] + checkpoint_files, ), - "lora_path": (['None'] + lora_files, ), - "patch_frames": ([12, 16], ), - "gpu_id": ("INT", {"default": 0}, ), + "checkpoint": (checkpoint_files, ), + "lora": (lora_files, ), + "vae": (vae_files, ), + "patch_frames": ([12], ), } } @@ -112,34 +151,15 @@ class HMVideoPipelineLoader: FUNCTION = "load_pipeline" CATEGORY = "hellomeme" - def load_pipeline(self, checkpoint_path=None, lora_path=None, patch_frames=12, gpu_id=0): + def load_pipeline(self, checkpoint=None, lora=None, vae=None, patch_frames=12): dtype = torch.float16 - if gpu_id >= 0: - device = torch.device("cuda:{}".format(gpu_id)) - else: - device = torch.device("cpu") - pipeline = HMVideoPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5").to(dtype=dtype, - device=device) + pipeline = HMVideoPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5") + pipeline.to(dtype=dtype) pipeline.caryomitosis(patch_frames=patch_frames) - ### load customized checkpoint or lora here: - ## checkpoints + append_pipline_weights(pipeline, checkpoint=checkpoint, lora=lora, vae=vae) - if checkpoint_path and osp.isfile(checkpoint_path): - if checkpoint_path.endswith('.safetensors'): - state_dict = load_unet_from_safetensors(checkpoint_path, pipeline.unet_ref.config) - pipeline.unet.load_state_dict(state_dict, strict=False) - elif osp.splitext(checkpoint_path)[-1] in ['.pt', '.pth', '.ckpt']: - state_dict = torch.load(checkpoint_path) - pipeline.unet.load_state_dict(state_dict, strict=False) - else: - print("Invalid checkpoint path", checkpoint_path) - - ### lora - if lora_path and osp.isfile(lora_path): - pipeline.load_lora_weights(osp.dirname(lora_path), weight_name=osp.basename(lora_path), adapter_name="lora") - - pipeline.insert_hm_modules(dtype=dtype, device=device) + pipeline.insert_hm_modules(dtype=dtype) return (pipeline,) @@ -149,7 +169,7 @@ class HMFaceToolkitsLoader: def INPUT_TYPES(s): return { "required": { - "gpu_id": ("INT", {"default": 0}), + "gpu_id": ("INT", {"default": 0, "min": -1, "max": 16}, ), } } @@ -159,20 +179,20 @@ class HMFaceToolkitsLoader: CATEGORY = "hellomeme" def load_face_toolkits(self, gpu_id): dtype = torch.float16 - if gpu_id < 0: - device = torch.device('cpu') - else: - device = torch.device(f'cuda:{gpu_id}') + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + 'h94/IP-Adapter', subfolder='models/image_encoder') + image_encoder.to(dtype=dtype).cpu() + pd_fpg_motion = FanEncoder.from_pretrained("songkey/pd_fgc_motion") + pd_fpg_motion.to(dtype=dtype).cpu() return ( dict( - device=device, + device=get_torch_device(gpu_id), dtype=dtype, - pd_fpg_motion=FanEncoder.from_pretrained("songkey/pd_fgc_motion").to(dtype=dtype), + pd_fpg_motion=pd_fpg_motion, face_aligner=HelloCameraDemo(face_alignment_module=HelloFaceAlignment(gpu_id=gpu_id), reset=False), harkit_bs=HelloARKitBSPred(gpu_id=gpu_id), h3dmm=Hello3DMMPred(gpu_id=gpu_id), - image_encoder=CLIPVisionModelWithProjection.from_pretrained( - 'h94/IP-Adapter', subfolder='models/image_encoder').to(dtype=dtype) + image_encoder=image_encoder ), ) @@ -307,9 +327,12 @@ class HMPipelineImage: "trans_ratio": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), "prompt": ("STRING", {"default": DEFAULT_PROMPT}), "negative_prompt": ("STRING", {"default": ''}), - "steps": ("INT", {"default": 25, "min": 1, "max": 1000}), - "seed": ("INT", {"default": -1, "min": -1, "max": 100000}), + "steps": ("INT", {"default": 25, "min": 1, "max": 10000, + "tooltip": "The number of steps used in the denoising process."}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, + "tooltip": "The random seed used for creating the noise."}), "guidance_scale": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 100.0, "step": 0.01}), + "gpu_id": ("INT", {"default": 0, "min": -1, "max": 16}, ), }, "optional": { "drive_exp": ("DRIVE_EXPRESSION", {"default": None},), @@ -317,7 +340,7 @@ class HMPipelineImage: } } - RETURN_TYPES = ("IMAGE",) + RETURN_TYPES = ("IMAGE", "LATENT", ) FUNCTION = "sample" CATEGORY = "hellomeme" @@ -332,9 +355,12 @@ class HMPipelineImage: prompt=DEFAULT_PROMPT, negative_prompt='', steps=25, - seed=-1, - guidance_scale=2.0 + seed=0, + guidance_scale=2.0, + gpu_id=0 ): + device = get_torch_device(gpu_id) + image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8) image_np = cv2.resize(image_np, (512, 512)) image_pil = Image.fromarray(image_np) @@ -357,12 +383,9 @@ class HMPipelineImage: if isinstance(drive_exp2, dict): drive_params.update(drive_exp2) - if seed < 0: - generator = torch.Generator().manual_seed(random.randint(0, 100000)) - else: - generator = torch.Generator().manual_seed(seed) + generator = torch.Generator().manual_seed(seed) - result_img = hm_image_pipeline( + result_img, latents = hm_image_pipeline( prompt=[prompt], strength=1.0, image=image_pil, @@ -371,9 +394,10 @@ class HMPipelineImage: negative_prompt=[negative_prompt], guidance_scale=guidance_scale, generator=generator, + device=device, output_type='np' ) - return (torch.from_numpy(np.clip(result_img[0], 0, 1)), ) + return (torch.from_numpy(np.clip(result_img[0], 0, 1)), dict(samples=latents), ) class HMPipelineVideo: @@ -389,9 +413,12 @@ class HMPipelineVideo: "patch_overlap": ("INT", {"default": 4, "min": 0, "max": 5}), "prompt": ("STRING", {"default": DEFAULT_PROMPT}), "negative_prompt": ("STRING", {"default": ''}), - "steps": ("INT", {"default": 25, "min": 1, "max": 1000}), - "seed": ("INT", {"default": -1, "min": -1, "max": 100000}), + "steps": ("INT", {"default": 25, "min": 1, "max": 10000, + "tooltip": "The number of steps used in the denoising process."}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, + "tooltip": "The random seed used for creating the noise."}), "guidance_scale": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 100.0, "step": 0.01}), + "gpu_id": ("INT", {"default": 0, "min": -1, "max": 16}, ), }, "optional": { "drive_exp": ("DRIVE_EXPRESSION", {"default": None},), @@ -399,7 +426,7 @@ class HMPipelineVideo: } } - RETURN_TYPES = ("IMAGE",) + RETURN_TYPES = ("IMAGE", "LATENT", ) FUNCTION = "sample" CATEGORY = "hellomeme" @@ -415,9 +442,12 @@ class HMPipelineVideo: prompt=DEFAULT_PROMPT, negative_prompt="", steps=25, - seed=-1, - guidance_scale=2.0 + seed=0, + guidance_scale=2.0, + gpu_id=0 ): + device = get_torch_device(gpu_id) + image_np = (ref_image[0] * 255).cpu().numpy().astype(np.uint8) image_np = cv2.resize(image_np, (512, 512)) image_pil = Image.fromarray(image_np) @@ -432,10 +462,7 @@ class HMPipelineVideo: _, ref_trans = face_toolkits['h3dmm'].forward_params(image_np, ref_landmark) - if seed < 0: - generator = torch.Generator().manual_seed(random.randint(0, 100000)) - else: - generator = torch.Generator().manual_seed(seed) + generator = torch.Generator().manual_seed(seed) drive_rot, drive_trans = drive_pose['rot'], drive_pose['trans'] condition = gen_control_heatmaps(drive_rot, drive_trans, ref_trans, 512, trans_ratio) @@ -446,7 +473,7 @@ class HMPipelineVideo: if isinstance(drive_exp2, dict): drive_params.update(drive_exp2) - res_frames = hm_video_pipeline( + res_frames, latents = hm_video_pipeline( prompt=[prompt], strength=1.0, image=image_pil, @@ -456,10 +483,13 @@ class HMPipelineVideo: negative_prompt=[negative_prompt], guidance_scale=guidance_scale, generator=generator, + device=device, output_type='np' ) res_frames = [np.clip(x[0], 0, 1) for x in res_frames] - return (torch.from_numpy(np.array(res_frames)), ) + latents = rearrange(latents[0], 'c f h w -> f c h w') + + return (torch.from_numpy(np.array(res_frames)), dict(samples=latents), ) NODE_CLASS_MAPPINGS = { diff --git a/workflows/hellomeme_image_example.png b/workflows/hellomeme_image_example.png index c5038a3..7cf0c38 100644 Binary files a/workflows/hellomeme_image_example.png and b/workflows/hellomeme_image_example.png differ diff --git a/workflows/hellomeme_image_sr_workflow.json b/workflows/hellomeme_image_sr_workflow.json new file mode 100644 index 0000000..9576293 --- /dev/null +++ b/workflows/hellomeme_image_sr_workflow.json @@ -0,0 +1,663 @@ +{ + "last_node_id": 65, + "last_link_id": 133, + "nodes": [ + { + "id": 39, + "type": "CropPortrait", + "pos": [ + 516, + 537 + ], + "size": [ + 241.79998779296875, + 46 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 128 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 57 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 118 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropPortrait" + }, + "widgets_values": [] + }, + { + "id": 44, + "type": "GetDrivePose", + "pos": [ + 852, + 661 + ], + "size": [ + 304.79998779296875, + 66 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 65 + }, + { + "name": "images", + "type": "IMAGE", + "link": 116 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 64 + } + ], + "outputs": [ + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "links": [ + 120 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDrivePose" + }, + "widgets_values": [] + }, + { + "id": 57, + "type": "LoadImage", + "pos": [ + 45, + 867 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 115, + 116, + 125 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "yao.jpg", + "image" + ] + }, + { + "id": 43, + "type": "GetFaceLandmarks", + "pos": [ + 465, + 771 + ], + "size": [ + 292.20001220703125, + 46 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 62 + }, + { + "name": "images", + "type": "IMAGE", + "link": 115 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 64, + 126 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": [ + 43, + 699 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 57, + 62, + 65, + 119, + 127 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 61, + "type": "GetDriveExpression", + "pos": [ + 857, + 823 + ], + "size": [ + 292.20001220703125, + 66 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 127 + }, + { + "name": "images", + "type": "IMAGE", + "link": 125 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 126 + } + ], + "outputs": [ + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "links": [ + 124 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDriveExpression" + }, + "widgets_values": [] + }, + { + "id": 62, + "type": "LoadImage", + "pos": [ + 52, + 289 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 128 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 63, + "type": "HMImagePipelineLoader", + "pos": [ + 817, + 358 + ], + "size": [ + 352.79998779296875, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "links": [ + 129 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMImagePipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 0 + ] + }, + { + "id": 58, + "type": "HMPipelineImage", + "pos": [ + 1274, + 584 + ], + "size": [ + 315, + 302 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "link": 129 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 119 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 118 + }, + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "link": 120 + }, + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "link": 124, + "shape": 7 + }, + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 131 + ], + "slot_index": 0 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "HMPipelineImage" + }, + "widgets_values": [ + 0, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 12021, + "randomize", + 2 + ] + }, + { + "id": 65, + "type": "ImageUpscaleWithModel", + "pos": [ + 1630, + 489 + ], + "size": [ + 340.20001220703125, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "upscale_model", + "type": "UPSCALE_MODEL", + "link": 130 + }, + { + "name": "image", + "type": "IMAGE", + "link": 131 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 133 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageUpscaleWithModel" + } + }, + { + "id": 59, + "type": "PreviewImage", + "pos": [ + 2006, + 588 + ], + "size": [ + 210, + 246 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 133 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 64, + "type": "UpscaleModelLoader", + "pos": [ + 1270, + 397 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "UPSCALE_MODEL", + "type": "UPSCALE_MODEL", + "links": [ + 130 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UpscaleModelLoader" + }, + "widgets_values": [ + "RealESRGAN_x2.pth" + ] + } + ], + "links": [ + [ + 57, + 2, + 0, + 39, + 1, + "FACE_TOOLKITS" + ], + [ + 62, + 2, + 0, + 43, + 0, + "FACE_TOOLKITS" + ], + [ + 64, + 43, + 0, + 44, + 2, + "FACELANDMARKS222" + ], + [ + 65, + 2, + 0, + 44, + 0, + "FACE_TOOLKITS" + ], + [ + 115, + 57, + 0, + 43, + 1, + "IMAGE" + ], + [ + 116, + 57, + 0, + 44, + 1, + "IMAGE" + ], + [ + 118, + 39, + 0, + 58, + 2, + "IMAGE" + ], + [ + 119, + 2, + 0, + 58, + 1, + "FACE_TOOLKITS" + ], + [ + 120, + 44, + 0, + 58, + 3, + "DRIVE_POSE" + ], + [ + 124, + 61, + 0, + 58, + 4, + "DRIVE_EXPRESSION" + ], + [ + 125, + 57, + 0, + 61, + 1, + "IMAGE" + ], + [ + 126, + 43, + 0, + 61, + 2, + "FACELANDMARKS222" + ], + [ + 127, + 2, + 0, + 61, + 0, + "FACE_TOOLKITS" + ], + [ + 128, + 62, + 0, + 39, + 0, + "IMAGE" + ], + [ + 129, + 63, + 0, + 58, + 0, + "HMIMAGEPIPELINE" + ], + [ + 130, + 64, + 0, + 65, + 0, + "UPSCALE_MODEL" + ], + [ + 131, + 58, + 0, + 65, + 1, + "IMAGE" + ], + [ + 133, + 65, + 0, + 59, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917359, + "offset": [ + 12.461092318990445, + -168.96359429993936 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/workflows/hellomeme_image_v2_sr_workflow.json b/workflows/hellomeme_image_v2_sr_workflow.json new file mode 100644 index 0000000..f710be0 --- /dev/null +++ b/workflows/hellomeme_image_v2_sr_workflow.json @@ -0,0 +1,664 @@ +{ + "last_node_id": 66, + "last_link_id": 136, + "nodes": [ + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": [ + 43, + 699 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 57, + 62, + 65, + 119, + 132 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 39, + "type": "CropPortrait", + "pos": [ + 516, + 537 + ], + "size": [ + 241.79998779296875, + 46 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 128 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 57 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 118 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropPortrait" + }, + "widgets_values": [] + }, + { + "id": 43, + "type": "GetFaceLandmarks", + "pos": [ + 465, + 771 + ], + "size": [ + 292.20001220703125, + 46 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 62 + }, + { + "name": "images", + "type": "IMAGE", + "link": 115 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 64, + 130 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 44, + "type": "GetDrivePose", + "pos": [ + 852, + 661 + ], + "size": [ + 304.79998779296875, + 66 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 65 + }, + { + "name": "images", + "type": "IMAGE", + "link": 116 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 64 + } + ], + "outputs": [ + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "links": [ + 120 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDrivePose" + }, + "widgets_values": [] + }, + { + "id": 57, + "type": "LoadImage", + "pos": [ + 45, + 867 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 115, + 116, + 131 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "yao.jpg", + "image" + ] + }, + { + "id": 58, + "type": "HMPipelineImage", + "pos": [ + 1274, + 584 + ], + "size": [ + 315, + 302 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "link": 129 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 119 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 118 + }, + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "link": 120 + }, + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "link": null, + "shape": 7 + }, + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "link": 133, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 134 + ], + "slot_index": 0 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "HMPipelineImage" + }, + "widgets_values": [ + 0, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 39876, + "randomize", + 2 + ] + }, + { + "id": 59, + "type": "PreviewImage", + "pos": [ + 2040, + 610 + ], + "size": [ + 210, + 246 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 135 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 62, + "type": "LoadImage", + "pos": [ + 52, + 289 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 128 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 63, + "type": "HMImagePipelineLoader", + "pos": [ + 817, + 358 + ], + "size": [ + 352.79998779296875, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "links": [ + 129 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMImagePipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 0 + ] + }, + { + "id": 64, + "type": "GetDriveExpression2", + "pos": [ + 850, + 827 + ], + "size": [ + 304.79998779296875, + 66 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 132 + }, + { + "name": "images", + "type": "IMAGE", + "link": 131 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 130 + } + ], + "outputs": [ + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "links": [ + 133 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDriveExpression2" + }, + "widgets_values": [] + }, + { + "id": 65, + "type": "UpscaleModelLoader", + "pos": [ + 1282, + 374 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "UPSCALE_MODEL", + "type": "UPSCALE_MODEL", + "links": [ + 136 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UpscaleModelLoader" + }, + "widgets_values": [ + "RealESRGAN_x2.pth" + ] + }, + { + "id": 66, + "type": "ImageUpscaleWithModel", + "pos": [ + 1644, + 503 + ], + "size": [ + 340.20001220703125, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "upscale_model", + "type": "UPSCALE_MODEL", + "link": 136 + }, + { + "name": "image", + "type": "IMAGE", + "link": 134 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 135 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageUpscaleWithModel" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 57, + 2, + 0, + 39, + 1, + "FACE_TOOLKITS" + ], + [ + 62, + 2, + 0, + 43, + 0, + "FACE_TOOLKITS" + ], + [ + 64, + 43, + 0, + 44, + 2, + "FACELANDMARKS222" + ], + [ + 65, + 2, + 0, + 44, + 0, + "FACE_TOOLKITS" + ], + [ + 115, + 57, + 0, + 43, + 1, + "IMAGE" + ], + [ + 116, + 57, + 0, + 44, + 1, + "IMAGE" + ], + [ + 118, + 39, + 0, + 58, + 2, + "IMAGE" + ], + [ + 119, + 2, + 0, + 58, + 1, + "FACE_TOOLKITS" + ], + [ + 120, + 44, + 0, + 58, + 3, + "DRIVE_POSE" + ], + [ + 128, + 62, + 0, + 39, + 0, + "IMAGE" + ], + [ + 129, + 63, + 0, + 58, + 0, + "HMIMAGEPIPELINE" + ], + [ + 130, + 43, + 0, + 64, + 2, + "FACELANDMARKS222" + ], + [ + 131, + 57, + 0, + 64, + 1, + "IMAGE" + ], + [ + 132, + 2, + 0, + 64, + 0, + "FACE_TOOLKITS" + ], + [ + 133, + 64, + 0, + 58, + 5, + "DRIVE_EXPRESSION2" + ], + [ + 134, + 58, + 0, + 66, + 1, + "IMAGE" + ], + [ + 135, + 66, + 0, + 59, + 0, + "IMAGE" + ], + [ + 136, + 65, + 0, + 66, + 0, + "UPSCALE_MODEL" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917359, + "offset": [ + 12.461092318990445, + -168.96359429993936 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/workflows/hellomeme_image_v2_workflow.json b/workflows/hellomeme_image_v2_workflow.json index 80c78fd..31a715f 100644 --- a/workflows/hellomeme_image_v2_workflow.json +++ b/workflows/hellomeme_image_v2_workflow.json @@ -1,127 +1,7 @@ { - "last_node_id": 60, - "last_link_id": 123, + "last_node_id": 64, + "last_link_id": 133, "nodes": [ - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 24, - "1": 133 - }, - "size": { - "0": 426.5283508300781, - "1": 476.4518127441406 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 56 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, - { - "id": 43, - "type": "GetFaceLandmarks", - "pos": { - "0": 465, - "1": 771 - }, - "size": { - "0": 292.20001220703125, - "1": 46 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 62 - }, - { - "name": "images", - "type": "IMAGE", - "link": 115 - } - ], - "outputs": [ - { - "name": "landmarks", - "type": "FACELANDMARKS222", - "links": [ - 64, - 112 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetFaceLandmarks" - }, - "widgets_values": [] - }, - { - "id": 57, - "type": "LoadImage", - "pos": { - "0": 45, - "1": 867 - }, - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 115, - 116, - 117 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "yao.jpg", - "image" - ] - }, { "id": 39, "type": "CropPortrait", @@ -140,7 +20,7 @@ { "name": "image", "type": "IMAGE", - "link": 56 + "link": 128 }, { "name": "face_toolkits", @@ -163,42 +43,6 @@ }, "widgets_values": [] }, - { - "id": 2, - "type": "HMFaceToolkitsLoader", - "pos": { - "0": 43, - "1": 699 - }, - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "links": [ - 57, - 62, - 65, - 114, - 119 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMFaceToolkitsLoader" - }, - "widgets_values": [ - 0 - ] - }, { "id": 44, "type": "GetDrivePose", @@ -263,7 +107,7 @@ { "name": "hm_image_pipeline", "type": "HMIMAGEPIPELINE", - "link": 123 + "link": 129 }, { "name": "face_toolkits", @@ -289,7 +133,7 @@ { "name": "drive_exp2", "type": "DRIVE_EXPRESSION2", - "link": 121, + "link": 133, "shape": 7 } ], @@ -311,45 +155,11 @@ "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", "", 25, - 55468, + 38454, "randomize", 2 ] }, - { - "id": 60, - "type": "HMImagePipelineLoader", - "pos": { - "0": 815, - "1": 390 - }, - "size": { - "0": 352.79998779296875, - "1": 106 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_image_pipeline", - "type": "HMIMAGEPIPELINE", - "links": [ - 123 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMImagePipelineLoader" - }, - "widgets_values": [ - "SD1.5", - "None", - 0 - ] - }, { "id": 59, "type": "PreviewImage", @@ -357,10 +167,10 @@ "0": 1688, "1": 584 }, - "size": [ - 210, - 246 - ], + "size": { + "0": 210, + "1": 246 + }, "flags": {}, "order": 9, "mode": 0, @@ -374,14 +184,206 @@ "outputs": [], "properties": { "Node name for S&R": "PreviewImage" - } + }, + "widgets_values": [] }, { - "id": 56, + "id": 62, + "type": "LoadImage", + "pos": { + "0": 52, + "1": 289 + }, + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 128 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 63, + "type": "HMImagePipelineLoader", + "pos": { + "0": 817, + "1": 358 + }, + "size": { + "0": 352.79998779296875, + "1": 130 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "links": [ + 129 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMImagePipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 0 + ] + }, + { + "id": 43, + "type": "GetFaceLandmarks", + "pos": { + "0": 465, + "1": 771 + }, + "size": { + "0": 292.20001220703125, + "1": 46 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 62 + }, + { + "name": "images", + "type": "IMAGE", + "link": 115 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 64, + 130 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 57, + "type": "LoadImage", + "pos": { + "0": 45, + "1": 867 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 115, + 116, + 131 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "yao.jpg", + "image" + ] + }, + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": { + "0": 43, + "1": 699 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 57, + 62, + 65, + 119, + 132 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 64, "type": "GetDriveExpression2", "pos": { - "0": 852, - "1": 806 + "0": 850, + "1": 827 }, "size": { "0": 304.79998779296875, @@ -394,17 +396,17 @@ { "name": "face_toolkits", "type": "FACE_TOOLKITS", - "link": 114 + "link": 132 }, { "name": "images", "type": "IMAGE", - "link": 117 + "link": 131 }, { "name": "landmarks", "type": "FACELANDMARKS222", - "link": 112 + "link": 130 } ], "outputs": [ @@ -412,7 +414,7 @@ "name": "drive_exp2", "type": "DRIVE_EXPRESSION2", "links": [ - 121 + 133 ], "slot_index": 0 } @@ -423,14 +425,6 @@ } ], "links": [ - [ - 56, - 8, - 0, - 39, - 0, - "IMAGE" - ], [ 57, 2, @@ -463,22 +457,6 @@ 0, "FACE_TOOLKITS" ], - [ - 112, - 43, - 0, - 56, - 2, - "FACELANDMARKS222" - ], - [ - 114, - 2, - 0, - 56, - 0, - "FACE_TOOLKITS" - ], [ 115, 57, @@ -495,14 +473,6 @@ 1, "IMAGE" ], - [ - 117, - 57, - 0, - 56, - 1, - "IMAGE" - ], [ 118, 39, @@ -527,14 +497,6 @@ 3, "DRIVE_POSE" ], - [ - 121, - 56, - 0, - 58, - 5, - "DRIVE_EXPRESSION2" - ], [ 122, 58, @@ -544,22 +506,62 @@ "IMAGE" ], [ - 123, - 60, + 128, + 62, + 0, + 39, + 0, + "IMAGE" + ], + [ + 129, + 63, 0, 58, 0, "HMIMAGEPIPELINE" + ], + [ + 130, + 43, + 0, + 64, + 2, + "FACELANDMARKS222" + ], + [ + 131, + 57, + 0, + 64, + 1, + "IMAGE" + ], + [ + 132, + 2, + 0, + 64, + 0, + "FACE_TOOLKITS" + ], + [ + 133, + 64, + 0, + 58, + 5, + "DRIVE_EXPRESSION2" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.7513148009015777, + "scale": 0.8264462809917359, "offset": [ - 328.1068082440709, - 3.5829604274896685 + 161.88372916607042, + -64.01845981558924 ] } }, diff --git a/workflows/hellomeme_image_workflow.json b/workflows/hellomeme_image_workflow.json index 47520c1..25220ea 100644 --- a/workflows/hellomeme_image_workflow.json +++ b/workflows/hellomeme_image_workflow.json @@ -1,45 +1,7 @@ { - "last_node_id": 61, - "last_link_id": 127, + "last_node_id": 63, + "last_link_id": 129, "nodes": [ - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 24, - "1": 133 - }, - "size": { - "0": 426.5283508300781, - "1": 476.4518127441406 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 56 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, { "id": 39, "type": "CropPortrait", @@ -52,13 +14,13 @@ "1": 46 }, "flags": {}, - "order": 4, + "order": 5, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", - "link": 56 + "link": 128 }, { "name": "face_toolkits", @@ -145,7 +107,7 @@ { "name": "hm_image_pipeline", "type": "HMIMAGEPIPELINE", - "link": 123 + "link": 129 }, { "name": "face_toolkits", @@ -193,45 +155,11 @@ "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", "", 25, - 63934, + 15604, "randomize", 2 ] }, - { - "id": 60, - "type": "HMImagePipelineLoader", - "pos": { - "0": 815, - "1": 390 - }, - "size": { - "0": 352.79998779296875, - "1": 106 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_image_pipeline", - "type": "HMIMAGEPIPELINE", - "links": [ - 123 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMImagePipelineLoader" - }, - "widgets_values": [ - "SD1.5", - "None", - 0 - ] - }, { "id": 59, "type": "PreviewImage", @@ -239,10 +167,10 @@ "0": 1688, "1": 584 }, - "size": [ - 210, - 246 - ], + "size": { + "0": 210, + "1": 246 + }, "flags": {}, "order": 9, "mode": 0, @@ -256,7 +184,8 @@ "outputs": [], "properties": { "Node name for S&R": "PreviewImage" - } + }, + "widgets_values": [] }, { "id": 57, @@ -265,12 +194,12 @@ "0": 45, "1": 867 }, - "size": [ - 315, - 314 - ], + "size": { + "0": 315, + "1": 314 + }, "flags": {}, - "order": 2, + "order": 0, "mode": 0, "inputs": [], "outputs": [ @@ -310,7 +239,7 @@ "1": 46 }, "flags": {}, - "order": 5, + "order": 4, "mode": 0, "inputs": [ { @@ -352,7 +281,7 @@ "1": 58 }, "flags": {}, - "order": 3, + "order": 1, "mode": 0, "inputs": [], "outputs": [ @@ -419,18 +348,84 @@ ], "properties": { "Node name for S&R": "GetDriveExpression" - } + }, + "widgets_values": [] + }, + { + "id": 62, + "type": "LoadImage", + "pos": { + "0": 52, + "1": 289 + }, + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 128 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 63, + "type": "HMImagePipelineLoader", + "pos": { + "0": 817, + "1": 358 + }, + "size": { + "0": 352.79998779296875, + "1": 130 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_image_pipeline", + "type": "HMIMAGEPIPELINE", + "links": [ + 129 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMImagePipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 0 + ] } ], "links": [ - [ - 56, - 8, - 0, - 39, - 0, - "IMAGE" - ], [ 57, 2, @@ -511,14 +506,6 @@ 0, "IMAGE" ], - [ - 123, - 60, - 0, - 58, - 0, - "HMIMAGEPIPELINE" - ], [ 124, 61, @@ -550,16 +537,32 @@ 61, 0, "FACE_TOOLKITS" + ], + [ + 128, + 62, + 0, + 39, + 0, + "IMAGE" + ], + [ + 129, + 63, + 0, + 58, + 0, + "HMIMAGEPIPELINE" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.7513148009015777, + "scale": 0.8264462809917359, "offset": [ - 328.1068082440709, - 3.5829604274896685 + 161.88372916607042, + -64.01845981558924 ] } }, diff --git a/workflows/hellomeme_video_example.png b/workflows/hellomeme_video_example.png index ac64683..6a8bcb8 100644 Binary files a/workflows/hellomeme_video_example.png and b/workflows/hellomeme_video_example.png differ diff --git a/workflows/hellomeme_video_sr_workflow.json b/workflows/hellomeme_video_sr_workflow.json new file mode 100644 index 0000000..65ececb --- /dev/null +++ b/workflows/hellomeme_video_sr_workflow.json @@ -0,0 +1,766 @@ +{ + "last_node_id": 62, + "last_link_id": 125, + "nodes": [ + { + "id": 39, + "type": "CropPortrait", + "pos": [ + 516, + 537 + ], + "size": [ + 241.79998779296875, + 46 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 116 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 57 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 99 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropPortrait" + }, + "widgets_values": [] + }, + { + "id": 58, + "type": "LoadImage", + "pos": [ + 56, + 273 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 116 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 44, + "type": "GetDrivePose", + "pos": [ + 842, + 677 + ], + "size": [ + 304.79998779296875, + 66 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 65 + }, + { + "name": "images", + "type": "IMAGE", + "link": 83 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 64 + } + ], + "outputs": [ + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "links": [ + 100 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDrivePose" + }, + "widgets_values": [] + }, + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": [ + 43, + 699 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 57, + 62, + 65, + 97, + 119 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 43, + "type": "GetFaceLandmarks", + "pos": [ + 465, + 771 + ], + "size": [ + 292.20001220703125, + 46 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 62 + }, + { + "name": "images", + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 64, + 120 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 48, + "type": "VHS_LoadVideo", + "pos": [ + 106, + 867 + ], + "size": [ + 247.455078125, + 262 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82, + 83, + 121 + ], + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 118 + ], + "slot_index": 2 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "jgz.mp4", + "force_rate": 15, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "force_rate": 15, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "filename": "jgz.mp4", + "type": "input", + "format": "video/mp4" + }, + "muted": false + } + } + }, + { + "id": 60, + "type": "GetDriveExpression", + "pos": [ + 845, + 813 + ], + "size": [ + 292.20001220703125, + 66 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 119 + }, + { + "name": "images", + "type": "IMAGE", + "link": 121 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 120 + } + ], + "outputs": [ + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "links": [ + 122 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDriveExpression" + }, + "widgets_values": [] + }, + { + "id": 53, + "type": "HMPipelineVideo", + "pos": [ + 1258, + 562 + ], + "size": [ + 315, + 326 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "link": 115 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 97 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 99 + }, + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "link": 100 + }, + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "link": 122, + "shape": 7 + }, + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 124 + ], + "slot_index": 0 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "HMPipelineVideo" + }, + "widgets_values": [ + 0, + 4, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 98354, + "fixed", + 2 + ] + }, + { + "id": 62, + "type": "ImageUpscaleWithModel", + "pos": [ + 1617, + 490 + ], + "size": [ + 340.20001220703125, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "upscale_model", + "type": "UPSCALE_MODEL", + "link": 123 + }, + { + "name": "image", + "type": "IMAGE", + "link": 124 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 125 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageUpscaleWithModel" + } + }, + { + "id": 59, + "type": "VHS_VideoCombine", + "pos": [ + 2023, + 561 + ], + "size": [ + 214.7587890625, + 518.7587890625 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 125 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 118, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 18, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00054-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + }, + "muted": false + } + } + }, + { + "id": 61, + "type": "UpscaleModelLoader", + "pos": [ + 1261, + 425 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "UPSCALE_MODEL", + "type": "UPSCALE_MODEL", + "links": [ + 123 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UpscaleModelLoader" + }, + "widgets_values": [ + "RealESRGAN_x2.pth" + ] + }, + { + "id": 57, + "type": "HMVideoPipelineLoader", + "pos": [ + 813, + 343 + ], + "size": [ + 352.79998779296875, + 154 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 115 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 12, + 0 + ] + } + ], + "links": [ + [ + 57, + 2, + 0, + 39, + 1, + "FACE_TOOLKITS" + ], + [ + 62, + 2, + 0, + 43, + 0, + "FACE_TOOLKITS" + ], + [ + 64, + 43, + 0, + 44, + 2, + "FACELANDMARKS222" + ], + [ + 65, + 2, + 0, + 44, + 0, + "FACE_TOOLKITS" + ], + [ + 82, + 48, + 0, + 43, + 1, + "IMAGE" + ], + [ + 83, + 48, + 0, + 44, + 1, + "IMAGE" + ], + [ + 97, + 2, + 0, + 53, + 1, + "FACE_TOOLKITS" + ], + [ + 99, + 39, + 0, + 53, + 2, + "IMAGE" + ], + [ + 100, + 44, + 0, + 53, + 3, + "DRIVE_POSE" + ], + [ + 115, + 57, + 0, + 53, + 0, + "HMVIDEOPIPELINE" + ], + [ + 116, + 58, + 0, + 39, + 0, + "IMAGE" + ], + [ + 118, + 48, + 2, + 59, + 1, + "AUDIO" + ], + [ + 119, + 2, + 0, + 60, + 0, + "FACE_TOOLKITS" + ], + [ + 120, + 43, + 0, + 60, + 2, + "FACELANDMARKS222" + ], + [ + 121, + 48, + 0, + 60, + 1, + "IMAGE" + ], + [ + 122, + 60, + 0, + 53, + 4, + "DRIVE_EXPRESSION" + ], + [ + 123, + 61, + 0, + 62, + 0, + "UPSCALE_MODEL" + ], + [ + 124, + 53, + 0, + 62, + 1, + "IMAGE" + ], + [ + 125, + 62, + 0, + 59, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.9090909090909095, + "offset": [ + 108.08728006313046, + -158.7703124395875 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/workflows/hellomeme_video_v2_sr_workflow.json b/workflows/hellomeme_video_v2_sr_workflow.json new file mode 100644 index 0000000..21af6d8 --- /dev/null +++ b/workflows/hellomeme_video_v2_sr_workflow.json @@ -0,0 +1,765 @@ +{ + "last_node_id": 62, + "last_link_id": 121, + "nodes": [ + { + "id": 39, + "type": "CropPortrait", + "pos": [ + 516, + 537 + ], + "size": [ + 241.79998779296875, + 46 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 116 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 57 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 99 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropPortrait" + }, + "widgets_values": [] + }, + { + "id": 43, + "type": "GetFaceLandmarks", + "pos": [ + 465, + 771 + ], + "size": [ + 292.20001220703125, + 46 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 62 + }, + { + "name": "images", + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 64, + 113 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": [ + 43, + 699 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 57, + 62, + 65, + 97, + 114 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 58, + "type": "LoadImage", + "pos": [ + 56, + 273 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 116 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 44, + "type": "GetDrivePose", + "pos": [ + 842, + 677 + ], + "size": [ + 304.79998779296875, + 66 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 65 + }, + { + "name": "images", + "type": "IMAGE", + "link": 83 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 64 + } + ], + "outputs": [ + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "links": [ + 100 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDrivePose" + }, + "widgets_values": [] + }, + { + "id": 56, + "type": "GetDriveExpression2", + "pos": [ + 844, + 816 + ], + "size": [ + 304.79998779296875, + 66 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 114 + }, + { + "name": "images", + "type": "IMAGE", + "link": 112 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 113 + } + ], + "outputs": [ + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "links": [ + 111 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDriveExpression2" + }, + "widgets_values": [] + }, + { + "id": 48, + "type": "VHS_LoadVideo", + "pos": [ + 106, + 867 + ], + "size": [ + 247.455078125, + 262 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82, + 83, + 112 + ], + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 118 + ], + "slot_index": 2 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "jgz.mp4", + "force_rate": 15, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "force_rate": 15, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "filename": "jgz.mp4", + "type": "input", + "format": "video/mp4" + }, + "muted": false + } + } + }, + { + "id": 53, + "type": "HMPipelineVideo", + "pos": [ + 1258, + 562 + ], + "size": [ + 315, + 326 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "link": 115 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 97 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 99 + }, + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "link": 100 + }, + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "link": null, + "shape": 7 + }, + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "link": 111, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 120 + ], + "slot_index": 0 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "HMPipelineVideo" + }, + "widgets_values": [ + 0, + 4, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 98354, + "fixed", + 2 + ] + }, + { + "id": 62, + "type": "ImageUpscaleWithModel", + "pos": [ + 1620, + 494 + ], + "size": [ + 340.20001220703125, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "upscale_model", + "type": "UPSCALE_MODEL", + "link": 121 + }, + { + "name": "image", + "type": "IMAGE", + "link": 120 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 119 + ] + } + ], + "properties": { + "Node name for S&R": "ImageUpscaleWithModel" + } + }, + { + "id": 57, + "type": "HMVideoPipelineLoader", + "pos": [ + 813, + 343 + ], + "size": [ + 352.79998779296875, + 154 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 115 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 12, + 0 + ] + }, + { + "id": 60, + "type": "UpscaleModelLoader", + "pos": [ + 1255, + 397 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "UPSCALE_MODEL", + "type": "UPSCALE_MODEL", + "links": [ + 121 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UpscaleModelLoader" + }, + "widgets_values": [ + "RealESRGAN_x2.pth" + ] + }, + { + "id": 59, + "type": "VHS_VideoCombine", + "pos": [ + 2019, + 574 + ], + "size": [ + 562.3756713867188, + 866.3756713867188 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 119 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 118, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 18, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00087-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + }, + "muted": false + } + } + } + ], + "links": [ + [ + 57, + 2, + 0, + 39, + 1, + "FACE_TOOLKITS" + ], + [ + 62, + 2, + 0, + 43, + 0, + "FACE_TOOLKITS" + ], + [ + 64, + 43, + 0, + 44, + 2, + "FACELANDMARKS222" + ], + [ + 65, + 2, + 0, + 44, + 0, + "FACE_TOOLKITS" + ], + [ + 82, + 48, + 0, + 43, + 1, + "IMAGE" + ], + [ + 83, + 48, + 0, + 44, + 1, + "IMAGE" + ], + [ + 97, + 2, + 0, + 53, + 1, + "FACE_TOOLKITS" + ], + [ + 99, + 39, + 0, + 53, + 2, + "IMAGE" + ], + [ + 100, + 44, + 0, + 53, + 3, + "DRIVE_POSE" + ], + [ + 111, + 56, + 0, + 53, + 5, + "DRIVE_EXPRESSION2" + ], + [ + 112, + 48, + 0, + 56, + 1, + "IMAGE" + ], + [ + 113, + 43, + 0, + 56, + 2, + "FACELANDMARKS222" + ], + [ + 114, + 2, + 0, + 56, + 0, + "FACE_TOOLKITS" + ], + [ + 115, + 57, + 0, + 53, + 0, + "HMVIDEOPIPELINE" + ], + [ + 116, + 58, + 0, + 39, + 0, + "IMAGE" + ], + [ + 118, + 48, + 2, + 59, + 1, + "AUDIO" + ], + [ + 119, + 62, + 0, + 59, + 0, + "IMAGE" + ], + [ + 120, + 53, + 0, + 62, + 1, + "IMAGE" + ], + [ + 121, + 60, + 0, + 62, + 0, + "UPSCALE_MODEL" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917359, + "offset": [ + 352.06785745570926, + -160.8969030157596 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/workflows/hellomeme_video_v2_workflow.json b/workflows/hellomeme_video_v2_workflow.json index 529f4de..b8266ce 100644 --- a/workflows/hellomeme_video_v2_workflow.json +++ b/workflows/hellomeme_video_v2_workflow.json @@ -1,163 +1,7 @@ { - "last_node_id": 56, - "last_link_id": 114, + "last_node_id": 59, + "last_link_id": 118, "nodes": [ - { - "id": 44, - "type": "GetDrivePose", - "pos": { - "0": 852, - "1": 661 - }, - "size": { - "0": 304.79998779296875, - "1": 66 - }, - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 65 - }, - { - "name": "images", - "type": "IMAGE", - "link": 83 - }, - { - "name": "landmarks", - "type": "FACELANDMARKS222", - "link": 64 - } - ], - "outputs": [ - { - "name": "drive_pose", - "type": "DRIVE_POSE", - "links": [ - 100 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetDrivePose" - }, - "widgets_values": [] - }, - { - "id": 53, - "type": "HMPipelineVideo", - "pos": { - "0": 1258, - "1": 562 - }, - "size": { - "0": 315, - "1": 326 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "link": 98 - }, - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 97 - }, - { - "name": "ref_image", - "type": "IMAGE", - "link": 99 - }, - { - "name": "drive_pose", - "type": "DRIVE_POSE", - "link": 100 - }, - { - "name": "drive_exp", - "type": "DRIVE_EXPRESSION", - "link": null, - "shape": 7 - }, - { - "name": "drive_exp2", - "type": "DRIVE_EXPRESSION2", - "link": 111, - "shape": 7 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 96 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMPipelineVideo" - }, - "widgets_values": [ - 0, - 4, - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 98354, - "fixed", - 2 - ] - }, - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 24, - "1": 133 - }, - "size": { - "0": 426.5283508300781, - "1": 476.4518127441406 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 56 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, { "id": 39, "type": "CropPortrait", @@ -176,7 +20,7 @@ { "name": "image", "type": "IMAGE", - "link": 56 + "link": 116 }, { "name": "face_toolkits", @@ -199,125 +43,6 @@ }, "widgets_values": [] }, - { - "id": 52, - "type": "VHS_VideoCombine", - "pos": { - "0": 1658, - "1": 894 - }, - "size": [ - 289.33306884765625, - 593.3330688476562 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 96 - }, - { - "name": "audio", - "type": "AUDIO", - "link": 92, - "shape": 7 - }, - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null, - "shape": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": null, - "shape": 7 - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null - } - ], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 15, - "loop_count": 0, - "filename_prefix": "AnimateDiff", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 18, - "save_metadata": true, - "pingpong": false, - "save_output": true, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "AnimateDiff_00026-audio.mp4", - "subfolder": "", - "type": "output", - "format": "video/h264-mp4", - "frame_rate": 15 - }, - "muted": false - } - } - }, - { - "id": 56, - "type": "GetDriveExpression2", - "pos": { - "0": 852, - "1": 801 - }, - "size": { - "0": 304.79998779296875, - "1": 66 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 114 - }, - { - "name": "images", - "type": "IMAGE", - "link": 112 - }, - { - "name": "landmarks", - "type": "FACELANDMARKS222", - "link": 113 - } - ], - "outputs": [ - { - "name": "drive_exp2", - "type": "DRIVE_EXPRESSION2", - "links": [ - 111 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetDriveExpression2" - }, - "widgets_values": [] - }, { "id": 43, "type": "GetFaceLandmarks", @@ -372,7 +97,7 @@ "1": 58 }, "flags": {}, - "order": 1, + "order": 0, "mode": 0, "inputs": [], "outputs": [ @@ -397,53 +122,293 @@ ] }, { - "id": 49, - "type": "HMVideoPipelineLoader", + "id": 53, + "type": "HMPipelineVideo", "pos": { - "0": 849, - "1": 375 + "0": 1258, + "1": 562 }, "size": { - "0": 352.79998779296875, - "1": 130 + "0": 315, + "1": 326 }, "flags": {}, - "order": 2, + "order": 8, "mode": 0, - "inputs": [], - "outputs": [ + "inputs": [ { "name": "hm_video_pipeline", "type": "HMVIDEOPIPELINE", + "link": 115 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 97 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 99 + }, + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "link": 100 + }, + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "link": null, + "shape": 7 + }, + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "link": 111, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", "links": [ - 98 + 117 ], "slot_index": 0 } ], "properties": { - "Node name for S&R": "HMVideoPipelineLoader" + "Node name for S&R": "HMPipelineVideo" }, "widgets_values": [ - "SD1.5", - "None", - 12, - 0 + 0, + 4, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 98354, + "fixed", + 2 ] }, + { + "id": 58, + "type": "LoadImage", + "pos": { + "0": 56, + "1": 273 + }, + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 116 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, + { + "id": 44, + "type": "GetDrivePose", + "pos": { + "0": 842, + "1": 677 + }, + "size": { + "0": 304.79998779296875, + "1": 66 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 65 + }, + { + "name": "images", + "type": "IMAGE", + "link": 83 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 64 + } + ], + "outputs": [ + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "links": [ + 100 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDrivePose" + }, + "widgets_values": [] + }, + { + "id": 56, + "type": "GetDriveExpression2", + "pos": { + "0": 844, + "1": 816 + }, + "size": { + "0": 304.79998779296875, + "1": 66 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 114 + }, + { + "name": "images", + "type": "IMAGE", + "link": 112 + }, + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "link": 113 + } + ], + "outputs": [ + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "links": [ + 111 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetDriveExpression2" + }, + "widgets_values": [] + }, + { + "id": 59, + "type": "VHS_VideoCombine", + "pos": { + "0": 1619, + "1": 800 + }, + "size": [ + 214.7587890625, + 518.7587890625 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 117 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 118, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 18, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00054-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + }, + "muted": false + } + } + }, { "id": 48, "type": "VHS_LoadVideo", "pos": { - "0": 75, - "1": 900 + "0": 106, + "1": 867 }, "size": [ 247.455078125, 503.455078125 ], "flags": {}, - "order": 3, + "order": 2, "mode": 0, "inputs": [ { @@ -479,7 +444,7 @@ "name": "audio", "type": "AUDIO", "links": [ - 92 + 118 ], "slot_index": 2 }, @@ -517,17 +482,45 @@ "muted": false } } + }, + { + "id": 57, + "type": "HMVideoPipelineLoader", + "pos": { + "0": 813, + "1": 343 + }, + "size": { + "0": 352.79998779296875, + "1": 154 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 115 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 12, + 0 + ] } ], "links": [ - [ - 56, - 8, - 0, - 39, - 0, - "IMAGE" - ], [ 57, 2, @@ -576,22 +569,6 @@ 1, "IMAGE" ], - [ - 92, - 48, - 2, - 52, - 1, - "AUDIO" - ], - [ - 96, - 53, - 0, - 52, - 0, - "IMAGE" - ], [ 97, 2, @@ -600,14 +577,6 @@ 1, "FACE_TOOLKITS" ], - [ - 98, - 49, - 0, - 53, - 0, - "HMVIDEOPIPELINE" - ], [ 99, 39, @@ -655,16 +624,48 @@ 56, 0, "FACE_TOOLKITS" + ], + [ + 115, + 57, + 0, + 53, + 0, + "HMVIDEOPIPELINE" + ], + [ + 116, + 58, + 0, + 39, + 0, + "IMAGE" + ], + [ + 117, + 53, + 0, + 59, + 0, + "IMAGE" + ], + [ + 118, + 48, + 2, + 59, + 1, + "AUDIO" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.7513148009015777, + "scale": 0.8264462809917359, "offset": [ - 760.8618864750447, - 109.23033266581292 + 159.4635814609923, + -64.01844135245447 ] } }, diff --git a/workflows/hellomeme_video_workflow.json b/workflows/hellomeme_video_workflow.json index 6ddf007..ba7cdee 100644 --- a/workflows/hellomeme_video_workflow.json +++ b/workflows/hellomeme_video_workflow.json @@ -1,13 +1,164 @@ { - "last_node_id": 55, - "last_link_id": 110, + "last_node_id": 60, + "last_link_id": 122, "nodes": [ + { + "id": 39, + "type": "CropPortrait", + "pos": { + "0": 516, + "1": 537 + }, + "size": { + "0": 241.79998779296875, + "1": 46 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 116 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 57 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 99 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CropPortrait" + }, + "widgets_values": [] + }, + { + "id": 53, + "type": "HMPipelineVideo", + "pos": { + "0": 1258, + "1": 562 + }, + "size": { + "0": 315, + "1": 326 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "link": 115 + }, + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 97 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 99 + }, + { + "name": "drive_pose", + "type": "DRIVE_POSE", + "link": 100 + }, + { + "name": "drive_exp", + "type": "DRIVE_EXPRESSION", + "link": 122, + "shape": 7 + }, + { + "name": "drive_exp2", + "type": "DRIVE_EXPRESSION2", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 117 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMPipelineVideo" + }, + "widgets_values": [ + 0, + 4, + "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", + "", + 25, + 98354, + "fixed", + 2 + ] + }, + { + "id": 58, + "type": "LoadImage", + "pos": { + "0": 56, + "1": 273 + }, + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 116 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "majicmix2.jpg", + "image" + ] + }, { "id": 44, "type": "GetDrivePose", "pos": { - "0": 852, - "1": 661 + "0": 842, + "1": 677 }, "size": { "0": 304.79998779296875, @@ -49,331 +200,15 @@ "widgets_values": [] }, { - "id": 53, - "type": "HMPipelineVideo", - "pos": { - "0": 1258, - "1": 562 - }, - "size": { - "0": 315, - "1": 326 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "link": 98 - }, - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 97 - }, - { - "name": "ref_image", - "type": "IMAGE", - "link": 99 - }, - { - "name": "drive_pose", - "type": "DRIVE_POSE", - "link": 100 - }, - { - "name": "drive_exp", - "type": "DRIVE_EXPRESSION", - "link": 110, - "shape": 7 - }, - { - "name": "drive_exp2", - "type": "DRIVE_EXPRESSION2", - "link": null, - "shape": 7 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 96 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMPipelineVideo" - }, - "widgets_values": [ - 0, - 4, - "(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture", - "", - 25, - 98354, - "fixed", - 2 - ] - }, - { - "id": 2, - "type": "HMFaceToolkitsLoader", - "pos": { - "0": 43, - "1": 699 - }, - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "links": [ - 57, - 62, - 65, - 97, - 109 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMFaceToolkitsLoader" - }, - "widgets_values": [ - 0 - ] - }, - { - "id": 8, - "type": "LoadImage", - "pos": { - "0": 24, - "1": 133 - }, - "size": { - "0": 426.5283508300781, - "1": 476.4518127441406 - }, - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 56 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "majicmix2.jpg", - "image" - ] - }, - { - "id": 39, - "type": "CropPortrait", - "pos": { - "0": 516, - "1": 537 - }, - "size": { - "0": 241.79998779296875, - "1": 46 - }, - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 56 - }, - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 57 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 99 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CropPortrait" - }, - "widgets_values": [] - }, - { - "id": 43, - "type": "GetFaceLandmarks", - "pos": { - "0": 465, - "1": 771 - }, - "size": { - "0": 292.20001220703125, - "1": 46 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "face_toolkits", - "type": "FACE_TOOLKITS", - "link": 62 - }, - { - "name": "images", - "type": "IMAGE", - "link": 82 - } - ], - "outputs": [ - { - "name": "landmarks", - "type": "FACELANDMARKS222", - "links": [ - 64, - 107 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "GetFaceLandmarks" - }, - "widgets_values": [] - }, - { - "id": 48, - "type": "VHS_LoadVideo", - "pos": { - "0": 75, - "1": 900 - }, - "size": { - "0": 247.455078125, - "1": 262 - }, - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null, - "shape": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": null, - "shape": 7 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 82, - 83, - 108 - ], - "slot_index": 0 - }, - { - "name": "frame_count", - "type": "INT", - "links": null - }, - { - "name": "audio", - "type": "AUDIO", - "links": [ - 92 - ], - "slot_index": 2 - }, - { - "name": "video_info", - "type": "VHS_VIDEOINFO", - "links": null - } - ], - "properties": { - "Node name for S&R": "VHS_LoadVideo" - }, - "widgets_values": { - "video": "jgz.mp4", - "force_rate": 15, - "force_size": "Disabled", - "custom_width": 512, - "custom_height": 512, - "frame_load_cap": 0, - "skip_first_frames": 0, - "select_every_nth": 1, - "choose video to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "force_rate": 15, - "frame_load_cap": 0, - "skip_first_frames": 0, - "select_every_nth": 1, - "filename": "jgz.mp4", - "type": "input", - "format": "video/mp4" - }, - "muted": false - } - } - }, - { - "id": 52, + "id": 59, "type": "VHS_VideoCombine", "pos": { - "0": 1658, - "1": 894 + "0": 1619, + "1": 800 }, "size": [ - 289.33306884765625, - 593.3330688476562 + 214.7587890625, + 518.7587890625 ], "flags": {}, "order": 9, @@ -382,12 +217,12 @@ { "name": "images", "type": "IMAGE", - "link": 96 + "link": 117 }, { "name": "audio", "type": "AUDIO", - "link": 92, + "link": 118, "shape": 7 }, { @@ -427,10 +262,10 @@ "hidden": false, "paused": false, "params": { - "filename": "AnimateDiff_00023.gif", + "filename": "AnimateDiff_00054-audio.mp4", "subfolder": "", "type": "output", - "format": "image/gif", + "format": "video/h264-mp4", "frame_rate": 15 }, "muted": false @@ -438,11 +273,212 @@ } }, { - "id": 55, + "id": 57, + "type": "HMVideoPipelineLoader", + "pos": { + "0": 813, + "1": 343 + }, + "size": { + "0": 352.79998779296875, + "1": 154 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "hm_video_pipeline", + "type": "HMVIDEOPIPELINE", + "links": [ + 115 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMVideoPipelineLoader" + }, + "widgets_values": [ + "SD1.5", + "None", + "same as checkpoint", + 12, + 0 + ] + }, + { + "id": 2, + "type": "HMFaceToolkitsLoader", + "pos": { + "0": 43, + "1": 699 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "links": [ + 57, + 62, + 65, + 97, + 119 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "HMFaceToolkitsLoader" + }, + "widgets_values": [ + 0 + ] + }, + { + "id": 43, + "type": "GetFaceLandmarks", + "pos": { + "0": 465, + "1": 771 + }, + "size": { + "0": 292.20001220703125, + "1": 46 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "face_toolkits", + "type": "FACE_TOOLKITS", + "link": 62 + }, + { + "name": "images", + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "landmarks", + "type": "FACELANDMARKS222", + "links": [ + 64, + 120 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "GetFaceLandmarks" + }, + "widgets_values": [] + }, + { + "id": 48, + "type": "VHS_LoadVideo", + "pos": { + "0": 106, + "1": 867 + }, + "size": [ + 247.455078125, + 503.455078125 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82, + 83, + 121 + ], + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 118 + ], + "slot_index": 2 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "jgz.mp4", + "force_rate": 15, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "force_rate": 15, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "filename": "jgz.mp4", + "type": "input", + "format": "video/mp4" + }, + "muted": false + } + } + }, + { + "id": 60, "type": "GetDriveExpression", "pos": { - "0": 857, - "1": 815 + "0": 845, + "1": 813 }, "size": { "0": 292.20001220703125, @@ -455,17 +491,17 @@ { "name": "face_toolkits", "type": "FACE_TOOLKITS", - "link": 109 + "link": 119 }, { "name": "images", "type": "IMAGE", - "link": 108 + "link": 121 }, { "name": "landmarks", "type": "FACELANDMARKS222", - "link": 107 + "link": 120 } ], "outputs": [ @@ -473,61 +509,17 @@ "name": "drive_exp", "type": "DRIVE_EXPRESSION", "links": [ - 110 + 122 ], "slot_index": 0 } ], "properties": { "Node name for S&R": "GetDriveExpression" - }, - "widgets_values": [] - }, - { - "id": 49, - "type": "HMVideoPipelineLoader", - "pos": { - "0": 849, - "1": 375 - }, - "size": { - "0": 352.79998779296875, - "1": 130 - }, - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "hm_video_pipeline", - "type": "HMVIDEOPIPELINE", - "links": [ - 98 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "HMVideoPipelineLoader" - }, - "widgets_values": [ - "SD1.5", - "None", - 12, - 0 - ] + } } ], "links": [ - [ - 56, - 8, - 0, - 39, - 0, - "IMAGE" - ], [ 57, 2, @@ -576,22 +568,6 @@ 1, "IMAGE" ], - [ - 92, - 48, - 2, - 52, - 1, - "AUDIO" - ], - [ - 96, - 53, - 0, - 52, - 0, - "IMAGE" - ], [ 97, 2, @@ -600,14 +576,6 @@ 1, "FACE_TOOLKITS" ], - [ - 98, - 49, - 0, - 53, - 0, - "HMVIDEOPIPELINE" - ], [ 99, 39, @@ -625,32 +593,64 @@ "DRIVE_POSE" ], [ - 107, - 43, + 115, + 57, 0, - 55, - 2, - "FACELANDMARKS222" + 53, + 0, + "HMVIDEOPIPELINE" ], [ - 108, - 48, + 116, + 58, + 0, + 39, 0, - 55, - 1, "IMAGE" ], [ - 109, + 117, + 53, + 0, + 59, + 0, + "IMAGE" + ], + [ + 118, + 48, + 2, + 59, + 1, + "AUDIO" + ], + [ + 119, 2, 0, - 55, + 60, 0, "FACE_TOOLKITS" ], [ - 110, - 55, + 120, + 43, + 0, + 60, + 2, + "FACELANDMARKS222" + ], + [ + 121, + 48, + 0, + 60, + 1, + "IMAGE" + ], + [ + 122, + 60, 0, 53, 4, @@ -661,10 +661,10 @@ "config": {}, "extra": { "ds": { - "scale": 0.7513148009015777, + "scale": 0.8264462809917359, "offset": [ - 760.8618864750447, - 107.45573369730707 + 159.4635814609923, + -64.01844135245447 ] } },