diff --git a/README.md b/README.md index 21ab920..e1d3173 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ ## πŸ”† New Features/Updates +- βœ… `02/09/2025` **HelloMemeV3** is now available. - βœ… `12/17/2024` Support modelscope ([Modelscope Demo](https://www.modelscope.cn/studios/songkey/HelloMeme)). - βœ… `12/08/2024` Added **HelloMemeV2** (select "v2" in the version option of the LoadHelloMemeImage/Video Node). Its features include: a. Improved expression consistency between the generated video and the driving video. @@ -54,23 +55,19 @@ This repository is the official implementation of the [`HelloMeme`](https://arxi > **Loras** under: `ComfyUI/models/checkpoints` ## Recommended Third-party Checkpoints/Loras -| Name | Checkpoints/Loras | Recommenders | -|------|-------------------|--------------| -| [realisticVisionV60B1_v51VAE](https://civitai.com/models/4201/realistic-vision-v60-b1) | Checkpoints | | -| [disneyPixarCartoon_v10](https://civitai.com/models/65203/disney-pixar-cartoon-type-a) | Checkpoints | | +| Name | Checkpoints/Loras | +|------|-------------------| +| [realisticVisionV60B1_v51VAE](https://civitai.com/models/4201/realistic-vision-v60-b1) | Checkpoints | +| [disneyPixarCartoon_v10](https://civitai.com/models/65203/disney-pixar-cartoon-type-a) | Checkpoints | ### Workflows -| workflow file | Video Generation | Image Generation | HMControlNet | HMControlNet2 | Super-Resolution | -|---------------|------------------|------------------|-----------|---------------|-----------------| -| hellomeme_image_workflow.json | | βœ… | βœ… | | | -| hellomeme_video_workflow.json | βœ… | | βœ… | | | -| hellomeme_image_sr_workflow.json | | βœ… | βœ… | | βœ… | -| hellomeme_video_sr_workflow.json | βœ… | | βœ… | | βœ… | -| hellomeme_image_v2_workflow.json | | βœ… | | βœ… | | -| hellomeme_video_v2_workflow.json | βœ… | | | βœ… | | -| hellomeme_image_v2_sr_workflow.json | | βœ… | | βœ… | βœ… | -| hellomeme_video_v2_sr_workflow.json | βœ… | | | βœ… | βœ… | +| workflow file | Video Generation | Image Generation | HMControlNet | HMControlNet2 | +|---------------|------------------|------------------|-----------|---------------| +| hellomeme_image_workflow.json | | βœ… | βœ… | | +| hellomeme_video_workflow.json | βœ… | | βœ… | | +| hellomeme_image_v2_workflow.json | | βœ… | | βœ… | +| hellomeme_video_v2_workflow.json | βœ… | | | βœ… | ### Image Generation Interface diff --git a/examples/tiktok.mp4 b/examples/tiktok.mp4 new file mode 100644 index 0000000..0fbef95 Binary files /dev/null and b/examples/tiktok.mp4 differ diff --git a/hellomeme/models/hm3_denoising_3d.py b/hellomeme/models/hm3_denoising_3d.py new file mode 100644 index 0000000..e5120d5 --- /dev/null +++ b/hellomeme/models/hm3_denoising_3d.py @@ -0,0 +1,295 @@ +# coding: utf-8 + +""" +@File : models6/hm_denoising_3d.py +@Author : Songkey +@Email : songkey@pku.edu.cn +@Date : 1/3/2025 +@Desc : +adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unets/unet_2d_condition.py +""" + +import torch +import torch.utils.checkpoint +from typing import Any, Dict, Optional, Tuple, Union + +from einops import rearrange + +from diffusers.utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel, UNet2DConditionOutput +from .hm_blocks import CopyWeights, InsertReferenceAdapter + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +class HM3Denoising3D(UNet2DConditionModel, CopyWeights, InsertReferenceAdapter): + def forward( + self, + sample: torch.Tensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + reference_hidden_states: Optional[dict] = None, + control_hidden_states: Optional[dict] = None, + motion_pad_hidden_states: Optional[dict] = None, + use_motion: bool = False, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet2DConditionOutput, Tuple]: + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + for dim in sample.shape[-2:]: + if dim % default_overall_up_factor != 0: + # Forward upsample size to force interpolation output size. + forward_upsample_size = True + break + + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + if attention_mask is not None: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None: + encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + # 0. center input if necessary + if self.config.center_input_sample: + sample = 2 * sample - 1.0 + + # 1. time + t_emb = self.get_time_embed(sample=sample, timestep=timestep) + emb = self.time_embedding(t_emb, timestep_cond) + + class_emb = self.get_class_embed(sample=sample, class_labels=class_labels) + if class_emb is not None: + if self.config.class_embeddings_concat: + emb = torch.cat([emb, class_emb], dim=-1) + else: + emb = emb + class_emb + + aug_emb = self.get_aug_embed( + emb=emb, encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs + ) + if self.config.addition_embed_type == "image_hint": + aug_emb, hint = aug_emb + sample = torch.cat([sample, hint], dim=1) + + emb = emb + aug_emb if aug_emb is not None else emb + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + num_frames = sample.shape[2] + emb = emb.repeat_interleave(repeats=num_frames, dim=0) + + if not added_cond_kwargs is None: + if 'image_embeds' in added_cond_kwargs: + if isinstance(added_cond_kwargs['image_embeds'], torch.Tensor): + added_cond_kwargs['image_embeds'] = added_cond_kwargs['image_embeds'].repeat_interleave(repeats=num_frames, dim=0) + else: + added_cond_kwargs['image_embeds'] = [x.repeat_interleave(repeats=num_frames, dim=0) for x in added_cond_kwargs['image_embeds']] + + if len(encoder_hidden_states.shape) == 3: + encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0) + elif len(encoder_hidden_states.shape) == 4: + encoder_hidden_states = rearrange(encoder_hidden_states, "b f l d -> (b f) l d") + + encoder_hidden_states = self.process_encoder_hidden_states( + encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs + ) + + # 2. pre-process + sample = rearrange(sample, "b c f h w -> (b f) c h w") + sample = self.conv_in(sample) + + # 2.5 GLIGEN position net + if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None: + cross_attention_kwargs = cross_attention_kwargs.copy() + gligen_args = cross_attention_kwargs.pop("gligen") + cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)} + + # 3. down + # we're popping the `scale` instead of getting it because otherwise `scale` will be propagated + # to the internal blocks and will raise deprecation warnings. this will be confusing for our users. + if cross_attention_kwargs is not None: + cross_attention_kwargs = cross_attention_kwargs.copy() + lora_scale = cross_attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + + is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None + # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets + is_adapter = down_intrablock_additional_residuals is not None + # maintain backward compatibility for legacy usage, where + # T2I-Adapter and ControlNet both use down_block_additional_residuals arg + # but can only use one or the other + if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None: + deprecate( + "T2I should not use down_block_additional_residuals", + "1.3.0", + "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \ + and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \ + for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", + standard_warn=False, + ) + down_intrablock_additional_residuals = down_block_additional_residuals + is_adapter = True + + res_cache = dict() + down_block_res_samples = (sample,) + for idx, downsample_block in enumerate(self.down_blocks): + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + # For t2i-adapter CrossAttnDownBlock2D + additional_residuals = {} + if is_adapter and len(down_intrablock_additional_residuals) > 0: + additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0) + + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + **additional_residuals, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb) + if is_adapter and len(down_intrablock_additional_residuals) > 0: + sample += down_intrablock_additional_residuals.pop(0) + + res_cache[f"down_{idx}"] = sample.clone() + if not control_hidden_states is None and f'down3_v1_{idx}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'down3_v1_{idx}'], "b c f h w -> (b f) c h w") + if not control_hidden_states is None and f'down3_v2_{idx}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'down3_v2_{idx}'], "b c f h w -> (b f) c h w") + if hasattr(self, 'motion_down') and use_motion: + sample = self.motion_down[idx](sample, motion_pad_hidden_states[f'down_{idx}'], emb, num_frames) + + down_block_res_samples += res_samples + + if is_controlnet: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples = new_down_block_res_samples + (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = self.mid_block(sample, emb) + + # To support T2I-Adapter-XL + if ( + is_adapter + and len(down_intrablock_additional_residuals) > 0 + and sample.shape == down_intrablock_additional_residuals[0].shape + ): + sample += down_intrablock_additional_residuals.pop(0) + + if is_controlnet: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + res_cache[f"up_{i}"] = sample.clone() + if not control_hidden_states is None and f'up3_v1_{i}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'up3_v1_{i}'], "b c f h w -> (b f) c h w") + if not control_hidden_states is None and f'up3_v2_{i}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'up3_v2_{i}'], "b c f h w -> (b f) c h w") + if hasattr(self, "reference_modules_up") and not reference_hidden_states is None and f'up_{i}' in reference_hidden_states: + sample = self.reference_modules_up[i](sample, reference_hidden_states[f'up_{i}'], num_frames) + if hasattr(self, 'motion_up') and use_motion: + sample = self.motion_up[i](sample, motion_pad_hidden_states[f'up_{i}'], emb, num_frames) + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + upsample_size=upsample_size, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + # reshape to (batch, channel, framerate, width, height) + sample = rearrange(sample, "(b f) c h w -> b c f h w", f=num_frames) + + if not return_dict: + return (sample, res_cache) + + return (UNet2DConditionOutput(sample=sample), res_cache) diff --git a/hellomeme/models/hm3_denoising_motion.py b/hellomeme/models/hm3_denoising_motion.py new file mode 100644 index 0000000..90f099f --- /dev/null +++ b/hellomeme/models/hm3_denoising_motion.py @@ -0,0 +1,245 @@ +# coding: utf-8 + +""" +@File : models6/hm_denoising_motion.py +@Author : Songkey +@Email : songkey@pku.edu.cn +@Date : 1/3/2025 +@Desc : +adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/unets/unet_motion_model.py +""" + +import torch +import torch.utils.checkpoint +from typing import Any, Dict, Optional, Tuple, Union + +from einops import rearrange + +from diffusers.utils import logging +from diffusers.models.unets.unet_motion_model import UNetMotionModel, UNetMotionOutput +from .hm_blocks import InsertReferenceAdapter + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class HM3DenoisingMotion(UNetMotionModel, InsertReferenceAdapter): + def forward( + self, + sample: torch.Tensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + reference_hidden_states: Optional[dict] = None, + control_hidden_states: Optional[torch.Tensor] = None, + motion_pad_hidden_states: Optional[dict] = None, + use_motion: bool = False, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNetMotionOutput, Tuple[torch.Tensor]]: + + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layears). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2 ** self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + logger.info("Forward upsample size to force interpolation output size.") + forward_upsample_size = True + + # prepare attention_mask + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + num_frames = sample.shape[2] + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=self.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + if self.config.addition_embed_type == "text_time": + if "text_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + + text_embeds = added_cond_kwargs.get("text_embeds") + if "time_ids" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get("time_ids") + time_embeds = self.add_time_proj(time_ids.flatten()) + time_embeds = time_embeds.reshape((text_embeds.shape[0], -1)) + + add_embeds = torch.concat([text_embeds, time_embeds], dim=-1) + add_embeds = add_embeds.to(emb.dtype) + aug_emb = self.add_embedding(add_embeds) + + emb = emb if aug_emb is None else emb + aug_emb + emb = emb.repeat_interleave(repeats=num_frames, dim=0) + + if len(encoder_hidden_states.shape) == 3: + encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0) + elif len(encoder_hidden_states.shape) == 4: + encoder_hidden_states = rearrange(encoder_hidden_states, "b f l d -> (b f) l d") + + if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj": + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + image_embeds = self.encoder_hid_proj(image_embeds) + image_embeds = [image_embed.repeat_interleave(repeats=num_frames, dim=0) for image_embed in image_embeds] + encoder_hidden_states = (encoder_hidden_states, image_embeds) + + # 2. pre-process + sample = sample.permute(0, 2, 1, 3, 4).reshape((sample.shape[0] * num_frames, -1) + sample.shape[3:]) + sample = self.conv_in(sample) + + # 3. down + down_block_res_samples = (sample,) + for idx, downsample_block in enumerate(self.down_blocks): + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb, num_frames=num_frames) + + if not control_hidden_states is None and f'down3_v1_{idx}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'down3_v1_{idx}'], "b c f h w -> (b f) c h w") + if not control_hidden_states is None and f'down3_v2_{idx}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'down3_v2_{idx}'], "b c f h w -> (b f) c h w") + if hasattr(self, 'motion_down') and use_motion: + sample = self.motion_down[idx](sample, motion_pad_hidden_states[f'down_{idx}'], emb, num_frames) + + down_block_res_samples += res_samples + + if down_block_additional_residuals is not None: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples += (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + # To support older versions of motion modules that don't have a mid_block + if hasattr(self.mid_block, "motion_modules"): + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if mid_block_additional_residual is not None: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets):] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if not control_hidden_states is None and f'up3_v1_{i}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'up3_v1_{i}'], "b c f h w -> (b f) c h w") + if not control_hidden_states is None and f'up3_v2_{i}' in control_hidden_states: + sample += rearrange(control_hidden_states[f'up3_v2_{i}'], "b c f h w -> (b f) c h w") + if hasattr(self, "reference_modules_up") and not reference_hidden_states is None and f'up_{i}' in reference_hidden_states: + sample = self.reference_modules_up[i](sample, reference_hidden_states[f'up_{i}'], num_frames) + if hasattr(self, 'motion_up') and use_motion: + sample = self.motion_up[i](sample, motion_pad_hidden_states[f'up_{i}'], emb, num_frames) + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + upsample_size=upsample_size, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + num_frames=num_frames, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + + sample = self.conv_out(sample) + + # reshape to (batch, channel, framerate, width, height) + sample = sample[None, :].reshape((-1, num_frames) + sample.shape[1:]).permute(0, 2, 1, 3, 4) + + if not return_dict: + return (sample,) + + return UNetMotionOutput(sample=sample) \ No newline at end of file diff --git a/hellomeme/pipelines/pipline_hm3_image.py b/hellomeme/pipelines/pipline_hm3_image.py new file mode 100644 index 0000000..d7efa08 --- /dev/null +++ b/hellomeme/pipelines/pipline_hm3_image.py @@ -0,0 +1,374 @@ +# coding: utf-8 + +""" +@File : hm_pipline_image.py +@Author : Songkey +@Email : songkey@pku.edu.cn +@Date : 1/3/2025 +@Desc : +adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +""" + +import copy +from typing import Any, Callable, Dict, List, Optional, Union +import torch + +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import PipelineImageInput +from diffusers.utils import deprecate +from diffusers.utils.torch_utils import randn_tensor +from diffusers.pipelines.stable_diffusion.pipeline_output import StableDiffusionPipelineOutput +from diffusers import DPMSolverMultistepScheduler +from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents + +from ..models import HM3Denoising3D, HMV3ControlNet, HMV3ControlNet2, HMPipeline, HM3ReferenceAdapter + + +class HM3ImagePipeline(HMPipeline): + def caryomitosis(self, **kwargs): + if hasattr(self, "unet_ref"): + del self.unet_ref + self.unet_ref = HM3Denoising3D.from_unet2d(self.unet) + self.unet_ref.cpu() + + if not isinstance(self.unet, HM3Denoising3D): + unet = HM3Denoising3D.from_unet2d(unet=self.unet) + # todo: δΈε€ŸδΌ˜ι›… + del self.unet + self.unet = unet + self.unet.cpu() + + self.vae.cpu() + self.vae_decode = copy.deepcopy(self.vae) + self.text_encoder.cpu() + self.text_encoder_ref = copy.deepcopy(self.text_encoder) + + def insert_hm_modules(self, version='v3', dtype=torch.float16, modelscope=False): + + if modelscope: + from modelscope import snapshot_download + hm_reference_dir = snapshot_download('songkey/hm3_reference') + hm_control_dir = snapshot_download('songkey/hm3_control') + hm_control2_dir = snapshot_download('songkey/hm3_control2') + else: + hm_reference_dir = 'songkey/hm3_reference' + hm_control_dir = 'songkey/hm3_control' + hm_control2_dir = 'songkey/hm3_control2' + + if isinstance(self.unet, HM3Denoising3D): + hm_adapter = HM3ReferenceAdapter.from_pretrained(hm_reference_dir) + self.unet.insert_reference_adapter(hm_adapter) + self.unet.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "unet_ref"): + self.unet_ref.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "mp_control"): + del self.mp_control + + self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir) + self.mp_control.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "mp_control2"): + del self.mp_control2 + self.mp_control2 = HMV3ControlNet2.from_pretrained(hm_control2_dir) + self.mp_control2.to(device='cpu', dtype=dtype).eval() + + self.vae.to(device='cpu', dtype=dtype).eval() + self.vae_decode.to(device='cpu', dtype=dtype).eval() + self.text_encoder.to(device='cpu', dtype=dtype).eval() + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + drive_params: Dict[str, Any] = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + timesteps: List[int] = None, + sigmas: List[float] = None, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + output_type: Optional[str] = "pil", + device: Optional[str] = "cpu", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: int = None, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + num_images_per_prompt = 1 + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + strength, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # device = self.device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + self.text_encoder_ref.to(device=device) + prompt_embeds_ref, negative_prompt_embeds_ref = self.encode_prompt_sk( + self.text_encoder_ref, + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + self.text_encoder_ref.cpu() + + self.text_encoder.to(device=device) + prompt_embeds, negative_prompt_embeds = self.encode_prompt_sk( + self.text_encoder, + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + self.text_encoder.cpu() + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + prompt_embeds_ref = torch.cat([negative_prompt_embeds_ref, prompt_embeds_ref]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Preprocess + image = self.image_processor.preprocess(image).to(device=device, dtype=prompt_embeds.dtype) + + scheduler = DPMSolverMultistepScheduler( + num_train_timesteps=1000, + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + # use_karras_sigmas=True, + algorithm_type="sde-dpmsolver++", + ) + + # 5. set timesteps + timesteps, num_inference_steps = retrieve_timesteps(scheduler, num_inference_steps, device, timesteps, sigmas) + + # 6. Prepare reference latents + self.vae.to(device=device) + ref_latents = [ + retrieve_latents(self.vae.encode(image[i: i + 1].to(device=device)), generator=generator) + for i in range(batch_size) + ] + self.vae.cpu() + + ref_latents = torch.cat(ref_latents, dim=0) + ref_latents = self.vae.config.scaling_factor * ref_latents + c, h, w = ref_latents.shape[1:] + + condition = drive_params['condition'].clone().to(device=device) + if self.do_classifier_free_guidance: + condition = torch.cat([torch.ones_like(condition) * -1, condition], dim=0) + + control_latents = {} + if 'drive_coeff' in drive_params: + self.mp_control.to(device=device) + drive_coeff = drive_params['drive_coeff'].clone().to(device=device) + face_parts = drive_params['face_parts'].clone().to(device=device) + if self.do_classifier_free_guidance: + drive_coeff = torch.cat([torch.zeros_like(drive_coeff), drive_coeff], dim=0) + face_parts = torch.cat([torch.zeros_like(face_parts), face_parts], dim=0) + control_latents1 = self.mp_control(condition=condition, drive_coeff=drive_coeff, face_parts=face_parts) + control_latents.update(control_latents1) + self.mp_control.cpu() + + # debug_dict = torch.load(r'C:\Users\songkey-win\Downloads\debug\debug_dict.pth') + # # latents_cntrl = debug_dict['latents_cntrl'] + # ref_latents = debug_dict['ref_latents'] + # ref_latents_neg = debug_dict['ref_latents_neg'] + # # cached_res = debug_dict['cached_res'] + # tgt_skl_tensor = debug_dict['tgt_skl_tensor'] + # emo_embedding = debug_dict['emo_embedding'] + + if 'pd_fpg' in drive_params: + self.mp_control2.to(device=device) + pd_fpg = drive_params['pd_fpg'].clone().to(device=device) + # pd_fpg = emo_embedding[:,:1] + # condition = tgt_skl_tensor[:,:,:1] + if self.do_classifier_free_guidance: + pd_fpg = torch.cat([torch.zeros_like(pd_fpg), pd_fpg], dim=0) + control_latents2 = self.mp_control2(condition=condition, emo_embedding=pd_fpg) + control_latents.update(control_latents2) + self.mp_control2.cpu() + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + base_noise = randn_tensor([batch_size, c, h, w], dtype=prompt_embeds.dtype, generator=generator).to(device=device) + + latent_model_input = torch.cat([torch.zeros_like(ref_latents), ref_latents]) if ( + self.do_classifier_free_guidance) else ref_latents + # latent_model_input = torch.cat([ref_latents_neg, ref_latents], dim=0) + self.unet_ref.to(device=device) + cached_res = self.unet_ref( + latent_model_input.unsqueeze(2), + 0, + encoder_hidden_states=prompt_embeds_ref, + return_dict=False, + )[1] + self.unet_ref.cpu() + + # 7.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=prompt_embeds.dtype) + + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + # base_noise = randn_tensor([batch_size, c, h, w], dtype=prompt_embeds.dtype, generator=generator).to(device=device) + latents = base_noise * scheduler.init_noise_sigma + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * scheduler.order + self._num_timesteps = len(timesteps) + self.unet.to(device=device) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input.unsqueeze(2), + t, + encoder_hidden_states=prompt_embeds, + reference_hidden_states=cached_res, + control_hidden_states=control_latents, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0][:,:,0,:,:] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(scheduler, "order", 1) + callback(step_idx, t, latents) + + self.unet.cpu() + + self.vae_decode.to(device=device) + if not output_type == "latent": + image = self.vae_decode.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + else: + image = latents + self.vae_decode.cpu() + + do_denormalize = [True] * image.shape[0] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=None), latents.detach().cpu() / self.vae.config.scaling_factor diff --git a/hellomeme/pipelines/pipline_hm3_video.py b/hellomeme/pipelines/pipline_hm3_video.py new file mode 100644 index 0000000..5100ff5 --- /dev/null +++ b/hellomeme/pipelines/pipline_hm3_video.py @@ -0,0 +1,573 @@ +# coding: utf-8 + +""" +@File : hm_pipline_image.py +@Author : Songkey +@Email : songkey@pku.edu.cn +@Date : 1/3/2025 +@Desc : +adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +""" + +import copy +from typing import Any, Callable, Dict, List, Optional, Union + +import cv2 +import torch +from einops import rearrange + +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import PipelineImageInput +from diffusers.utils import deprecate + +from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents +from diffusers import MotionAdapter, DPMSolverMultistepScheduler +from diffusers.utils.torch_utils import randn_tensor + +from ..models import (HM3Denoising3D, HM3DenoisingMotion, HMV3ControlNet2, HM3MotionAdapter, + HMV3ControlNet, HM3ReferenceAdapter, HMPipeline) +from ..utils import np_rgb_to_tensor, tensor_to_np_rgb + +class HM3VideoPipeline(HMPipeline): + def caryomitosis(self, version='v3', modelscope=False, **kwargs): + if hasattr(self, "unet_ref"): + del self.unet_ref + self.unet_ref = HM3Denoising3D.from_unet2d(self.unet) + self.unet_ref.cpu() + + if hasattr(self, "unet_pre"): + del self.unet_pre + self.unet_pre = HM3Denoising3D.from_unet2d(self.unet) + self.unet_pre.cpu() + + if modelscope: + from modelscope import snapshot_download + hm_animatediff_dir = snapshot_download('songkey/hm3_animatediff') + else: + hm_animatediff_dir = 'songkey/hm3_animatediff' + + self.num_frames = 12 + adapter = MotionAdapter.from_pretrained(hm_animatediff_dir, torch_dtype=torch.float16) + + unet = HM3DenoisingMotion.from_unet2d(unet=self.unet, motion_adapter=adapter, load_weights=True) + del self.unet + self.unet = unet + + self.vae.cpu() + self.vae_decode = copy.deepcopy(self.vae) + self.text_encoder.cpu() + self.text_encoder_ref = copy.deepcopy(self.text_encoder) + + def insert_hm_modules(self, version='v3', dtype=torch.float16, modelscope=False): + if modelscope: + from modelscope import snapshot_download + hm_reference_dir = snapshot_download('songkey/hm3_reference') + hm_control_dir = snapshot_download('songkey/hm3_control') + hm_control2_dir = snapshot_download('songkey/hm3_control2') + hm_motion_dir = snapshot_download('songkey/hm3_motion') + else: + hm_reference_dir = 'songkey/hm3_reference' + hm_control_dir = 'songkey/hm3_control' + hm_control2_dir = 'songkey/hm3_control2' + hm_motion_dir = 'songkey/hm3_motion' + + hm_adapter = HM3ReferenceAdapter.from_pretrained(hm_reference_dir) + motion_adapter = HM3MotionAdapter.from_pretrained(hm_motion_dir) + + if isinstance(self.unet, HM3DenoisingMotion): + self.unet.insert_reference_adapter(hm_adapter) + self.unet.insert_reference_adapter(motion_adapter) + self.unet.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "unet_pre"): + self.unet_pre.insert_reference_adapter(hm_adapter) + self.unet_pre.insert_reference_adapter(motion_adapter) + self.unet_pre.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "unet_ref"): + self.unet_ref.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "mp_control"): + del self.mp_control + + self.mp_control = HMV3ControlNet.from_pretrained(hm_control_dir) + self.mp_control.to(device='cpu', dtype=dtype).eval() + + if hasattr(self, "mp_control2"): + del self.mp_control2 + + self.mp_control2 = HMV3ControlNet2.from_pretrained(hm_control2_dir) + self.mp_control2.to(device='cpu', dtype=dtype).eval() + + self.vae.to(device='cpu', dtype=dtype).eval() + self.vae_decode.to(device='cpu', dtype=dtype).eval() + self.text_encoder.to(device='cpu', dtype=dtype).eval() + + @torch.no_grad() + def gen_ref_cache(self, latent, prompt_embeds, added_cond_kwargs, device, add_axis=False): + self.unet_ref.to(device=device) + latent = latent.to(device=device, dtype=prompt_embeds.dtype) + latent_model_input = torch.cat([torch.zeros_like(latent), latent]) if self.do_classifier_free_guidance else latent + cached_res = self.unet_ref( + latent_model_input.unsqueeze(2), + 0, + encoder_hidden_states=prompt_embeds, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[1] + self.unet_ref.cpu() + + if add_axis: + for k, v in cached_res.items(): + cached_res[k] = rearrange(v, 'b c h w -> b c 1 h w') + + return dicts_to_device([cached_res], device='cpu')[0] + + @torch.no_grad() + def gen_video(self, indexes, pad_dict, base_noise, depth, control_dict, step, more_params): + device = more_params['device'] + timesteps = more_params['timesteps'] + timestep_cond = more_params['timestep_cond'] + sigmas = more_params['sigmas'] + prompt_embeds = more_params['prompt_embeds'] + cached_res = more_params['cached_res'] + added_cond_kwargs = more_params['added_cond_kwargs'] + extra_step_kwargs = more_params['extra_step_kwargs'] + + num_indexes = len(indexes) + if num_indexes <= self.num_frames: + paded_indexes = indexes + + paded_indexes_noise = list(range(len(indexes))) + + paded_indexes = copy.deepcopy(paded_indexes) + margin_indexes = [indexes[0], indexes[-1]] + + scheduler = DPMSolverMultistepScheduler( + num_train_timesteps=1000, + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + algorithm_type="sde-dpmsolver++", + ) + + tmp_timesteps, _ = retrieve_timesteps(scheduler, step, device, timesteps, sigmas) + + margin_latent = base_noise[:,:,[0, 0]] * scheduler.init_noise_sigma + margin_latent = margin_latent.to(device=device, dtype=prompt_embeds.dtype) + control_latent = cat_dicts([control_dict[x] for x in margin_indexes], dim=2) + + self.unet_pre.to(device=device) + + with self.progress_bar(total=len(tmp_timesteps)) as progress_bar: + for i, t in enumerate(tmp_timesteps): + progress_bar.set_description(f"D{depth} GEN Margin") + progress_bar.update() + latent_model_input = torch.cat([margin_latent] * 2, dim=0) if \ + self.do_classifier_free_guidance else margin_latent + latent_model_input = scheduler.scale_model_input(latent_model_input, t) + latent_model_input = latent_model_input.to(device=device, dtype=prompt_embeds.dtype) + noise_pred = self.unet_pre( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + reference_hidden_states=dicts_to_device([cached_res], device=device)[0], + control_hidden_states=dicts_to_device([control_latent], device=device)[0], + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + margin_latent = scheduler.step(noise_pred, t, margin_latent, + **extra_step_kwargs, return_dict=False)[0] + + self.unet_pre.cpu() + + if not margin_indexes[0] in pad_dict: + pad_dict[margin_indexes[0]] = self.gen_ref_cache(margin_latent[:,:,0], + prompt_embeds, added_cond_kwargs, device, add_axis=True) + if not margin_indexes[1] in pad_dict: + pad_dict[margin_indexes[1]] = self.gen_ref_cache(margin_latent[:,:,-1], + prompt_embeds, added_cond_kwargs, device, add_axis=True) + + drive_idx_chunks = [paded_indexes] + drive_idx_chunks_noise = [paded_indexes_noise] + else: + stride = self.num_frames - 1 + paded_indexes = indexes + paded_indexes = copy.deepcopy(paded_indexes) + + num_paded_indexes = len(paded_indexes) + paded_indexes_noise = list(range(num_paded_indexes)) + + drive_idx_chunks = [paded_indexes[i:min(i + self.num_frames, num_paded_indexes)] for i in + range(0, num_paded_indexes, stride)] + drive_idx_chunks_noise = [paded_indexes_noise[i:min(i + self.num_frames, num_paded_indexes)] for i in + range(0, num_paded_indexes, stride)] + + if len(drive_idx_chunks[-1]) == 1: + drive_idx_chunks[-1] = drive_idx_chunks[:-1] + drive_idx_chunks_noise[-1] = drive_idx_chunks_noise[:-1] + + margin_indexes = set() + for chunk in drive_idx_chunks: + margin_indexes.add(chunk[0]) + margin_indexes.add(chunk[-1]) + margin_indexes = sorted(list(margin_indexes)) + + self.gen_video(margin_indexes, pad_dict, base_noise, depth+1, control_dict, step, more_params) + + ## generate latent + scheduler = DPMSolverMultistepScheduler( + num_train_timesteps=1000, + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + algorithm_type="sde-dpmsolver++", + ) + timesteps, _ = retrieve_timesteps(scheduler, step, device, timesteps, sigmas) + + N, C, _, H, W = base_noise.shape + + latents = base_noise[:,:,:len(paded_indexes)] * scheduler.init_noise_sigma + + self.unet.to(device=device) + with self.progress_bar(total=len(timesteps)) as progress_bar: + for i, t in enumerate(timesteps): + progress_bar.set_description(f"D{depth} GEN Frames") + progress_bar.update() + + noise_pred = torch.zeros_like(latents) + noise_pred = torch.cat([noise_pred, noise_pred], dim=0) if \ + self.do_classifier_free_guidance else noise_pred + noise_pred_counter = torch.zeros([1, 1, noise_pred.size(2), 1, 1], dtype=prompt_embeds.dtype).cpu() + + latent_model_input_all = torch.cat([latents] * 2, dim=0) if \ + self.do_classifier_free_guidance else latents + + latent_model_input_all = scheduler.scale_model_input(latent_model_input_all, t) + for chunk, chunk_noise in zip(drive_idx_chunks, drive_idx_chunks_noise): + if self.interrupt: + continue + + control_latent = cat_dicts([control_dict[ix] for ix in chunk], dim=2) + pad_latent = cat_dicts([pad_dict[chunk[0]], pad_dict[chunk[-1]]], dim=2) + + latent_model_input = latent_model_input_all[:, :, chunk_noise].to(device=device) + + noise_pred_chunk = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + reference_hidden_states=dicts_to_device([cached_res], device=device)[0], + control_hidden_states=dicts_to_device([control_latent], device=device)[0], + motion_pad_hidden_states=dicts_to_device([pad_latent], device=device)[0], + use_motion=True, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0].cpu() + + noise_pred[:, :, chunk_noise] = noise_pred[:, :, chunk_noise] + noise_pred_chunk + noise_pred_counter[:, :, chunk_noise] = noise_pred_counter[:, :, chunk_noise] + 1 + + noise_pred = noise_pred / noise_pred_counter + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + latents = scheduler.step(noise_pred.cpu(), t, latents, + **extra_step_kwargs, return_dict=False)[0] + self.unet.cpu() + + # images = self.vae_decode_latents(self.vae_decode, latents / self.vae_decode.config.scaling_factor, device) + # for idx, image in enumerate(images): + # image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + # cv2.imwrite(f"C:/Users/songkey-win/Downloads/debug0203/output_d{depth:04d}_{idx:04d}.png", image) + + + if depth == 0: + return latents[:,:,:num_indexes] + else: + for chunk, chunk_noise in zip(drive_idx_chunks, drive_idx_chunks_noise): + for idx, idx_noise in zip(chunk, chunk_noise): + if not idx in pad_dict: + pad_dict[idx] = self.gen_ref_cache(latents[:,:,idx_noise], prompt_embeds, + added_cond_kwargs, device, add_axis=True) + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + chunk_overlap: int = 4, + drive_params: Dict[str, Any] = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + timesteps: List[int] = None, + sigmas: List[float] = None, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + output_type: Optional[str] = "pil", + device: Optional[str] = "cpu", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: int = None, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + num_images_per_prompt = 1 + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + strength, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # device = self.device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + self.text_encoder_ref.to(device=device) + prompt_embeds_ref, negative_prompt_embeds_ref = self.encode_prompt_sk( + self.text_encoder_ref, + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + self.text_encoder_ref.cpu() + + self.text_encoder.to(device=device) + prompt_embeds, negative_prompt_embeds = self.encode_prompt_sk( + self.text_encoder, + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + self.text_encoder.cpu() + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + prompt_embeds_ref = torch.cat([negative_prompt_embeds_ref, prompt_embeds_ref]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Preprocess image + image = self.image_processor.preprocess(image).to(device=device, dtype=prompt_embeds.dtype) + + raw_video_len = drive_params['condition'].size(2) + + # 6. Prepare latent variables + self.vae.to(device=device) + ref_latents = [ + retrieve_latents(self.vae.encode(image[i: i + 1].to(device=device)), generator=generator) + for i in range(batch_size) + ] + self.vae.cpu() + + ref_latents = torch.cat(ref_latents, dim=0) + ref_latents = self.vae.config.scaling_factor * ref_latents + c, h, w = ref_latents.shape[1:] + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 7.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=prompt_embeds.dtype) + + input_noise = randn_tensor([batch_size, c, raw_video_len + self.num_frames + 1, h, w], + generator=generator, dtype=prompt_embeds.dtype) + + cached_res = self.gen_ref_cache(ref_latents, prompt_embeds_ref, added_cond_kwargs, device) + + control_latents = [] + control_latent1_neg, control_latent2_neg = None, None + self.mp_control.to(device=device) + self.mp_control2.to(device=device) + with self.progress_bar(total=raw_video_len) as progress_bar: + for idx in range(raw_video_len): + progress_bar.set_description(f"GEN stage1") + progress_bar.update() + + condition = drive_params['condition'][:, :, idx:idx+1].clone().to(device=device) + + control_latent_input = {} + if 'drive_coeff' in drive_params: + drive_coeff = drive_params['drive_coeff'][:, idx:idx+1].clone().to(device=device) + face_parts = drive_params['face_parts'][:, idx:idx+1].clone().to(device=device) + + if control_latent1_neg is None and self.do_classifier_free_guidance: + control_latent1_neg = self.mp_control(condition=torch.ones_like(condition)*-1, + drive_coeff=torch.zeros_like(drive_coeff), + face_parts=torch.zeros_like(face_parts)) + + control_latent1 = self.mp_control(condition=condition, + drive_coeff=drive_coeff, + face_parts=face_parts) + if self.do_classifier_free_guidance: + control_latent1 = cat_dicts([control_latent1_neg, control_latent1], dim=0) + + control_latent_input.update(control_latent1) + + if 'pd_fpg' in drive_params: + pd_fpg = drive_params['pd_fpg'][:, idx:idx+1].clone().to(device=device) + + if control_latent2_neg is None and self.do_classifier_free_guidance: + control_latent2_neg = self.mp_control2(condition=torch.ones_like(condition)*-1, + emo_embedding=torch.zeros_like(pd_fpg)) + + control_latent2 = self.mp_control2(condition=condition, emo_embedding=pd_fpg) + if self.do_classifier_free_guidance: + control_latent2 = cat_dicts([control_latent2_neg, control_latent2], dim=0) + + control_latent_input.update(control_latent2) + control_latents.append(dicts_to_device([control_latent_input], device='cpu')[0]) + + self.mp_control2.cpu() + self.mp_control.cpu() + + more_params = dict( + timesteps=timesteps, + timestep_cond=timestep_cond, + sigmas=sigmas, + prompt_embeds=prompt_embeds, + cached_res=cached_res, + device=device, + added_cond_kwargs=added_cond_kwargs, + extra_step_kwargs=extra_step_kwargs, + ) + + indexes = list(range(raw_video_len)) + latents = self.gen_video(indexes, dict(), input_noise, 0, control_latents, num_inference_steps, more_params) + latents_res = latents / self.vae_decode.config.scaling_factor + + self.vae_decode.to(device=device) + res_frames = [] + with self.progress_bar(total=raw_video_len) as progress_bar: + for i in range(raw_video_len): + ret_tensor = self.vae_decode.decode(latents_res[:, :, i, :, :].to(device=device), + return_dict=False, generator=generator)[0] + ret_image = self.image_processor.postprocess(ret_tensor, output_type=output_type) + res_frames.append(ret_image) + progress_bar.set_description(f"VAE DECODE") + progress_bar.update() + self.vae_decode.cpu() + return res_frames, latents_res + + +def merge_dicts(dictl, dictr, wl=0.5): + res = {} + for k in dictl.keys(): + res[k] = dictl[k] * wl + dictr[k] * (1-wl) + return res + + +def cat_dicts(dicts, dim=0): + res = {} + for k in dicts[0].keys(): + res[k] = torch.cat([d[k].clone() for d in dicts], dim=dim) + return res + + +def cat_dicts_ref(dicts): + res = {} + for k in dicts[0].keys(): + res[k] = rearrange(torch.cat([d[k].clone().unsqueeze(1) for d in dicts], dim=1), "b f c h w -> (b f) c h w ") + return res + + +def dicts_to_device(dicts, device): + ret = [] + for d in dicts: + tmpd = {} + for k in d.keys(): + tmpd[k] = d[k].clone().to(device) + ret.append(tmpd) + return ret \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f4af499..25c30f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_hellomeme" description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface" -version = "1.0.4" +version = "1.1.0" license = {file = "LICENSE"} [project.urls] diff --git a/workflows/hellomeme_image_example.png b/workflows/hellomeme_image_example.png index 7cf0c38..c916eea 100644 Binary files a/workflows/hellomeme_image_example.png and b/workflows/hellomeme_image_example.png differ diff --git a/workflows/hellomeme_image_sr_workflow.json b/workflows/hellomeme_image_sr_workflow.json deleted file mode 100644 index 5d41209..0000000 --- a/workflows/hellomeme_image_sr_workflow.json +++ /dev/null @@ -1 +0,0 @@ -{"last_node_id":65,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,127],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":6,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,126],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,125],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":9,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":124,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[131],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,1124606751085208,"randomize",2,0]},{"id":61,"type":"GetDriveExpression","pos":[857,823],"size":[292.20001220703125,66],"flags":{},"order":8,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":127},{"name":"images","type":"IMAGE","link":125},{"name":"landmarks","type":"FACELANDMARKS222","link":126}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[124],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":64,"type":"UpscaleModelLoader","pos":[1270,397],"size":[315,58],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"UPSCALE_MODEL","type":"UPSCALE_MODEL","links":[130],"slot_index":0}],"properties":{"Node name for S&R":"UpscaleModelLoader"},"widgets_values":["RealESRGAN_x2.pth"]},{"id":65,"type":"ImageUpscaleWithModel","pos":[1630,489],"size":[340.20001220703125,46],"flags":{},"order":10,"mode":0,"inputs":[{"name":"upscale_model","type":"UPSCALE_MODEL","link":130},{"name":"image","type":"IMAGE","link":131}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"ImageUpscaleWithModel"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[2006,588],"size":[328.01544189453125,354.2547302246094],"flags":{},"order":11,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":133}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[811.676025390625,342.02801513671875],"size":[352.79998779296875,202],"flags":{},"order":4,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v1","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[124,61,0,58,4,"DRIVE_EXPRESSION"],[125,57,0,61,1,"IMAGE"],[126,43,0,61,2,"FACELANDMARKS222"],[127,2,0,61,0,"FACE_TOOLKITS"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,64,0,65,0,"UPSCALE_MODEL"],[131,58,0,65,1,"IMAGE"],[133,65,0,59,0,"IMAGE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[803.279414217647,-32.35099556997285]}},"version":0.4} \ No newline at end of file diff --git a/workflows/hellomeme_image_v2_sr_workflow.json b/workflows/hellomeme_image_v2_sr_workflow.json deleted file mode 100644 index 4113309..0000000 --- a/workflows/hellomeme_image_v2_sr_workflow.json +++ /dev/null @@ -1 +0,0 @@ -{"last_node_id":66,"last_link_id":136,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":6,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":9,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":133,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[134],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,976916956647463,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":8,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":65,"type":"UpscaleModelLoader","pos":[1282,374],"size":[315,58],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"UPSCALE_MODEL","type":"UPSCALE_MODEL","links":[136],"slot_index":0}],"properties":{"Node name for S&R":"UpscaleModelLoader"},"widgets_values":["RealESRGAN_x2.pth"]},{"id":66,"type":"ImageUpscaleWithModel","pos":[1644,503],"size":[340.20001220703125,46],"flags":{},"order":10,"mode":0,"inputs":[{"name":"upscale_model","type":"UPSCALE_MODEL","link":136},{"name":"image","type":"IMAGE","link":134}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[135],"slot_index":0}],"properties":{"Node name for S&R":"ImageUpscaleWithModel"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":4,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v2","x1","huggingface",1]},{"id":59,"type":"PreviewImage","pos":[2040,610],"size":[414.0865478515625,448.31201171875],"flags":{},"order":11,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":135}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"],[134,58,0,66,1,"IMAGE"],[135,66,0,59,0,"IMAGE"],[136,65,0,66,0,"UPSCALE_MODEL"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[518.4454142176471,45.73436484262483]}},"version":0.4} \ No newline at end of file diff --git a/workflows/hellomeme_image_v2_workflow.json b/workflows/hellomeme_image_v2_workflow.json index 348de0e..0bc1797 100644 --- a/workflows/hellomeme_image_v2_workflow.json +++ b/workflows/hellomeme_image_v2_workflow.json @@ -1 +1 @@ -{"last_node_id":64,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":133,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,639327002938488,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v2","x1","huggingface",2]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[290.7474365234375,283.26800537109375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[518.4454142176471,44.84700443002717]}},"version":0.4} \ No newline at end of file +{"last_node_id":64,"last_link_id":133,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,132],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,130],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,131],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":133,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293309029136420,"randomize",2,0]},{"id":64,"type":"GetDriveExpression2","pos":[850,827],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":132},{"name":"images","type":"IMAGE","link":131},{"name":"landmarks","type":"FACELANDMARKS222","link":130}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[133],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[290.7474365234375,283.26800537109375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"],[130,43,0,64,2,"FACELANDMARKS222"],[131,57,0,64,1,"IMAGE"],[132,2,0,64,0,"FACE_TOOLKITS"],[133,64,0,58,5,"DRIVE_EXPRESSION2"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/workflows/hellomeme_image_workflow.json b/workflows/hellomeme_image_workflow.json index d255464..0a3ebc3 100644 --- a/workflows/hellomeme_image_workflow.json +++ b/workflows/hellomeme_image_workflow.json @@ -1 +1 @@ -{"last_node_id":63,"last_link_id":129,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,127],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":115}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,126],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":116},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,125],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"ref_image","type":"IMAGE","link":118},{"name":"drive_pose","type":"DRIVE_POSE","link":120},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":124,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,334630685133663,"randomize",2,0]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[210,246],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":61,"type":"GetDriveExpression","pos":[857,823],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":127},{"name":"images","type":"IMAGE","link":125},{"name":"landmarks","type":"FACELANDMARKS222","link":126}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[124],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v1","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[124,61,0,58,4,"DRIVE_EXPRESSION"],[125,57,0,61,1,"IMAGE"],[126,43,0,61,2,"FACELANDMARKS222"],[127,2,0,61,0,"FACE_TOOLKITS"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[518.4454142176471,44.84700443002717]}},"version":0.4} \ No newline at end of file +{"last_node_id":63,"last_link_id":129,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,119,127],"slot_index":0,"localized_name":"face_toolkits"}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":128,"localized_name":"image"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57,"localized_name":"face_toolkits"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[118],"slot_index":0,"localized_name":"IMAGE"}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":115,"localized_name":"images"}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,126],"slot_index":0,"localized_name":"landmarks"}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[852,661],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":116,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":64,"localized_name":"landmarks"}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[120],"slot_index":0,"localized_name":"drive_pose"}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":57,"type":"LoadImage","pos":[45,867],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[115,116,125],"slot_index":0,"localized_name":"IMAGE"},{"name":"MASK","type":"MASK","links":null,"localized_name":"MASK"}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["yao.jpg","image"]},{"id":58,"type":"HMPipelineImage","pos":[1274,584],"size":[315,326],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","link":129,"localized_name":"hm_image_pipeline"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119,"localized_name":"face_toolkits"},{"name":"ref_image","type":"IMAGE","link":118,"localized_name":"ref_image"},{"name":"drive_pose","type":"DRIVE_POSE","link":120,"localized_name":"drive_pose"},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":124,"shape":7,"localized_name":"drive_exp"},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7,"localized_name":"drive_exp2"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[122],"slot_index":0,"localized_name":"IMAGE"},{"name":"LATENT","type":"LATENT","links":null,"localized_name":"LATENT"}],"properties":{"Node name for S&R":"HMPipelineImage"},"widgets_values":[0,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,68992207739336,"randomize",2,0]},{"id":61,"type":"GetDriveExpression","pos":[857,823],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":127,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":125,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":126,"localized_name":"landmarks"}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[124],"slot_index":0,"localized_name":"drive_exp"}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":62,"type":"LoadImage","pos":[52,289],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[128],"slot_index":0,"localized_name":"IMAGE"},{"name":"MASK","type":"MASK","links":null,"localized_name":"MASK"}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["chillout.jpg","image"]},{"id":59,"type":"PreviewImage","pos":[1688,584],"size":[210,246],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":122,"localized_name":"images"}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":63,"type":"HMImagePipelineLoader","pos":[817,358],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_image_pipeline","type":"HMIMAGEPIPELINE","links":[129],"slot_index":0,"localized_name":"hm_image_pipeline"}],"properties":{"Node name for S&R":"HMImagePipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",2]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[115,57,0,43,1,"IMAGE"],[116,57,0,44,1,"IMAGE"],[118,39,0,58,2,"IMAGE"],[119,2,0,58,1,"FACE_TOOLKITS"],[120,44,0,58,3,"DRIVE_POSE"],[122,58,0,59,0,"IMAGE"],[124,61,0,58,4,"DRIVE_EXPRESSION"],[125,57,0,61,1,"IMAGE"],[126,43,0,61,2,"FACELANDMARKS222"],[127,2,0,61,0,"FACE_TOOLKITS"],[128,62,0,39,0,"IMAGE"],[129,63,0,58,0,"HMIMAGEPIPELINE"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/workflows/hellomeme_video_example.png b/workflows/hellomeme_video_example.png index 6a8bcb8..6f6091e 100644 Binary files a/workflows/hellomeme_video_example.png and b/workflows/hellomeme_video_example.png differ diff --git a/workflows/hellomeme_video_sr_workflow.json b/workflows/hellomeme_video_sr_workflow.json deleted file mode 100644 index b1d8ef5..0000000 --- a/workflows/hellomeme_video_sr_workflow.json +++ /dev/null @@ -1 +0,0 @@ -{"last_node_id":63,"last_link_id":129,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,119],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":126}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,120],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":127},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":9,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":122,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[124],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,992982116813133,"randomize",2,0]},{"id":60,"type":"GetDriveExpression","pos":[845,813],"size":[292.20001220703125,66],"flags":{},"order":8,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"images","type":"IMAGE","link":128},{"name":"landmarks","type":"FACELANDMARKS222","link":120}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[122],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":61,"type":"UpscaleModelLoader","pos":[1261,425],"size":[315,58],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"UPSCALE_MODEL","type":"UPSCALE_MODEL","links":[123],"slot_index":0}],"properties":{"Node name for S&R":"UpscaleModelLoader"},"widgets_values":["RealESRGAN_x2.pth"]},{"id":62,"type":"ImageUpscaleWithModel","pos":[1617,490],"size":[340.20001220703125,46],"flags":{},"order":10,"mode":0,"inputs":[{"name":"upscale_model","type":"UPSCALE_MODEL","link":123},{"name":"image","type":"IMAGE","link":124}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[125],"slot_index":0}],"properties":{"Node name for S&R":"ImageUpscaleWithModel"},"widgets_values":[]},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]PixelPortraitV1","same as checkpoint","v1","x1","huggingface",1]},{"id":59,"type":"VHS_VideoCombine","pos":[2023,561],"size":[214.7587890625,518.7587890625],"flags":{},"order":11,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":125},{"name":"audio","type":"AUDIO","link":129,"shape":7},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00406-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":63,"type":"VHS_LoadVideoFFmpeg","pos":[38.35624313354492,920.7100219726562],"size":[303.46875,535.46875],"flags":{},"order":4,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7,"localized_name":"meta_batch"},{"name":"vae","type":"VAE","link":null,"shape":7,"localized_name":"vae"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[126,127,128],"localized_name":"图像","slot_index":0},{"name":"mask","type":"MASK","links":null,"localized_name":"mask"},{"name":"audio","type":"AUDIO","links":[129],"localized_name":"audio","slot_index":2},{"name":"video_info","type":"VHS_VIDEOINFO","links":null,"localized_name":"video_info"}],"properties":{"Node name for S&R":"VHS_LoadVideoFFmpeg"},"widgets_values":{"video":"amns.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"start_time":0,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"start_time":0,"filename":"amns.mp4","type":"input","format":"video/mp4"},"muted":false}}}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[119,2,0,60,0,"FACE_TOOLKITS"],[120,43,0,60,2,"FACELANDMARKS222"],[122,60,0,53,4,"DRIVE_EXPRESSION"],[123,61,0,62,0,"UPSCALE_MODEL"],[124,53,0,62,1,"IMAGE"],[125,62,0,59,0,"IMAGE"],[126,63,0,43,1,"IMAGE"],[127,63,0,44,1,"IMAGE"],[128,63,0,60,1,"IMAGE"],[129,63,2,59,1,"AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[517.5581350428424,44.84700443002717]}},"version":0.4} \ No newline at end of file diff --git a/workflows/hellomeme_video_v2_sr_workflow.json b/workflows/hellomeme_video_v2_sr_workflow.json deleted file mode 100644 index a537095..0000000 --- a/workflows/hellomeme_video_v2_sr_workflow.json +++ /dev/null @@ -1 +0,0 @@ -{"last_node_id":62,"last_link_id":121,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,114],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":82}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,113],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":83},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":9,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":111,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[120],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,1072798685734367,"randomize",2,0]},{"id":56,"type":"GetDriveExpression2","pos":[844,816],"size":[304.79998779296875,66],"flags":{},"order":8,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":114},{"name":"images","type":"IMAGE","link":112},{"name":"landmarks","type":"FACELANDMARKS222","link":113}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[111],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":60,"type":"UpscaleModelLoader","pos":[1255,397],"size":[315,58],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"UPSCALE_MODEL","type":"UPSCALE_MODEL","links":[121],"slot_index":0}],"properties":{"Node name for S&R":"UpscaleModelLoader"},"widgets_values":["RealESRGAN_x2.pth"]},{"id":62,"type":"ImageUpscaleWithModel","pos":[1620,494],"size":[340.20001220703125,46],"flags":{},"order":10,"mode":0,"inputs":[{"name":"upscale_model","type":"UPSCALE_MODEL","link":121},{"name":"image","type":"IMAGE","link":120}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[119]}],"properties":{"Node name for S&R":"ImageUpscaleWithModel"},"widgets_values":[]},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]PixelPortraitV1","same as checkpoint","v2","x1","huggingface",1]},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":59,"type":"VHS_VideoCombine","pos":[2019,574],"size":[332.5562438964844,636.5562744140625],"flags":{},"order":11,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":119},{"name":"audio","type":"AUDIO","link":118,"shape":7},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00407-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,503.455078125],"flags":{},"order":4,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[82,83,112],"slot_index":0},{"name":"frame_count","type":"INT","links":null},{"name":"audio","type":"AUDIO","links":[118],"slot_index":2},{"name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"amns.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"amns.mp4","type":"input","format":"video/mp4"},"muted":false}}}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[111,56,0,53,5,"DRIVE_EXPRESSION2"],[112,48,0,56,1,"IMAGE"],[113,43,0,56,2,"FACELANDMARKS222"],[114,2,0,56,0,"FACE_TOOLKITS"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[118,48,2,59,1,"AUDIO"],[119,62,0,59,0,"IMAGE"],[120,53,0,62,1,"IMAGE"],[121,60,0,62,0,"UPSCALE_MODEL"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[304.59813504284244,15.564923192234204]}},"version":0.4} \ No newline at end of file diff --git a/workflows/hellomeme_video_v2_workflow.json b/workflows/hellomeme_video_v2_workflow.json index fcfc3aa..ee35b0a 100644 --- a/workflows/hellomeme_video_v2_workflow.json +++ b/workflows/hellomeme_video_v2_workflow.json @@ -1 +1 @@ -{"last_node_id":59,"last_link_id":118,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,114],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":82}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,113],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":83},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":111,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[117],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293728504444278,"randomize",2,0]},{"id":56,"type":"GetDriveExpression2","pos":[844,816],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":114},{"name":"images","type":"IMAGE","link":112},{"name":"landmarks","type":"FACELANDMARKS222","link":113}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[111],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]PixelPortraitV1","same as checkpoint","v2","x1","huggingface",1]},{"id":59,"type":"VHS_VideoCombine","pos":[1668.69091796875,739.6614379882812],"size":[329.22479248046875,633.2247924804688],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":117},{"name":"audio","type":"AUDIO","link":118,"shape":7},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00408-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,503.455078125],"flags":{},"order":3,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[82,83,112],"slot_index":0},{"name":"frame_count","type":"INT","links":null},{"name":"audio","type":"AUDIO","links":[118],"slot_index":2},{"name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"amns.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"amns.mp4","type":"input","format":"video/mp4"},"muted":false}}}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[111,56,0,53,5,"DRIVE_EXPRESSION2"],[112,48,0,56,1,"IMAGE"],[113,43,0,56,2,"FACELANDMARKS222"],[114,2,0,56,0,"FACE_TOOLKITS"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[305.48541421764713,14.677644017429516]}},"version":0.4} \ No newline at end of file +{"last_node_id":59,"last_link_id":118,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,114],"slot_index":0,"localized_name":"face_toolkits"}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116,"localized_name":"image"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57,"localized_name":"face_toolkits"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0,"localized_name":"IMAGE"}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":82,"localized_name":"images"}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,113],"slot_index":0,"localized_name":"landmarks"}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":83,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":64,"localized_name":"landmarks"}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0,"localized_name":"drive_pose"}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115,"localized_name":"hm_video_pipeline"},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97,"localized_name":"face_toolkits"},{"name":"ref_image","type":"IMAGE","link":99,"localized_name":"ref_image"},{"name":"drive_pose","type":"DRIVE_POSE","link":100,"localized_name":"drive_pose"},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":null,"shape":7,"localized_name":"drive_exp"},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":111,"shape":7,"localized_name":"drive_exp2"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[117],"slot_index":0,"localized_name":"IMAGE"},{"name":"LATENT","type":"LATENT","links":null,"localized_name":"LATENT"}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,293728504444278,"randomize",2,0]},{"id":56,"type":"GetDriveExpression2","pos":[844,816],"size":[304.79998779296875,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":114,"localized_name":"face_toolkits"},{"name":"images","type":"IMAGE","link":112,"localized_name":"images"},{"name":"landmarks","type":"FACELANDMARKS222","link":113,"localized_name":"landmarks"}],"outputs":[{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","links":[111],"slot_index":0,"localized_name":"drive_exp2"}],"properties":{"Node name for S&R":"GetDriveExpression2"},"widgets_values":[]},{"id":59,"type":"VHS_VideoCombine","pos":[1668.69091796875,739.6614379882812],"size":[329.22479248046875,633.2247924804688],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":117,"localized_name":"images"},{"name":"audio","type":"AUDIO","link":118,"shape":7,"localized_name":"audio"},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7,"localized_name":"meta_batch"},{"name":"vae","type":"VAE","link":null,"shape":7,"localized_name":"vae"}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null,"localized_name":"Filenames"}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00408-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0,"localized_name":"IMAGE"},{"name":"MASK","type":"MASK","links":null,"localized_name":"MASK"}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,680.3645629882812],"flags":{},"order":3,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7,"localized_name":"meta_batch"},{"name":"vae","type":"VAE","link":null,"shape":7,"localized_name":"vae"}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[82,83,112],"slot_index":0,"localized_name":"IMAGE"},{"name":"frame_count","type":"INT","links":null,"localized_name":"frame_count"},{"name":"audio","type":"AUDIO","links":[118],"slot_index":2,"localized_name":"audio"},{"name":"video_info","type":"VHS_VIDEOINFO","links":null,"localized_name":"video_info"}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0,"localized_name":"hm_video_pipeline"}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]RealisticVisionV60B1","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",1]}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[111,56,0,53,5,"DRIVE_EXPRESSION2"],[112,48,0,56,1,"IMAGE"],[113,43,0,56,2,"FACELANDMARKS222"],[114,2,0,56,0,"FACE_TOOLKITS"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.8264462809917358,"offset":[405.930209956171,-178.15274132497817]}},"version":0.4} \ No newline at end of file diff --git a/workflows/hellomeme_video_workflow.json b/workflows/hellomeme_video_workflow.json index 409680d..2f183e8 100644 --- a/workflows/hellomeme_video_workflow.json +++ b/workflows/hellomeme_video_workflow.json @@ -1 +1 @@ -{"last_node_id":60,"last_link_id":122,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,119],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":82}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,120],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":83},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":122,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[117],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,896523808870162,"randomize",2,0]},{"id":60,"type":"GetDriveExpression","pos":[845,813],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"images","type":"IMAGE","link":121},{"name":"landmarks","type":"FACELANDMARKS222","link":120}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[122],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]PixelPortraitV1","same as checkpoint","v2","x1","huggingface",1]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,503.455078125],"flags":{},"order":3,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[82,83,121],"slot_index":0},{"name":"frame_count","type":"INT","links":null},{"name":"audio","type":"AUDIO","links":[118],"slot_index":2},{"name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"amns.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"amns.mp4","type":"input","format":"video/mp4"},"muted":false}}},{"id":59,"type":"VHS_VideoCombine","pos":[1619,800],"size":[280.4215087890625,584.4215087890625],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":117},{"name":"audio","type":"AUDIO","link":118,"shape":7},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00409-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"],[119,2,0,60,0,"FACE_TOOLKITS"],[120,43,0,60,2,"FACELANDMARKS222"],[121,48,0,60,1,"IMAGE"],[122,60,0,53,4,"DRIVE_EXPRESSION"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015778,"offset":[305.48541421764713,12.902760716648265]}},"version":0.4} \ No newline at end of file +{"last_node_id":60,"last_link_id":122,"nodes":[{"id":2,"type":"HMFaceToolkitsLoader","pos":[43,699],"size":[315,82],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","links":[57,62,65,97,119],"slot_index":0}],"properties":{"Node name for S&R":"HMFaceToolkitsLoader"},"widgets_values":[0,"huggingface"]},{"id":39,"type":"CropPortrait","pos":[516,537],"size":[241.79998779296875,46],"flags":{},"order":4,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":116},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":57}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[99],"slot_index":0}],"properties":{"Node name for S&R":"CropPortrait"},"widgets_values":[]},{"id":43,"type":"GetFaceLandmarks","pos":[465,771],"size":[292.20001220703125,46],"flags":{},"order":5,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":62},{"name":"images","type":"IMAGE","link":82}],"outputs":[{"name":"landmarks","type":"FACELANDMARKS222","links":[64,120],"slot_index":0}],"properties":{"Node name for S&R":"GetFaceLandmarks"},"widgets_values":[]},{"id":44,"type":"GetDrivePose","pos":[842,677],"size":[304.79998779296875,66],"flags":{},"order":6,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":65},{"name":"images","type":"IMAGE","link":83},{"name":"landmarks","type":"FACELANDMARKS222","link":64}],"outputs":[{"name":"drive_pose","type":"DRIVE_POSE","links":[100],"slot_index":0}],"properties":{"Node name for S&R":"GetDrivePose"},"widgets_values":[]},{"id":53,"type":"HMPipelineVideo","pos":[1258,562],"size":[315,350],"flags":{},"order":8,"mode":0,"inputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","link":115},{"name":"face_toolkits","type":"FACE_TOOLKITS","link":97},{"name":"ref_image","type":"IMAGE","link":99},{"name":"drive_pose","type":"DRIVE_POSE","link":100},{"name":"drive_exp","type":"DRIVE_EXPRESSION","link":122,"shape":7},{"name":"drive_exp2","type":"DRIVE_EXPRESSION2","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[117],"slot_index":0},{"name":"LATENT","type":"LATENT","links":null}],"properties":{"Node name for S&R":"HMPipelineVideo"},"widgets_values":[0,4,"(best quality), highly detailed, ultra-detailed, headshot, person, well-placed five sense organs, looking at the viewer, centered composition, sharp focus, realistic skin texture","",25,331732715385901,"randomize",2,0]},{"id":60,"type":"GetDriveExpression","pos":[845,813],"size":[292.20001220703125,66],"flags":{},"order":7,"mode":0,"inputs":[{"name":"face_toolkits","type":"FACE_TOOLKITS","link":119},{"name":"images","type":"IMAGE","link":121},{"name":"landmarks","type":"FACELANDMARKS222","link":120}],"outputs":[{"name":"drive_exp","type":"DRIVE_EXPRESSION","links":[122],"slot_index":0}],"properties":{"Node name for S&R":"GetDriveExpression"},"widgets_values":[]},{"id":58,"type":"LoadImage","pos":[56,273],"size":[315,314],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[116],"slot_index":0},{"name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["toon.png","image"]},{"id":59,"type":"VHS_VideoCombine","pos":[1619,800],"size":[280.4215087890625,584.4215087890625],"flags":{},"order":9,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":117},{"name":"audio","type":"AUDIO","link":118,"shape":7},{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"Filenames","type":"VHS_FILENAMES","links":null}],"properties":{"Node name for S&R":"VHS_VideoCombine"},"widgets_values":{"frame_rate":15,"loop_count":0,"filename_prefix":"AnimateDiff","format":"video/h264-mp4","pix_fmt":"yuv420p","crf":18,"save_metadata":true,"pingpong":false,"save_output":true,"videopreview":{"hidden":false,"paused":false,"params":{"filename":"AnimateDiff_00409-audio.mp4","subfolder":"","type":"output","format":"video/h264-mp4","frame_rate":15},"muted":false}}},{"id":57,"type":"HMVideoPipelineLoader","pos":[813,343],"size":[352.79998779296875,202],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[{"name":"hm_video_pipeline","type":"HMVIDEOPIPELINE","links":[115],"slot_index":0}],"properties":{"Node name for S&R":"HMVideoPipelineLoader"},"widgets_values":["[preset]DisneyPixarCartoonB","[preset]MoreDetails","same as checkpoint","v3","x1","huggingface",1]},{"id":48,"type":"VHS_LoadVideo","pos":[106,867],"size":[247.455078125,680.3645629882812],"flags":{},"order":3,"mode":0,"inputs":[{"name":"meta_batch","type":"VHS_BatchManager","link":null,"shape":7},{"name":"vae","type":"VAE","link":null,"shape":7}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[82,83,121],"slot_index":0},{"name":"frame_count","type":"INT","links":null},{"name":"audio","type":"AUDIO","links":[118],"slot_index":2},{"name":"video_info","type":"VHS_VIDEOINFO","links":null}],"properties":{"Node name for S&R":"VHS_LoadVideo"},"widgets_values":{"video":"tiktok.mp4","force_rate":15,"force_size":"Disabled","custom_width":512,"custom_height":512,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"choose video to upload":"image","videopreview":{"hidden":false,"paused":false,"params":{"force_rate":15,"frame_load_cap":0,"skip_first_frames":0,"select_every_nth":1,"filename":"tiktok.mp4","type":"input","format":"video/mp4"},"muted":false}}}],"links":[[57,2,0,39,1,"FACE_TOOLKITS"],[62,2,0,43,0,"FACE_TOOLKITS"],[64,43,0,44,2,"FACELANDMARKS222"],[65,2,0,44,0,"FACE_TOOLKITS"],[82,48,0,43,1,"IMAGE"],[83,48,0,44,1,"IMAGE"],[97,2,0,53,1,"FACE_TOOLKITS"],[99,39,0,53,2,"IMAGE"],[100,44,0,53,3,"DRIVE_POSE"],[115,57,0,53,0,"HMVIDEOPIPELINE"],[116,58,0,39,0,"IMAGE"],[117,53,0,59,0,"IMAGE"],[118,48,2,59,1,"AUDIO"],[119,2,0,60,0,"FACE_TOOLKITS"],[120,43,0,60,2,"FACELANDMARKS222"],[121,48,0,60,1,"IMAGE"],[122,60,0,53,4,"DRIVE_EXPRESSION"]],"groups":[],"config":{},"extra":{"ds":{"scale":0.7513148009015779,"offset":[634.1362099561715,-70.9744212566187]}},"version":0.4} \ No newline at end of file