From 8c284eb65d6422f2998afec1ec116c5cc4998f49 Mon Sep 17 00:00:00 2001 From: songkey-win Date: Sun, 9 Feb 2025 02:16:34 +0800 Subject: [PATCH] update --- hellomeme/__init__.py | 2 +- hellomeme/model_config.json | 5 ++++- hellomeme/models/__init__.py | 2 ++ hellomeme/models/hm_blocks.py | 8 ++++---- hellomeme/pipelines/__init__.py | 4 +++- meme.py | 17 +++++++++++------ pyproject.toml | 2 +- 7 files changed, 26 insertions(+), 14 deletions(-) diff --git a/hellomeme/__init__.py b/hellomeme/__init__.py index a5ac580..6b1739a 100644 --- a/hellomeme/__init__.py +++ b/hellomeme/__init__.py @@ -8,4 +8,4 @@ @Desc : """ -from .pipelines import HMImagePipeline, HMVideoPipeline \ No newline at end of file +from .pipelines import HMImagePipeline, HMVideoPipeline, HM3ImagePipeline, HM3VideoPipeline \ No newline at end of file diff --git a/hellomeme/model_config.json b/hellomeme/model_config.json index 06f5db7..57eed45 100644 --- a/hellomeme/model_config.json +++ b/hellomeme/model_config.json @@ -3,7 +3,10 @@ "checkpoints": { "SD1.5": "songkey/stable-diffusion-v1-5", "[preset]RealisticVisionV60B1": "songkey/realisticVisionV60B1_v51VAE", - "[preset]DisneyPixarCartoonB": "songkey/disney-pixar-cartoon-b" + "[preset]DisneyPixarCartoonB": "songkey/disney-pixar-cartoon-b", + "[preset]toonyou_beta6": "songkey/toonyou_beta6", + "[preset]LZ_2DCartoon_V2": "songkey/LZ_2DCartoon_V2", + "[preset]meinamix_v12Final": "songkey/meinamix_v12Final" }, "loras": { "[preset]BabyFaceV1": ["songkey/loras_sd_1_5", "baby_face_v1.safetensors"], diff --git a/hellomeme/models/__init__.py b/hellomeme/models/__init__.py index 8c4b693..f225943 100644 --- a/hellomeme/models/__init__.py +++ b/hellomeme/models/__init__.py @@ -12,3 +12,5 @@ from .hm_denoising_motion import HMDenoisingMotion from .hm_control import HMControlNet, HMControlNet2, HMV2ControlNet, HMV2ControlNet2, HMV3ControlNet2, HMV3ControlNet from .hm_blocks import HMReferenceAdapter, HM3ReferenceAdapter, HM3MotionAdapter, HMPipeline from .hm_denoising_3d import HMDenoising3D +from .hm3_denoising_3d import HM3Denoising3D +from .hm3_denoising_motion import HM3DenoisingMotion diff --git a/hellomeme/models/hm_blocks.py b/hellomeme/models/hm_blocks.py index 0a05cd7..a9d7081 100644 --- a/hellomeme/models/hm_blocks.py +++ b/hellomeme/models/hm_blocks.py @@ -380,18 +380,18 @@ class SKMotionModule(nn.Module): self.proj = zero_module(nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1)) def forward(self, hidden_states, pad_states, temb, num_frames): - temb = rearrange(self.time_emb_proj(temb), "(b f) c -> b c f 1 1", f=num_frames)[:, :, :1] + temb = rearrange(self.time_emb_proj(temb), "(b f) c -> b c f 1 1", f=num_frames)[:,:,:1] inputt = rearrange(hidden_states, "(b f) c h w -> b c f h w", f=num_frames) - inputt = torch.cat([pad_states[:, :, :1].clone(), inputt, pad_states[:, :, -1:].clone()], dim=2).contiguous() + inputt = torch.cat([pad_states[:,:,:1], inputt, pad_states[:,:,-1:]], dim=2).contiguous() h, w = inputt.shape[-2:] - inputt = rearrange(inputt + temb, "b c f h w -> (b h w) f c") + inputt = rearrange(inputt+temb, "b c f h w -> (b h w) f c") res_temp = self.ff(self.temp_attn(self.norm(self.pos_embed_t(inputt)))) res_temp = rearrange(res_temp, "(b h w) f c -> b c f h w", h=h, w=w) - res_temp = rearrange(res_temp[:, :, 1:-1], "b c f h w -> (b f) c h w") + res_temp = rearrange(res_temp[:,:,1:-1], "b c f h w -> (b f) c h w") return hidden_states + self.proj(res_temp) diff --git a/hellomeme/pipelines/__init__.py b/hellomeme/pipelines/__init__.py index aef9f54..dfc6a26 100644 --- a/hellomeme/pipelines/__init__.py +++ b/hellomeme/pipelines/__init__.py @@ -9,4 +9,6 @@ """ from .pipline_hm_image import HMImagePipeline -from .pipline_hm_video import HMVideoPipeline \ No newline at end of file +from .pipline_hm_video import HMVideoPipeline +from .pipline_hm3_image import HM3ImagePipeline +from .pipline_hm3_video import HM3VideoPipeline \ No newline at end of file diff --git a/meme.py b/meme.py index 563b089..60daf17 100644 --- a/meme.py +++ b/meme.py @@ -43,7 +43,7 @@ from .hellomeme.utils import (get_drive_expression, append_pipline_weights, load_face_toolkits ) -from .hellomeme import HMImagePipeline, HMVideoPipeline +from .hellomeme import HMImagePipeline, HMVideoPipeline, HM3ImagePipeline, HM3VideoPipeline config_path = osp.join(cur_dir, 'hellomeme', 'model_config.json') with open(config_path, 'r') as f: @@ -114,7 +114,7 @@ class HMImagePipelineLoader: "checkpoint": (checkpoint_files, ), "lora": (lora_files, ), "vae": (vae_files, ), - "version": (['v1', 'v2'], ), + "version": (['v1', 'v2', 'v3'], ), "stylize": (['x1', 'x2'], ), "deployment": (['huggingface', 'modelscope'], ), "lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), @@ -132,8 +132,10 @@ class HMImagePipelineLoader: sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5') else: sd1_5_dir = "songkey/stable-diffusion-v1-5" - - pipeline = HMImagePipeline.from_pretrained(sd1_5_dir) + if version == 'v3': + pipeline = HM3ImagePipeline.from_pretrained(sd1_5_dir) + else: + pipeline = HMImagePipeline.from_pretrained(sd1_5_dir) pipeline.to(dtype=dtype) pipeline.caryomitosis(version=version, modelscope=deployment=='modelscope') @@ -154,7 +156,7 @@ class HMVideoPipelineLoader: "checkpoint": (checkpoint_files, ), "lora": (lora_files, ), "vae": (vae_files, ), - "version": (['v1', 'v2'], ), + "version": (['v1', 'v2', 'v3'], ), "stylize": (['x1', 'x2'], ), "deployment": (['huggingface', 'modelscope'], ), "lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.1}), @@ -175,7 +177,10 @@ class HMVideoPipelineLoader: else: sd1_5_dir = "songkey/stable-diffusion-v1-5" - pipeline = HMVideoPipeline.from_pretrained(sd1_5_dir) + if version == 'v3': + pipeline = HM3VideoPipeline.from_pretrained(sd1_5_dir) + else: + pipeline = HMVideoPipeline.from_pretrained(sd1_5_dir) pipeline.to(dtype=dtype) pipeline.caryomitosis(version=version, modelscope=deployment=='modelscope') diff --git a/pyproject.toml b/pyproject.toml index e2e4f2b..f4af499 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_hellomeme" description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface" -version = "1.0.3" +version = "1.0.4" license = {file = "LICENSE"} [project.urls]