diff --git a/README.md b/README.md index ddb73e9..ed5e473 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,12 @@ ## 🔆 New Features/Updates - ✅ `02/09/2025` **HelloMemeV3** is now available. - - [![YouTube Demo](https://img.youtube.com/vi/DAUA0EYjsZA/0.jpg)](https://www.youtube.com/watch?v=DAUA0EYjsZA) +[YouTube Demo](https://www.youtube.com/watch?v=DAUA0EYjsZA) - ✅ `12/17/2024` Support modelscope ([Modelscope Demo](https://www.modelscope.cn/studios/songkey/HelloMeme)). - ✅ `12/08/2024` Added **HelloMemeV2** (select "v2" in the version option of the LoadHelloMemeImage/Video Node). Its features include: a. Improved expression consistency between the generated video and the driving video. -b. Better compatibility with third-party checkpoints (*we will continuously collect compatible free third-party checkpoints and share them on this page. If you'd like to recommend one, please post it on the issue page or email us at songkey@pku.edu.cn*). +b. Better compatibility with third-party checkpoints. c. Reduced VRAM usage. [YouTube Demo](https://www.youtube.com/watch?v=-2s_pLAKoRg) diff --git a/example_workflows/image_generation_ctrl2.jpg b/example_workflows/image_generation_ctrl2.jpg index a57e142..5ad39e5 100644 Binary files a/example_workflows/image_generation_ctrl2.jpg and b/example_workflows/image_generation_ctrl2.jpg differ diff --git a/hellomeme/pipelines/pipline_hm3_video.py b/hellomeme/pipelines/pipline_hm3_video.py index 73abd1f..e11c071 100644 --- a/hellomeme/pipelines/pipline_hm3_video.py +++ b/hellomeme/pipelines/pipline_hm3_video.py @@ -20,7 +20,7 @@ from diffusers.image_processor import PipelineImageInput from diffusers.utils import deprecate from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import retrieve_timesteps, retrieve_latents -from diffusers import MotionAdapter, DPMSolverMultistepScheduler +from diffusers import MotionAdapter, EulerDiscreteScheduler # DPMSolverMultistepScheduler from diffusers.utils.torch_utils import randn_tensor from ..models import (HM3Denoising3D, HM3DenoisingMotion, HM3MotionAdapter, @@ -40,11 +40,18 @@ class HM3VideoPipeline(HMPipeline): if modelscope: from modelscope import snapshot_download - hm_animatediff_dir = snapshot_download('songkey/hm3_animatediff') + if version == 'v3': + hm_animatediff_dir = snapshot_download('songkey/hm3_animatediff') + else: + hm_animatediff_dir = snapshot_download('songkey/hm4_animatediff') else: - hm_animatediff_dir = 'songkey/hm3_animatediff' + if version == 'v3': + hm_animatediff_dir = 'songkey/hm3_animatediff' + else: + hm_animatediff_dir = 'songkey/hm4_animatediff' self.num_frames = 12 + adapter = MotionAdapter.from_pretrained(hm_animatediff_dir, torch_dtype=torch.float16) unet = HM3DenoisingMotion.from_unet2d(unet=self.unet, motion_adapter=adapter, load_weights=True) @@ -83,7 +90,7 @@ class HM3VideoPipeline(HMPipeline): hm_motion_dir = 'songkey/hm4_motion' hm_adapter = HM3ReferenceAdapter.from_pretrained(hm_reference_dir) - motion_adapter = HM3MotionAdapter.from_pretrained(hm_motion_dir) + motion_adapter = HM3MotionAdapter().from_pretrained(hm_motion_dir) if isinstance(self.unet, HM3DenoisingMotion): self.unet.insert_reference_adapter(hm_adapter) @@ -156,12 +163,12 @@ class HM3VideoPipeline(HMPipeline): paded_indexes = copy.deepcopy(paded_indexes) margin_indexes = [indexes[0], indexes[-1]] - scheduler = DPMSolverMultistepScheduler( + scheduler = EulerDiscreteScheduler( num_train_timesteps=1000, beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", - algorithm_type="sde-dpmsolver++", + # algorithm_type="sde-dpmsolver++", ) tmp_timesteps, _ = retrieve_timesteps(scheduler, step, device, timesteps, sigmas) @@ -235,12 +242,12 @@ class HM3VideoPipeline(HMPipeline): self.gen_video(margin_indexes, pad_dict, base_noise, depth+1, control_dict, step, more_params) ## generate latent - scheduler = DPMSolverMultistepScheduler( + scheduler = EulerDiscreteScheduler( num_train_timesteps=1000, beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", - algorithm_type="sde-dpmsolver++", + # algorithm_type="sde-dpmsolver++", ) timesteps, _ = retrieve_timesteps(scheduler, step, device, timesteps, sigmas) diff --git a/meme.py b/meme.py index 407647f..7d7f6d4 100644 --- a/meme.py +++ b/meme.py @@ -142,7 +142,7 @@ class HMImagePipelineLoader: sd1_5_dir = snapshot_download('songkey/stable-diffusion-v1-5') else: sd1_5_dir = "songkey/stable-diffusion-v1-5" - if version == 'v3' or version == 'v4': + if version == 'v3' or version == 'v4': pipeline = HM3ImagePipeline.from_pretrained(sd1_5_dir) else: pipeline = HMImagePipeline.from_pretrained(sd1_5_dir) @@ -187,7 +187,7 @@ class HMVideoPipelineLoader: else: sd1_5_dir = "songkey/stable-diffusion-v1-5" - if version == 'v3' or version == 'v4': + if version == 'v3' or version == 'v4': pipeline = HM3VideoPipeline.from_pretrained(sd1_5_dir) else: pipeline = HMVideoPipeline.from_pretrained(sd1_5_dir) diff --git a/pyproject.toml b/pyproject.toml index 07161fd..61bf4ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_hellomeme" description = "This repository is the official implementation of the [a/HelloMeme](https://arxiv.org/pdf/2410.22901) ComfyUI interface" -version = "1.2.0" +version = "1.2.1" license = {file = "LICENSE"} [project.urls]