From a80a4dfa4b5e02a26d6944f51febeef265960e4e Mon Sep 17 00:00:00 2001 From: MrForExample <62230687+MrForExample@users.noreply.github.com> Date: Fri, 2 Feb 2024 14:48:30 +0100 Subject: [PATCH] Test Stable-Fast --- .gitignore | 1 + README.md | 7 +++++-- __init__.py | 2 +- requirements.txt | 2 +- src/models/main_diffuser.py | 21 +++++++++++++++++++++ src/models/unet_2d_condition.py | 4 ++-- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index c68c08a..98ca087 100644 --- a/.gitignore +++ b/.gitignore @@ -152,6 +152,7 @@ cython_debug/ #.idea/ # Model files +*.ckpt *.bin *.pth *.safetensors diff --git a/README.md b/README.md index eae3023..72ca9cf 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,13 @@ - **Result:** Generated result is not good enough when using DDIM Scheduler togather with RCFG, even though it speed up the generating process by about 4X.
In StreamDiffusion, RCFG works with LCM, could also be the case here, so keep it in another branch for now. +- [ ] Incorporate the implementation & Pre-trained Models from [Open-AnimateAnyone](https://github.com/guoqincode/Open-AnimateAnyone) & [AnimateAnyone](https://github.com/HumanAIGC/AnimateAnyone) once they released - [ ] Convert Model using [stable-fast](https://github.com/chengzeyi/stable-fast) (**Estimated speed up: 2X**) - [ ] Train a LCM Lora for denoise unet (**Estimated speed up: 5X**) -- [ ] Incorporate the implementation & Pre-trained Models from [Open-AnimateAnyone](https://github.com/guoqincode/Open-AnimateAnyone) & [AnimateAnyone](https://github.com/HumanAIGC/AnimateAnyone) once they released - [ ] Training a new Model using better dataset to improve results quality (Optional, we'll see if there is any need for me to do it ;) - Continuous research, always moving towards something better & faster🚀 -## Install (Will add it to ComfyUI Manager Soon!) +## Install (You can also use ComfyUI Manager) 1. Clone this repo into the `Your ComfyUI root directory\ComfyUI\custom_nodes\` and install dependent Python packages: ```bash @@ -55,6 +55,9 @@ git clone https://github.com/MrForExample/ComfyUI-AnimateAnyone-Evolved.git pip install -r requirements.txt + + # If you got error regards diffusers then run: + pip install --force-reinstall diffusers==0.26.1 ``` 2. Download pre-trained models: - [stable-diffusion-v1-5_unet](https://huggingface.co/runwayml/stable-diffusion-v1-5/tree/main/unet) diff --git a/__init__.py b/__init__.py index bb2ebef..d3fbcf7 100644 --- a/__init__.py +++ b/__init__.py @@ -10,7 +10,7 @@ for name, cls in inspect.getmembers(module, inspect.isclass): if cls.__module__ == module.__name__: name = name.replace("_", " ") - node = f"[ComfyUI-3D] {name}" + node = f"[AnimateAnyone] {name}" disp = f"{name}" NODE_CLASS_MAPPINGS[node] = cls diff --git a/requirements.txt b/requirements.txt index 46e2753..11cc1c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ accelerate clip @ https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip#sha256=b5842c25da441d6c581b53a5c60e0c2127ebafe0f746f8e15561a006c6c3be6a -diffusers==0.25.1 +diffusers>=0.26.1 einops gradio gradio_client diff --git a/src/models/main_diffuser.py b/src/models/main_diffuser.py index 038e9c8..c856c78 100644 --- a/src/models/main_diffuser.py +++ b/src/models/main_diffuser.py @@ -19,6 +19,7 @@ from diffusers.schedulers import ( PNDMScheduler, ) from diffusers.utils.torch_utils import randn_tensor +from sfast.compilers.diffusion_pipeline_compiler import compile_unet, CompilationConfig from .mutual_self_attention import ReferenceAttentionControl from ..utils.util import get_tensor_interpolation_method, get_context_scheduler @@ -320,6 +321,26 @@ class AADiffusion(DiffusionPipeline, LoraLoaderMixin): ) reference_control_reader.update(reference_control_writer) + config = CompilationConfig.Default() + # xformers and Triton are suggested for achieving best performance. + try: + import xformers + config.enable_xformers = True + except ImportError: + print('xformers not installed, skip') + try: + import triton + config.enable_triton = True + except ImportError: + print('Triton not installed, skip') + + # CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead. + # But it can increase the amount of GPU memory used. + # For StableVideoDiffusionPipeline it is not needed. + config.enable_cuda_graph = True + + self.unet = compile_unet(self.unet, config) + # Prepare timesteps self.scheduler.set_timesteps(loop_steps, device=device) timesteps = self.scheduler.timesteps diff --git a/src/models/unet_2d_condition.py b/src/models/unet_2d_condition.py index 6490c51..60a09ec 100644 --- a/src/models/unet_2d_condition.py +++ b/src/models/unet_2d_condition.py @@ -20,7 +20,7 @@ from diffusers.models.embeddings import ( ImageHintTimeEmbedding, ImageProjection, ImageTimeEmbedding, - PositionNet, + GLIGENTextBoundingboxProjection, TextImageProjection, TextImageTimeEmbedding, TextTimeEmbedding, @@ -662,7 +662,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin) positive_len = cross_attention_dim[0] feature_type = "text-only" if attention_type == "gated" else "text-image" - self.position_net = PositionNet( + self.position_net = GLIGENTextBoundingboxProjection( positive_len=positive_len, out_dim=cross_attention_dim, feature_type=feature_type,