Test Stable-Fast

.gitignore

@@ -152,6 +152,7 @@ cython_debug/
 #.idea/
 
 # Model files
+*.ckpt
 *.bin
 *.pth
 *.safetensors

README.md

@@ -40,13 +40,13 @@
   - **Result:**  
     Generated result is not good enough when using DDIM Scheduler togather with RCFG, even though it speed up the generating process by about 4X.<br>
     In StreamDiffusion, RCFG works with LCM, could also be the case here, so keep it in another branch for now.
+- [ ] Incorporate the implementation & Pre-trained Models from [Open-AnimateAnyone](https://github.com/guoqincode/Open-AnimateAnyone) & [AnimateAnyone](https://github.com/HumanAIGC/AnimateAnyone) once they released
 - [ ] Convert Model using [stable-fast](https://github.com/chengzeyi/stable-fast) (**Estimated speed up: 2X**)
 - [ ] Train a LCM Lora for denoise unet (**Estimated speed up: 5X**)
-- [ ] Incorporate the implementation & Pre-trained Models from [Open-AnimateAnyone](https://github.com/guoqincode/Open-AnimateAnyone) & [AnimateAnyone](https://github.com/HumanAIGC/AnimateAnyone) once they released
 - [ ] Training a new Model using better dataset to improve results quality (Optional, we'll see if there is any need for me to do it ;)
 - Continuous research, always moving towards something better & faster🚀
 
-## Install (Will add it to ComfyUI Manager Soon!)
+## Install (You can also use ComfyUI Manager)
 
 1.  Clone this repo into the  `Your ComfyUI root directory\ComfyUI\custom_nodes\` and install dependent Python packages:
     ```bash
@@ -55,6 +55,9 @@
     git clone https://github.com/MrForExample/ComfyUI-AnimateAnyone-Evolved.git
 
     pip install -r requirements.txt
+
+    # If you got error regards diffusers then run:
+    pip install --force-reinstall diffusers==0.26.1
     ```
 2. Download pre-trained models:
     - [stable-diffusion-v1-5_unet](https://huggingface.co/runwayml/stable-diffusion-v1-5/tree/main/unet)

__init__.py

@@ -10,7 +10,7 @@ for name, cls in inspect.getmembers(module, inspect.isclass):
     if cls.__module__ == module.__name__:
         name = name.replace("_", " ")
 
-        node = f"[ComfyUI-3D] {name}"
+        node = f"[AnimateAnyone] {name}"
         disp = f"{name}"
 
         NODE_CLASS_MAPPINGS[node] = cls

requirements.txt

@@ -1,6 +1,6 @@
 accelerate
 clip @ https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip#sha256=b5842c25da441d6c581b53a5c60e0c2127ebafe0f746f8e15561a006c6c3be6a
-diffusers==0.25.1
+diffusers>=0.26.1
 einops
 gradio
 gradio_client

src/models/main_diffuser.py

@@ -19,6 +19,7 @@ from diffusers.schedulers import (
     PNDMScheduler,
 )
 from diffusers.utils.torch_utils import randn_tensor
+from sfast.compilers.diffusion_pipeline_compiler import compile_unet, CompilationConfig
 
 from .mutual_self_attention import ReferenceAttentionControl
 from ..utils.util import get_tensor_interpolation_method, get_context_scheduler
@@ -320,6 +321,26 @@ class AADiffusion(DiffusionPipeline, LoraLoaderMixin):
         )
         reference_control_reader.update(reference_control_writer)
         
+        config = CompilationConfig.Default()
+        # xformers and Triton are suggested for achieving best performance.
+        try:
+            import xformers
+            config.enable_xformers = True
+        except ImportError:
+            print('xformers not installed, skip')
+        try:
+            import triton
+            config.enable_triton = True
+        except ImportError:
+            print('Triton not installed, skip')
+
+        # CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead.
+        # But it can increase the amount of GPU memory used.
+        # For StableVideoDiffusionPipeline it is not needed.
+        config.enable_cuda_graph = True
+
+        self.unet = compile_unet(self.unet, config)
+        
         # Prepare timesteps
         self.scheduler.set_timesteps(loop_steps, device=device)
         timesteps = self.scheduler.timesteps

src/models/unet_2d_condition.py

@@ -20,7 +20,7 @@ from diffusers.models.embeddings import (
     ImageHintTimeEmbedding,
     ImageProjection,
     ImageTimeEmbedding,
-    PositionNet,
+    GLIGENTextBoundingboxProjection,
     TextImageProjection,
     TextImageTimeEmbedding,
     TextTimeEmbedding,
@@ -662,7 +662,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin)
                 positive_len = cross_attention_dim[0]
 
             feature_type = "text-only" if attention_type == "gated" else "text-image"
-            self.position_net = PositionNet(
+            self.position_net = GLIGENTextBoundingboxProjection(
                 positive_len=positive_len,
                 out_dim=cross_attention_dim,
                 feature_type=feature_type,