nodes/MrForExample-ComfyUI-AnimateAnyone-Evolved
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update class reference to support newest diffuser >= v.0.26.1

Browse Source
This commit is contained in:
MrForExample
2024-02-02 15:19:18 +01:00
parent a80a4dfa4b
commit ea56d60cde
5 changed files with 293 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -57,7 +57,7 @@
pip install -r requirements.txt
# If you got error regards diffusers then run:
pip install --force-reinstall diffusers==0.26.1
pip install --force-reinstall diffusers>=0.26.1
```
2. Download pre-trained models:
- [stable-diffusion-v1-5_unet](https://huggingface.co/runwayml/stable-diffusion-v1-5/tree/main/unet)
_Example_Workflow/MooreAnimateAnyone_DDIM.json
+289 -289
View File
@@ -1,43 +1,7 @@
{
"last_node_id": 31,
"last_link_id": 56,
"last_node_id": 36,
"last_link_id": 64,
"nodes": [
{
"id": 15,
"type": "Reroute",
"pos": [
1578,
-110
],
"size": [
75,
26
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 14
}
],
"outputs": [
{
"name": "",
"type": "VAE",
"links": [
15
],
"slot_index": 0
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
},
{
"id": 14,
"type": "VAEDecode",
@@ -56,7 +20,7 @@
{
"name": "samples",
"type": "LATENT",
"link": 56
"link": 60
},
{
"name": "vae",
@@ -79,38 +43,6 @@
"Node name for S&R": "VAEDecode"
}
},
{
"id": 7,
"type": "[ComfyUI-3D] Load Pose Guider",
"pos": [
1106,
704
],
"size": {
"0": 366.0615234375,
"1": 58
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "pose_guider",
"type": "POSE_GUIDER",
"links": [
5
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[ComfyUI-3D] Load Pose Guider"
},
"widgets_values": [
"./pretrained_weights/pose_guider.pth"
]
},
{
"id": 6,
"type": "CLIPVisionLoader",
@@ -123,7 +55,7 @@
"1": 58
},
"flags": {},
"order": 1,
"order": 0,
"mode": 0,
"outputs": [
{
@@ -155,7 +87,7 @@
"1": 314
},
"flags": {},
"order": 2,
"order": 1,
"mode": 0,
"outputs": [
{
@@ -194,7 +126,7 @@
"1": 58
},
"flags": {},
"order": 3,
"order": 2,
"mode": 0,
"outputs": [
{
@@ -229,7 +161,7 @@
"1": 58
},
"flags": {},
"order": 4,
"order": 3,
"mode": 0,
"outputs": [
{
@@ -264,7 +196,7 @@
"1": 60.091087341308594
},
"flags": {},
"order": 5,
"order": 4,
"mode": 0,
"outputs": [
{
@@ -348,6 +280,38 @@
"disabled"
]
},
{
"id": 35,
"type": "[AnimateAnyone] Load Pose Guider",
"pos": [
1169,
703
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 5,
"mode": 0,
"outputs": [
{
"name": "pose_guider",
"type": "POSE_GUIDER",
"links": [
61
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[AnimateAnyone] Load Pose Guider"
},
"widgets_values": [
"./pretrained_weights/pose_guider.pth"
]
},
{
"id": 23,
"type": "ImageScale",
@@ -390,7 +354,7 @@
"name": "IMAGE",
"type": "IMAGE",
"links": [
27
62
],
"shape": 3,
"slot_index": 0
@@ -408,74 +372,94 @@
]
},
{
"id": 16,
"type": "VHS_VideoCombine",
"id": 32,
"type": "[AnimateAnyone] Animate Anyone Sampler",
"pos": [
2986,
-130
],
"size": [
315,
701.5
2095,
48
],
"size": {
"0": 315,
"1": 618
},
"flags": {},
"order": 17,
"order": 15,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 16
"name": "reference_unet",
"type": "UNET2D",
"link": 58
},
{
"name": "audio",
"type": "VHS_AUDIO",
"link": null
"name": "denoising_unet",
"type": "UNET3D",
"link": 59
},
{
"name": "ref_image_latent",
"type": "LATENT",
"link": 57
},
{
"name": "clip_image_embeds",
"type": "CLIP_VISION_OUTPUT",
"link": 64
},
{
"name": "pose_latent",
"type": "POSE_LATENT",
"link": 63
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null,
"shape": 3
"name": "latent",
"type": "LATENT",
"links": [
60
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
"Node name for S&R": "[AnimateAnyone] Animate Anyone Sampler"
},
"widgets_values": {
"frame_rate": 12,
"loop_count": 0,
"filename_prefix": "AnimateAnyone/video",
"format": "video/h265-mp4",
"pix_fmt": "yuv420p10le",
"crf": 22,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "video_00087.mp4",
"subfolder": "AnimateAnyone",
"type": "output",
"format": "video/h265-mp4"
}
}
}
"widgets_values": [
999999999,
"fixed",
20,
3.5,
1,
24,
1,
4,
1,
1,
"DDIM",
0.00085,
0.012,
"linear",
"v_prediction",
"trailing",
1,
false,
true,
false,
"cammy_white_v1.safetensors"
]
},
{
"id": 11,
"type": "[ComfyUI-3D] Load UNet2D ConditionModel",
"id": 33,
"type": "[AnimateAnyone] Load UNet2D ConditionModel",
"pos": [
1435,
195
1488,
202
],
"size": {
"0": 407.1247863769531,
"1": 86.651123046875
"0": 420.5655212402344,
"1": 82
},
"flags": {},
"order": 6,
@@ -485,14 +469,14 @@
"name": "unet2d",
"type": "UNET2D",
"links": [
51
58
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[ComfyUI-3D] Load UNet2D ConditionModel"
"Node name for S&R": "[AnimateAnyone] Load UNet2D ConditionModel"
},
"widgets_values": [
"./pretrained_weights/stable-diffusion-v1-5/unet/",
@@ -500,15 +484,15 @@
]
},
{
"id": 12,
"type": "[ComfyUI-3D] Load UNet3D ConditionModel",
"id": 34,
"type": "[AnimateAnyone] Load UNet3D ConditionModel",
"pos": [
1471,
358
1525,
343
],
"size": {
"0": 409.384033203125,
"1": 106
"0": 414.0246887207031,
"1": 111.3802261352539
},
"flags": {},
"order": 7,
@@ -518,14 +502,14 @@
"name": "unet3d",
"type": "UNET3D",
"links": [
52
59
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[ComfyUI-3D] Load UNet3D ConditionModel"
"Node name for S&R": "[AnimateAnyone] Load UNet3D ConditionModel"
},
"widgets_values": [
"./pretrained_weights/stable-diffusion-v1-5/unet/",
@@ -534,59 +518,52 @@
]
},
{
"id": 2,
"type": "VAEEncode",
"id": 36,
"type": "[AnimateAnyone] Pose Guider Encode",
"pos": [
1542,
88
1701,
702
],
"size": {
"0": 210,
"0": 292.20001220703125,
"1": 46
},
"flags": {},
"order": 12,
"order": 14,
"mode": 0,
"inputs": [
{
"name": "pixels",
"type": "IMAGE",
"link": 24
"name": "pose_guider",
"type": "POSE_GUIDER",
"link": 61
},
{
"name": "vae",
"type": "VAE",
"link": 1
"name": "pose_images",
"type": "IMAGE",
"link": 62
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"name": "pose_latent",
"type": "POSE_LATENT",
"links": [
53
63
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEEncode",
"ttNbgOverride": {
"color": "#223",
"bgcolor": "#335",
"groupcolor": "#88A"
}
},
"color": "#223",
"bgcolor": "#335"
"Node name for S&R": "[AnimateAnyone] Pose Guider Encode"
}
},
{
"id": 5,
"type": "CLIPVisionEncode",
"pos": [
1544,
592
1584,
588
],
"size": {
"0": 380.4000244140625,
@@ -612,7 +589,7 @@
"name": "CLIP_VISION_OUTPUT",
"type": "CLIP_VISION_OUTPUT",
"links": [
54
64
],
"shape": 3,
"slot_index": 0
@@ -623,124 +600,88 @@
}
},
{
"id": 8,
"type": "[ComfyUI-3D] Pose Guider Encode",
"id": 2,
"type": "VAEEncode",
"pos": [
1668,
704
1584,
85
],
"size": {
"0": 292.20001220703125,
"0": 210,
"1": 46
},
"flags": {},
"order": 14,
"order": 12,
"mode": 0,
"inputs": [
{
"name": "pose_guider",
"type": "POSE_GUIDER",
"link": 5
"name": "pixels",
"type": "IMAGE",
"link": 24
},
{
"name": "pose_images",
"type": "IMAGE",
"link": 27
"name": "vae",
"type": "VAE",
"link": 1
}
],
"outputs": [
{
"name": "pose_latent",
"type": "POSE_LATENT",
"name": "LATENT",
"type": "LATENT",
"links": [
55
57
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[ComfyUI-3D] Pose Guider Encode"
}
"Node name for S&R": "VAEEncode",
"ttNbgOverride": {
"color": "#223",
"bgcolor": "#335",
"groupcolor": "#88A"
}
},
"color": "#223",
"bgcolor": "#335"
},
{
"id": 31,
"type": "[ComfyUI-3D] Animate Anyone Sampler",
"id": 15,
"type": "Reroute",
"pos": [
2091,
47
1583,
-110
],
"size": [
75,
26
],
"size": {
"0": 315,
"1": 618
},
"flags": {},
"order": 15,
"order": 10,
"mode": 0,
"inputs": [
{
"name": "reference_unet",
"type": "UNET2D",
"link": 51
},
{
"name": "denoising_unet",
"type": "UNET3D",
"link": 52
},
{
"name": "ref_image_latent",
"type": "LATENT",
"link": 53
},
{
"name": "clip_image_embeds",
"type": "CLIP_VISION_OUTPUT",
"link": 54
},
{
"name": "pose_latent",
"type": "POSE_LATENT",
"link": 55
"name": "",
"type": "*",
"link": 14
}
],
"outputs": [
{
"name": "latent",
"type": "LATENT",
"name": "",
"type": "VAE",
"links": [
56
15
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[ComfyUI-3D] Animate Anyone Sampler"
},
"widgets_values": [
999999999,
"fixed",
20,
3.5,
1,
24,
1,
4,
1,
1,
"DDIM",
0.00085,
0.012,
"linear",
"v_prediction",
"trailing",
1,
false,
true,
false,
"cammy_white_v1.safetensors"
]
"showOutputText": false,
"horizontal": false
}
},
{
"id": 10,
@@ -788,7 +729,7 @@
"force_size": "Disabled",
"custom_width": 512,
"custom_height": 512,
"frame_load_cap": 96,
"frame_load_cap": 24,
"skip_first_frames": 0,
"select_every_nth": 1,
"choose video to upload": "image",
@@ -796,7 +737,7 @@
"hidden": false,
"paused": false,
"params": {
"frame_load_cap": 96,
"frame_load_cap": 24,
"skip_first_frames": 0,
"force_rate": 0,
"filename": "anyone-video-2_kps.mp4",
@@ -806,6 +747,65 @@
}
}
}
},
{
"id": 16,
"type": "VHS_VideoCombine",
"pos": [
2986,
-130
],
"size": [
315,
701.5
],
"flags": {},
"order": 17,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 16
},
{
"name": "audio",
"type": "VHS_AUDIO",
"link": null
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 12,
"loop_count": 0,
"filename_prefix": "AnimateAnyone",
"format": "video/h265-mp4",
"pix_fmt": "yuv420p10le",
"crf": 22,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "AnimateAnyone_00001.mp4",
"subfolder": "",
"type": "output",
"format": "video/h265-mp4"
}
}
}
}
],
"links": [
@@ -825,14 +825,6 @@
0,
"CLIP_VISION"
],
[
5,
7,
0,
8,
0,
"POSE_GUIDER"
],
[
14,
1,
@@ -881,14 +873,6 @@
0,
"IMAGE"
],
[
27,
23,
0,
8,
1,
"IMAGE"
],
[
28,
10,
@@ -930,52 +914,68 @@
"INT"
],
[
51,
11,
57,
2,
0,
31,
32,
2,
"LATENT"
],
[
58,
33,
0,
32,
0,
"UNET2D"
],
[
52,
12,
59,
34,
0,
31,
32,
1,
"UNET3D"
],
[
53,
2,
0,
31,
2,
"LATENT"
],
[
54,
5,
0,
31,
3,
"CLIP_VISION_OUTPUT"
],
[
55,
8,
0,
31,
4,
"POSE_LATENT"
],
[
56,
31,
60,
32,
0,
14,
0,
"LATENT"
],
[
61,
35,
0,
36,
0,
"POSE_GUIDER"
],
[
62,
23,
0,
36,
1,
"IMAGE"
],
[
63,
36,
0,
32,
4,
"POSE_LATENT"
],
[
64,
5,
0,
32,
3,
"CLIP_VISION_OUTPUT"
]
],
"groups": [],
_test.py
-38
View File
@@ -1,38 +0,0 @@
import time
import torch
from diffusers import (StableDiffusionPipeline,
EulerAncestralDiscreteScheduler)
from sfast.compilers.diffusion_pipeline_compiler import (compile,
CompilationConfig)
def load_model():
model = StableDiffusionPipeline.from_pretrained(
'runwayml/stable-diffusion-v1-5',
torch_dtype=torch.float16)
model.scheduler = EulerAncestralDiscreteScheduler.from_config(
model.scheduler.config)
model.safety_checker = None
model.to(torch.device('cuda'))
return model
model = load_model()
config = CompilationConfig.Default()
# xformers and Triton are suggested for achieving best performance.
try:
import xformers
config.enable_xformers = True
except ImportError:
print('xformers not installed, skip')
try:
import triton
config.enable_triton = True
except ImportError:
print('Triton not installed, skip')
# CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead.
# But it can increase the amount of GPU memory used.
# For StableVideoDiffusionPipeline it is not needed.
config.enable_cuda_graph = True
model = compile(model, config)
src/models/main_diffuser.py
+2 -23
View File
@@ -7,7 +7,7 @@ from typing import Callable, List, Optional, Union
import comfy.utils
import torch
from diffusers import DiffusionPipeline, StableDiffusionPipeline
from diffusers import DiffusionPipeline
from diffusers.loaders import LoraLoaderMixin
from diffusers.schedulers import (
DDIMScheduler,
@@ -19,7 +19,6 @@ from diffusers.schedulers import (
PNDMScheduler,
)
from diffusers.utils.torch_utils import randn_tensor
from sfast.compilers.diffusion_pipeline_compiler import compile_unet, CompilationConfig
from .mutual_self_attention import ReferenceAttentionControl
from ..utils.util import get_tensor_interpolation_method, get_context_scheduler
@@ -64,7 +63,7 @@ class AADiffusion(DiffusionPipeline, LoraLoaderMixin):
self.load_lora_into_unet(
state_dict,
network_alphas=network_alphas,
unet=getattr(self, self.denoising_unet_name) if not hasattr(self, "unet") else self.denoising_unet,
unet=self.unet,
low_cpu_mem_usage=low_cpu_mem_usage,
adapter_name=adapter_name,
_pipeline=self,
@@ -321,26 +320,6 @@ class AADiffusion(DiffusionPipeline, LoraLoaderMixin):
)
reference_control_reader.update(reference_control_writer)
config = CompilationConfig.Default()
# xformers and Triton are suggested for achieving best performance.
try:
import xformers
config.enable_xformers = True
except ImportError:
print('xformers not installed, skip')
try:
import triton
config.enable_triton = True
except ImportError:
print('Triton not installed, skip')
# CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead.
# But it can increase the amount of GPU memory used.
# For StableVideoDiffusionPipeline it is not needed.
config.enable_cuda_graph = True
self.unet = compile_unet(self.unet, config)
# Prepare timesteps
self.scheduler.set_timesteps(loop_steps, device=device)
timesteps = self.scheduler.timesteps
src/models/unet_2d_blocks.py
+1 -1
View File
@@ -6,7 +6,7 @@ import torch
import torch.nn.functional as F
from diffusers.models.activations import get_activation
from diffusers.models.attention_processor import Attention
from diffusers.models.dual_transformer_2d import DualTransformer2DModel
from diffusers.models.transformers.dual_transformer_2d import DualTransformer2DModel
from diffusers.models.resnet import Downsample2D, ResnetBlock2D, Upsample2D
from diffusers.utils import is_torch_version, logging
from diffusers.utils.torch_utils import apply_freeu