nodes/cubiq-ComfyUI_IPAdapter_plus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add unfold batch option

Browse Source
This commit is contained in:
matt3o
2023-11-29 18:51:24 +01:00
parent 6bb44bb0ab
commit 77554c835d
2 changed files with 54 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
IPAdapterPlus.py
+51 -9
View File
@@ -210,7 +210,7 @@ class IPAdapter(nn.Module):
class CrossAttentionPatch:
# forward for patching
def __init__(self, weight, ipadapter, dtype, number, cond, uncond, weight_type, mask=None, sigma_start=0.0, sigma_end=1.0):
def __init__(self, weight, ipadapter, dtype, number, cond, uncond, weight_type, mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False):
self.weights = [weight]
self.ipadapters = [ipadapter]
self.conds = [cond]
@@ -222,11 +222,12 @@ class CrossAttentionPatch:
self.masks = [mask]
self.sigma_start = [sigma_start]
self.sigma_end = [sigma_end]
self.unfold_batch = [unfold_batch]
self.k_key = str(self.number*2+1) + "_to_k_ip"
self.v_key = str(self.number*2+1) + "_to_v_ip"
def set_new_condition(self, weight, ipadapter, cond, uncond, dtype, number, weight_type, mask=None, sigma_start=0.0, sigma_end=1.0):
def set_new_condition(self, weight, ipadapter, cond, uncond, dtype, number, weight_type, mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False):
self.weights.append(weight)
self.ipadapters.append(ipadapter)
self.conds.append(cond)
@@ -237,6 +238,7 @@ class CrossAttentionPatch:
self.device = 'cuda'
self.sigma_start.append(sigma_start)
self.sigma_end.append(sigma_end)
self.unfold_batch.append(unfold_batch)
def __call__(self, n, context_attn2, value_attn2, extra_options):
org_dtype = n.dtype
@@ -252,15 +254,53 @@ class CrossAttentionPatch:
batch_prompt = b // len(cond_or_uncond)
out = optimized_attention(q, k, v, extra_options["n_heads"])
_, _, lh, lw = extra_options["original_shape"]
# extra options for AnimateDiff
ad_params = extra_options['ad_params'] if "ad_params" in extra_options else None
for weight, cond, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_type, self.sigma_start, self.sigma_end):
for weight, cond, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_type, self.sigma_start, self.sigma_end, self.unfold_batch):
if sigma > sigma_start or sigma < sigma_end:
continue
k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond).repeat(batch_prompt, 1, 1)
k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond).repeat(batch_prompt, 1, 1)
v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond).repeat(batch_prompt, 1, 1)
v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond).repeat(batch_prompt, 1, 1)
if unfold_batch and cond.shape[0] > 1:
# Check AnimateDiff context window
if ad_params is not None and ad_params["sub_idxs"] is not None:
# if images length matches or exceeds full_length get sub_idx images
if cond.shape[0] >= ad_params["full_length"]:
cond = torch.Tensor(cond[ad_params["sub_idxs"]])
uncond = torch.Tensor(uncond[ad_params["sub_idxs"]])
# otherwise, need to do more to get proper sub_idxs masks
else:
# check if images length matches full_length - if not, make it match
if cond.shape[0] < ad_params["full_length"]:
cond = torch.cat((cond, cond[-1:].repeat((ad_params["full_length"]-cond.shape[0], 1, 1))), dim=0)
uncond = torch.cat((uncond, uncond[-1:].repeat((ad_params["full_length"]-uncond.shape[0], 1, 1))), dim=0)
# if we have too many remove the excess (should not happen, but just in case)
if cond.shape[0] > ad_params["full_length"]:
cond = cond[:ad_params["full_length"]]
uncond = uncond[:ad_params["full_length"]]
cond = cond[ad_params["sub_idxs"]]
uncond = uncond[ad_params["sub_idxs"]]
# if we don't have enough reference images repeat the last one until we reach the right size
if cond.shape[0] < batch_prompt:
cond = torch.cat((cond, cond[-1:].repeat((batch_prompt-cond.shape[0], 1, 1))), dim=0)
uncond = torch.cat((uncond, uncond[-1:].repeat((batch_prompt-uncond.shape[0], 1, 1))), dim=0)
# if we have too many remove the exceeding
elif cond.shape[0] > batch_prompt:
cond = cond[:batch_prompt]
uncond = uncond[:batch_prompt]
k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond)
k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond)
v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond)
v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond)
else:
k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond).repeat(batch_prompt, 1, 1)
k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond).repeat(batch_prompt, 1, 1)
v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond).repeat(batch_prompt, 1, 1)
v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond).repeat(batch_prompt, 1, 1)
if weight_type.startswith("linear"):
ip_k = torch.cat([(k_cond, k_uncond)[i] for i in cond_or_uncond], dim=0) * weight
ip_v = torch.cat([(v_cond, v_uncond)[i] for i in cond_or_uncond], dim=0) * weight
@@ -293,7 +333,6 @@ class CrossAttentionPatch:
break
# check if using AnimateDiff and sliding context window
ad_params = extra_options['ad_params'] if "ad_params" in extra_options else None
if (mask.shape[0] > 1 and ad_params is not None and ad_params["sub_idxs"] is not None):
# if mask length matches or exceeds full_length, just get sub_idx masks, resize, and continue
if mask.shape[0] >= ad_params["full_length"]:
@@ -375,6 +414,7 @@ class IPAdapterApply:
"weight_type": (["original", "linear", "channel penalty"], ),
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
"unfold_batch": ("BOOLEAN", { "default": False }),
},
"optional": {
"attn_mask": ("MASK",),
@@ -385,7 +425,7 @@ class IPAdapterApply:
FUNCTION = "apply_ipadapter"
CATEGORY = "ipadapter"
def apply_ipadapter(self, ipadapter, model, weight, clip_vision=None, image=None, weight_type="original", noise=None, embeds=None, attn_mask=None, start_at=0.0, end_at=1.0):
def apply_ipadapter(self, ipadapter, model, weight, clip_vision=None, image=None, weight_type="original", noise=None, embeds=None, attn_mask=None, start_at=0.0, end_at=1.0, unfold_batch=False):
self.dtype = model.model.diffusion_model.dtype
self.device = comfy.model_management.get_torch_device()
self.weight = weight
@@ -459,6 +499,7 @@ class IPAdapterApply:
"mask": attn_mask,
"sigma_start": sigma_start,
"sigma_end": sigma_end,
"unfold_batch": unfold_batch,
}
if not self.is_sdxl:
@@ -636,6 +677,7 @@ class IPAdapterApplyEncoded(IPAdapterApply):
"weight_type": (["original", "linear", "channel penalty"], ),
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
"unfold_batch": ("BOOLEAN", { "default": False }),
},
"optional": {
"attn_mask": ("MASK",),
README.md
+3 -11
View File
@@ -5,6 +5,8 @@ IPAdapter implementation that follows the ComfyUI way of doing things. The code
## Important updates
**2023/11/29**: Added `unfold_batch` option to send the reference images sequentially to a latent batch. Useful for animations.
**2023/11/26**: Added [timestepping](#timestepping). You may need to delete the old nodes and recreate them. **Important:** For this to work you need to update ComfyUI to the latest version.
**2023/11/24**: Support for multiple attention masks.
@@ -183,17 +185,7 @@ In the `Apply IPAdapter` node you can set a start and an end point. The IPAdapte
## Troubleshooting
**Error: 'CLIPVisionModelOutput' object has no attribute 'penultimate_hidden_states'** and **AttributeError: 'NoneType' object has no attribute 'encode_image'**
You are using an old version of ComfyUI or of this extension. Update and you'll be fine. **Please note** that on Windows for a full update you might need to re-download the latest standalone version. Also sometimes the Manager fails to update even if everything seems fine. In that case you need to update manually or remove the extension and re-install it.
**size mismatch for proj_in.weight: copying a param with shape torch.Size([..., ...]) from checkpoint, the shape in current model is torch.Size([..., ...])**
You are using the wrong image encoder+IPAdapter Model+Checkpoint combo. Remember that you need to select the CLIP encoder v1.5 for all v1.5 IPAdapter models AND for all models ending with `vit-h` (even if they are for SDXL).
**Is it true that the input reference image must have the same size of the output image?**
No, that's a metropolitan legend. Your input and output images can be of any size. Remember that all input images are scaled and cropped to 224x224 anyway.
Please check the [troubleshooting](https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/108) before posting a new issue.
## Diffusers version