nodes/cubiq-ComfyUI_IPAdapter_plus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

attention masking

Browse Source
This commit is contained in:
matt3o
2023-11-08 20:54:08 +01:00
parent 4fefd8b9e5
commit bf0508bd59
5 changed files with 1862 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
IPAdapterPlus.py
+31 -6
View File
@@ -214,14 +214,16 @@ class CrossAttentionPatch:
def __call__(self, n, context_attn2, value_attn2, extra_options):
org_dtype = n.dtype
cond_or_uncond = extra_options["cond_or_uncond"] if "cond_or_uncond" in extra_options else None
cond_or_uncond = extra_options["cond_or_uncond"]
with torch.autocast(device_type=self.device, dtype=self.dtype):
q = n
k = context_attn2
v = value_attn2
b = q.shape[0]
qs = q.shape[1]
batch_prompt = b // len(cond_or_uncond)
out = optimized_attention(q, k, v, extra_options["n_heads"])
_, _, lh, lw = extra_options["original_shape"]
k = v = []
for weight, cond, uncond, ipadapter, mask, weight_type in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_type):
@@ -242,16 +244,29 @@ class CrossAttentionPatch:
# please read licensing notes https://github.com/lllyasviel/Fooocus/blob/main/fooocus_extras/ip_adapter.py#L225
ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True)
ip_v_offset = ip_v - ip_v_mean
B, F, C = ip_k.shape
_, _, C = ip_k.shape
channel_penalty = float(C) / 1280.0
W = weight * channel_penalty
ip_k = ip_k * W
ip_v = ip_v_offset + ip_v_mean * W
out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"])
out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"])
if weight_type.startswith("original"):
out_ip = out_ip * weight
if mask is not None:
# TODO: needs testing
for rate in [1, 2, 4, 8]:
mask_h = -(-lh//rate) # fancy ceil
mask_w = -(-lw//rate)
if mask_h*mask_w == qs:
break
mask_downsample = F.interpolate(mask.unsqueeze(0).unsqueeze(0), size=(mask_h, mask_w), mode="bilinear").squeeze(0)
mask_downsample = mask_downsample.view(1, -1, 1).repeat(out.shape[0], 1, out.shape[2])
out_ip = out_ip * mask_downsample
out = out + out_ip
return out.to(dtype=org_dtype)
@@ -303,13 +318,16 @@ class IPAdapterApply:
"noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }),
"weight_type": (["original", "linear", "channel penalty"], ),
},
"optional": {
"attn_mask": ("MASK",),
}
}
RETURN_TYPES = ("MODEL",)
FUNCTION = "apply_ipadapter"
CATEGORY = "ipadapter"
def apply_ipadapter(self, ipadapter, model, weight, clip_vision=None, image=None, weight_type="original", noise=None, embeds=None):
def apply_ipadapter(self, ipadapter, model, weight, clip_vision=None, image=None, weight_type="original", noise=None, embeds=None, attn_mask=None):
self.dtype = model.model.diffusion_model.dtype
self.device = comfy.model_management.get_torch_device()
self.weight = weight
@@ -364,6 +382,9 @@ class IPAdapterApply:
work_model = model.clone()
if attn_mask is not None:
attn_mask = attn_mask.squeeze().to(self.device)
patch_kwargs = {
"number": 0,
"weight": self.weight,
@@ -371,7 +392,8 @@ class IPAdapterApply:
"dtype": self.dtype,
"cond": image_prompt_embeds,
"uncond": uncond_image_prompt_embeds,
"weight_type": weight_type
"weight_type": weight_type,
"mask": attn_mask
}
if not self.is_sdxl:
@@ -541,6 +563,9 @@ class IPAdapterApplyEncoded(IPAdapterApply):
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
"weight_type": (["original", "linear", "channel penalty"], ),
},
"optional": {
"attn_mask": ("MASK",),
}
}
class IPAdapterSaveEmbeds:
README.md
+15 -1
View File
@@ -5,6 +5,8 @@ IPAdapter implementation that follows the ComfyUI way of doing things. The code
## Important updates
**2023/11/08**: Added attention masking.
**2023/11/07**: Added three ways to apply the weight. [See below](#weight-types) for more info. **This might break things!** Please let me know if you are having issues. When loading an old workflow try to reload the page a couple of times or delete the `IPAdapter Apply` node and insert a new one.
**2023/11/02**: Added compatibility with the new models in safetensors format (available on [huggingface](https://huggingface.co/h94/IP-Adapter)).
@@ -139,7 +141,7 @@ You can choose how the IPAdapter weight is applied to the image embeds. Options
- **original**: The weight is applied to the aggregated tensors. The weight works predictably for values greater and lower than 1.
- **linear**: The weight is applied to the individual tensors before aggretating them. Compared to `original` the influence is weaker when weight is <1 and stronger when >1. **Note:** at weight `1` the two methods are equivalent.
- **channel penalty**: This method is developed by Lvmin Zhang at Stanford University and implemented in the Fooocus UI. Results are sometimes sharper but the image burns a little. It works very well also when weight is >1.
- **channel penalty**: This method is a modified version of Lvmin Zhang's (Fooocus). Results are sometimes sharper. It works very well also when weight is >1. Still experimental, may change in the future.
The image below shows the difference (zoom in).
@@ -149,6 +151,18 @@ In the examples directory you can find [a workflow](examples/IPAdapter_weight_ty
**Note:** I'm not still sure whether all methods will stay. `Linear` seems the most sensible but I wanted to keep the `original` for backward compatibility. `channel penalty` has a weird non-commercial clause but it's still part of a GNU GPLv3 software (ie: there's a licensing clash) so I'm trying to understand how to deal with that.
### Attention masking
It's possible to add a mask to define the area where the IPAdapter will be applied to. Everything outside the mask will ignore the reference images and will only listen to the text prompt.
It is suggested to use a mask of the same size of the final generated image.
In the picture below I use two reference images masked one on the left and the other on the right. The image is generated only with IPAdapter and one ksampler (without in/outpainting or area conditioning).
<img src="./examples/masking.jpg" width="512" alt="masking" />
In the examples directory you'll find a couple of masking workflows: [simple](examples/IPAdapter_mask.json) and [two masks](examples/IPAdapter_2_masks.json).
## Troubleshooting
**Error: 'CLIPVisionModelOutput' object has no attribute 'penultimate_hidden_states'**
examples/IPAdapter_2_masks.json
+1151
View File
File diff suppressed because it is too large Load Diff
examples/IPAdapter_mask.json
+665
View File
@@ -0,0 +1,665 @@
{
"last_node_id": 51,
"last_link_id": 96,
"nodes": [
{
"id": 5,
"type": "CLIPTextEncode",
"pos": [
550,
510
],
"size": {
"0": 400,
"1": 160
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 6
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
3
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"blurry, malformed, video game, rendering, naked, cleavage, horror, zombie, text, watermark"
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 6,
"type": "VAEDecode",
"pos": [
1360,
270
],
"size": {
"0": 140,
"1": 50
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7
},
{
"name": "vae",
"type": "VAE",
"link": 8
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
9
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
}
},
{
"id": 7,
"type": "VAELoader",
"pos": [
1090,
570
],
"size": {
"0": 240,
"1": 60
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
8
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"vae-ft-mse-840000-ema-pruned.safetensors"
]
},
{
"id": 2,
"type": "CheckpointLoaderSimple",
"pos": [
210,
410
],
"size": {
"0": 290,
"1": 100
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
88
],
"slot_index": 0
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
5,
6
],
"slot_index": 1
},
{
"name": "VAE",
"type": "VAE",
"links": [],
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "CheckpointLoaderSimple"
},
"widgets_values": [
"sd15/dreamshaper_8.safetensors"
]
},
{
"id": 11,
"type": "CLIPVisionLoader",
"pos": [
490,
20
],
"size": {
"0": 300,
"1": 60
},
"flags": {},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "CLIP_VISION",
"type": "CLIP_VISION",
"links": [
86
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPVisionLoader"
},
"widgets_values": [
"IPAdapter_image_encoder_sd15.safetensors"
],
"color": "#223",
"bgcolor": "#335"
},
{
"id": 13,
"type": "PrepImageForClipVision",
"pos": [
540,
130
],
"size": {
"0": 243.60000610351562,
"1": 110
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "IMAGE",
"link": 32
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
87
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "PrepImageForClipVision"
},
"widgets_values": [
"LANCZOS",
"center",
0
],
"color": "#223",
"bgcolor": "#335"
},
{
"id": 4,
"type": "CLIPTextEncode",
"pos": [
550,
310
],
"size": {
"0": 400,
"1": 160
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 5
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
2
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"photography of a warrior woman in a cherry blossom forest, high quality, highly detailed"
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 3,
"type": "EmptyLatentImage",
"pos": [
740,
720
],
"size": {
"0": 210,
"1": 110
},
"flags": {},
"order": 3,
"mode": 0,
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
4
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "EmptyLatentImage"
},
"widgets_values": [
768,
512,
1
]
},
{
"id": 1,
"type": "KSampler",
"pos": [
1090,
270
],
"size": {
"0": 240,
"1": 262
},
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 93
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 2
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 3
},
{
"name": "latent_image",
"type": "LATENT",
"link": 4
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
40,
"fixed",
30,
5,
"ddim",
"ddim_uniform",
1
]
},
{
"id": 10,
"type": "IPAdapterModelLoader",
"pos": [
490,
-80
],
"size": {
"0": 300,
"1": 60
},
"flags": {},
"order": 4,
"mode": 0,
"outputs": [
{
"name": "IPADAPTER",
"type": "IPADAPTER",
"links": [
85
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "IPAdapterModelLoader"
},
"widgets_values": [
"ip-adapter-plus_sd15.bin"
],
"color": "#223",
"bgcolor": "#335"
},
{
"id": 12,
"type": "LoadImage",
"pos": [
230,
-80
],
"size": [
230,
320
],
"flags": {},
"order": 5,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
32
],
"shape": 3,
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": [
96
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"clipspace/clipspace-mask-790059.png [input]",
"image"
],
"color": "#223",
"bgcolor": "#335"
},
{
"id": 51,
"type": "IPAdapterApply",
"pos": [
830,
40
],
"size": [
210,
190
],
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "ipadapter",
"type": "IPADAPTER",
"link": 85
},
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 86
},
{
"name": "image",
"type": "IMAGE",
"link": 87
},
{
"name": "model",
"type": "MODEL",
"link": 88
},
{
"name": "attn_mask",
"type": "MASK",
"link": 96
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
93
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "IPAdapterApply"
},
"widgets_values": [
0.7000000000000001,
0,
"linear"
],
"color": "#223",
"bgcolor": "#335"
},
{
"id": 8,
"type": "PreviewImage",
"pos": [
1365,
369
],
"size": [
675.9466064453127,
480.1444152832032
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 9
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
}
],
"links": [
[
2,
4,
0,
1,
1,
"CONDITIONING"
],
[
3,
5,
0,
1,
2,
"CONDITIONING"
],
[
4,
3,
0,
1,
3,
"LATENT"
],
[
5,
2,
1,
4,
0,
"CLIP"
],
[
6,
2,
1,
5,
0,
"CLIP"
],
[
7,
1,
0,
6,
0,
"LATENT"
],
[
8,
7,
0,
6,
1,
"VAE"
],
[
9,
6,
0,
8,
0,
"IMAGE"
],
[
32,
12,
0,
13,
0,
"IMAGE"
],
[
85,
10,
0,
51,
0,
"IPADAPTER"
],
[
86,
11,
0,
51,
1,
"CLIP_VISION"
],
[
87,
13,
0,
51,
2,
"IMAGE"
],
[
88,
2,
0,
51,
3,
"MODEL"
],
[
93,
51,
0,
1,
0,
"MODEL"
],
[
96,
12,
1,
51,
4,
"MASK"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}
examples/masking.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 116 KiB