diff --git a/IPAdapterPlus.py b/IPAdapterPlus.py index 63ba4aa..03a26c8 100644 --- a/IPAdapterPlus.py +++ b/IPAdapterPlus.py @@ -214,14 +214,16 @@ class CrossAttentionPatch: def __call__(self, n, context_attn2, value_attn2, extra_options): org_dtype = n.dtype - cond_or_uncond = extra_options["cond_or_uncond"] if "cond_or_uncond" in extra_options else None + cond_or_uncond = extra_options["cond_or_uncond"] with torch.autocast(device_type=self.device, dtype=self.dtype): q = n k = context_attn2 v = value_attn2 b = q.shape[0] + qs = q.shape[1] batch_prompt = b // len(cond_or_uncond) out = optimized_attention(q, k, v, extra_options["n_heads"]) + _, _, lh, lw = extra_options["original_shape"] k = v = [] for weight, cond, uncond, ipadapter, mask, weight_type in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_type): @@ -242,16 +244,29 @@ class CrossAttentionPatch: # please read licensing notes https://github.com/lllyasviel/Fooocus/blob/main/fooocus_extras/ip_adapter.py#L225 ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True) ip_v_offset = ip_v - ip_v_mean - B, F, C = ip_k.shape + _, _, C = ip_k.shape channel_penalty = float(C) / 1280.0 W = weight * channel_penalty ip_k = ip_k * W ip_v = ip_v_offset + ip_v_mean * W - out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) if weight_type.startswith("original"): out_ip = out_ip * weight - + + if mask is not None: + # TODO: needs testing + for rate in [1, 2, 4, 8]: + mask_h = -(-lh//rate) # fancy ceil + mask_w = -(-lw//rate) + + if mask_h*mask_w == qs: + break + + mask_downsample = F.interpolate(mask.unsqueeze(0).unsqueeze(0), size=(mask_h, mask_w), mode="bilinear").squeeze(0) + mask_downsample = mask_downsample.view(1, -1, 1).repeat(out.shape[0], 1, out.shape[2]) + out_ip = out_ip * mask_downsample + out = out + out_ip return out.to(dtype=org_dtype) @@ -303,13 +318,16 @@ class IPAdapterApply: "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), "weight_type": (["original", "linear", "channel penalty"], ), }, + "optional": { + "attn_mask": ("MASK",), + } } RETURN_TYPES = ("MODEL",) FUNCTION = "apply_ipadapter" CATEGORY = "ipadapter" - def apply_ipadapter(self, ipadapter, model, weight, clip_vision=None, image=None, weight_type="original", noise=None, embeds=None): + def apply_ipadapter(self, ipadapter, model, weight, clip_vision=None, image=None, weight_type="original", noise=None, embeds=None, attn_mask=None): self.dtype = model.model.diffusion_model.dtype self.device = comfy.model_management.get_torch_device() self.weight = weight @@ -364,6 +382,9 @@ class IPAdapterApply: work_model = model.clone() + if attn_mask is not None: + attn_mask = attn_mask.squeeze().to(self.device) + patch_kwargs = { "number": 0, "weight": self.weight, @@ -371,7 +392,8 @@ class IPAdapterApply: "dtype": self.dtype, "cond": image_prompt_embeds, "uncond": uncond_image_prompt_embeds, - "weight_type": weight_type + "weight_type": weight_type, + "mask": attn_mask } if not self.is_sdxl: @@ -541,6 +563,9 @@ class IPAdapterApplyEncoded(IPAdapterApply): "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), "weight_type": (["original", "linear", "channel penalty"], ), }, + "optional": { + "attn_mask": ("MASK",), + } } class IPAdapterSaveEmbeds: diff --git a/README.md b/README.md index 35baae7..d2a6311 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ IPAdapter implementation that follows the ComfyUI way of doing things. The code ## Important updates +**2023/11/08**: Added attention masking. + **2023/11/07**: Added three ways to apply the weight. [See below](#weight-types) for more info. **This might break things!** Please let me know if you are having issues. When loading an old workflow try to reload the page a couple of times or delete the `IPAdapter Apply` node and insert a new one. **2023/11/02**: Added compatibility with the new models in safetensors format (available on [huggingface](https://huggingface.co/h94/IP-Adapter)). @@ -139,7 +141,7 @@ You can choose how the IPAdapter weight is applied to the image embeds. Options - **original**: The weight is applied to the aggregated tensors. The weight works predictably for values greater and lower than 1. - **linear**: The weight is applied to the individual tensors before aggretating them. Compared to `original` the influence is weaker when weight is <1 and stronger when >1. **Note:** at weight `1` the two methods are equivalent. -- **channel penalty**: This method is developed by Lvmin Zhang at Stanford University and implemented in the Fooocus UI. Results are sometimes sharper but the image burns a little. It works very well also when weight is >1. +- **channel penalty**: This method is a modified version of Lvmin Zhang's (Fooocus). Results are sometimes sharper. It works very well also when weight is >1. Still experimental, may change in the future. The image below shows the difference (zoom in). @@ -149,6 +151,18 @@ In the examples directory you can find [a workflow](examples/IPAdapter_weight_ty **Note:** I'm not still sure whether all methods will stay. `Linear` seems the most sensible but I wanted to keep the `original` for backward compatibility. `channel penalty` has a weird non-commercial clause but it's still part of a GNU GPLv3 software (ie: there's a licensing clash) so I'm trying to understand how to deal with that. +### Attention masking + +It's possible to add a mask to define the area where the IPAdapter will be applied to. Everything outside the mask will ignore the reference images and will only listen to the text prompt. + +It is suggested to use a mask of the same size of the final generated image. + +In the picture below I use two reference images masked one on the left and the other on the right. The image is generated only with IPAdapter and one ksampler (without in/outpainting or area conditioning). + +masking + +In the examples directory you'll find a couple of masking workflows: [simple](examples/IPAdapter_mask.json) and [two masks](examples/IPAdapter_2_masks.json). + ## Troubleshooting **Error: 'CLIPVisionModelOutput' object has no attribute 'penultimate_hidden_states'** diff --git a/examples/IPAdapter_2_masks.json b/examples/IPAdapter_2_masks.json new file mode 100644 index 0000000..d150ecd --- /dev/null +++ b/examples/IPAdapter_2_masks.json @@ -0,0 +1,1151 @@ +{ + "last_node_id": 51, + "last_link_id": 90, + "nodes": [ + { + "id": 5, + "type": "CLIPTextEncode", + "pos": [ + 550, + 510 + ], + "size": { + "0": 400, + "1": 160 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 6 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 3 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, malformed, illustration, video game, rendering, naked, cleavage, horror, zombie" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 6, + "type": "VAEDecode", + "pos": [ + 1410, + 280 + ], + "size": { + "0": 140, + "1": 50 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 7, + "type": "VAELoader", + "pos": [ + 1140, + 580 + ], + "size": { + "0": 240, + "1": 60 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "vae-ft-mse-840000-ema-pruned.safetensors" + ] + }, + { + "id": 2, + "type": "CheckpointLoaderSimple", + "pos": [ + 210, + 410 + ], + "size": { + "0": 290, + "1": 100 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 88 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 5, + 6 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/dreamshaper_8.safetensors" + ] + }, + { + "id": 3, + "type": "EmptyLatentImage", + "pos": [ + 740, + 710 + ], + "size": { + "0": 210, + "1": 110 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 768, + 512, + 1 + ] + }, + { + "id": 34, + "type": "SolidMask", + "pos": [ + -170, + -30 + ], + "size": { + "0": 210, + "1": 106 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 51, + 57 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 768, + 512 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 37, + "type": "SolidMask", + "pos": [ + 870, + 90 + ], + "size": { + "0": 210, + "1": 106 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 75 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 384, + 512 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 120, + -390 + ], + "size": [ + 230, + 320 + ], + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 32 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "woman.png", + "image" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 38, + "type": "MaskComposite", + "pos": [ + 1120, + -100 + ], + "size": { + "0": 210, + "1": 126 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "destination", + "type": "MASK", + "link": 57 + }, + { + "name": "source", + "type": "MASK", + "link": 76 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 83 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 384, + 0, + "add" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 48, + "type": "FeatherMask", + "pos": [ + 1120, + 70 + ], + "size": { + "0": 210, + "1": 130 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 75 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 76 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FeatherMask" + }, + "widgets_values": [ + 150, + 0, + 0, + 0 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 27, + "type": "LoadImage", + "pos": [ + 1080, + -490 + ], + "size": [ + 240, + 330 + ], + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 81 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "venere.jpg", + "image" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 47, + "type": "FeatherMask", + "pos": [ + 70, + 100 + ], + "size": [ + 210, + 130 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 71 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 72 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FeatherMask" + }, + "widgets_values": [ + 0, + 0, + 150, + 0 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 33, + "type": "MaskComposite", + "pos": [ + 310, + 100 + ], + "size": [ + 210, + 130 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "destination", + "type": "MASK", + "link": 51 + }, + { + "name": "source", + "type": "MASK", + "link": 72 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 89 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 0, + 0, + "add" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 10, + "type": "IPAdapterModelLoader", + "pos": [ + 380, + -330 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 7, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 79, + 85 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.bin" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 11, + "type": "CLIPVisionLoader", + "pos": [ + 380, + -230 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 8, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 80, + 86 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 13, + "type": "PrepImageForClipVision", + "pos": [ + 430, + -120 + ], + "size": { + "0": 243.60000610351562, + "1": 110 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 32 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 87 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PrepImageForClipVision" + }, + "widgets_values": [ + "LANCZOS", + "center", + 0 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 32, + "type": "SolidMask", + "pos": [ + -170, + 120 + ], + "size": { + "0": 210, + "1": 106 + }, + "flags": {}, + "order": 9, + "mode": 0, + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 71 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 384, + 512 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 4, + "type": "CLIPTextEncode", + "pos": [ + 550, + 310 + ], + "size": { + "0": 400, + "1": 160 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "two girl friends laughing" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 51, + "type": "IPAdapterApply", + "pos": [ + 720, + -210 + ], + "size": [ + 210, + 190 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 85 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 86 + }, + { + "name": "image", + "type": "IMAGE", + "link": 87 + }, + { + "name": "model", + "type": "MODEL", + "link": 88 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": 89 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 90 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterApply" + }, + "widgets_values": [ + 0.7000000000000001, + 0.5, + "channel penalty" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 50, + "type": "IPAdapterApply", + "pos": [ + 1400, + -240 + ], + "size": [ + 210, + 190 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 79 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 80 + }, + { + "name": "image", + "type": "IMAGE", + "link": 81 + }, + { + "name": "model", + "type": "MODEL", + "link": 90 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": 83 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 84 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterApply" + }, + "widgets_values": [ + 0.7000000000000001, + 0.5, + "channel penalty" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 1, + "type": "KSampler", + "pos": [ + 1140, + 280 + ], + "size": { + "0": 240, + "1": 262 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 84 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 3 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 4 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 37, + "fixed", + 30, + 5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 8, + "type": "PreviewImage", + "pos": [ + 1430, + 380 + ], + "size": [ + 675.9466064453127, + 480.1444152832032 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + } + ], + "links": [ + [ + 2, + 4, + 0, + 1, + 1, + "CONDITIONING" + ], + [ + 3, + 5, + 0, + 1, + 2, + "CONDITIONING" + ], + [ + 4, + 3, + 0, + 1, + 3, + "LATENT" + ], + [ + 5, + 2, + 1, + 4, + 0, + "CLIP" + ], + [ + 6, + 2, + 1, + 5, + 0, + "CLIP" + ], + [ + 7, + 1, + 0, + 6, + 0, + "LATENT" + ], + [ + 8, + 7, + 0, + 6, + 1, + "VAE" + ], + [ + 9, + 6, + 0, + 8, + 0, + "IMAGE" + ], + [ + 32, + 12, + 0, + 13, + 0, + "IMAGE" + ], + [ + 51, + 34, + 0, + 33, + 0, + "MASK" + ], + [ + 57, + 34, + 0, + 38, + 0, + "MASK" + ], + [ + 71, + 32, + 0, + 47, + 0, + "MASK" + ], + [ + 72, + 47, + 0, + 33, + 1, + "MASK" + ], + [ + 75, + 37, + 0, + 48, + 0, + "MASK" + ], + [ + 76, + 48, + 0, + 38, + 1, + "MASK" + ], + [ + 79, + 10, + 0, + 50, + 0, + "IPADAPTER" + ], + [ + 80, + 11, + 0, + 50, + 1, + "CLIP_VISION" + ], + [ + 81, + 27, + 0, + 50, + 2, + "IMAGE" + ], + [ + 83, + 38, + 0, + 50, + 4, + "MASK" + ], + [ + 84, + 50, + 0, + 1, + 0, + "MODEL" + ], + [ + 85, + 10, + 0, + 51, + 0, + "IPADAPTER" + ], + [ + 86, + 11, + 0, + 51, + 1, + "CLIP_VISION" + ], + [ + 87, + 13, + 0, + 51, + 2, + "IMAGE" + ], + [ + 88, + 2, + 0, + 51, + 3, + "MODEL" + ], + [ + 89, + 33, + 0, + 51, + 4, + "MASK" + ], + [ + 90, + 51, + 0, + 50, + 3, + "MODEL" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/IPAdapter_mask.json b/examples/IPAdapter_mask.json new file mode 100644 index 0000000..b82d79b --- /dev/null +++ b/examples/IPAdapter_mask.json @@ -0,0 +1,665 @@ +{ + "last_node_id": 51, + "last_link_id": 96, + "nodes": [ + { + "id": 5, + "type": "CLIPTextEncode", + "pos": [ + 550, + 510 + ], + "size": { + "0": 400, + "1": 160 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 6 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 3 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, malformed, video game, rendering, naked, cleavage, horror, zombie, text, watermark" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 6, + "type": "VAEDecode", + "pos": [ + 1360, + 270 + ], + "size": { + "0": 140, + "1": 50 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 7, + "type": "VAELoader", + "pos": [ + 1090, + 570 + ], + "size": { + "0": 240, + "1": 60 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "vae-ft-mse-840000-ema-pruned.safetensors" + ] + }, + { + "id": 2, + "type": "CheckpointLoaderSimple", + "pos": [ + 210, + 410 + ], + "size": { + "0": 290, + "1": 100 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 88 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 5, + 6 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/dreamshaper_8.safetensors" + ] + }, + { + "id": 11, + "type": "CLIPVisionLoader", + "pos": [ + 490, + 20 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 86 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 13, + "type": "PrepImageForClipVision", + "pos": [ + 540, + 130 + ], + "size": { + "0": 243.60000610351562, + "1": 110 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 32 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 87 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PrepImageForClipVision" + }, + "widgets_values": [ + "LANCZOS", + "center", + 0 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 4, + "type": "CLIPTextEncode", + "pos": [ + 550, + 310 + ], + "size": { + "0": 400, + "1": 160 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "photography of a warrior woman in a cherry blossom forest, high quality, highly detailed" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 3, + "type": "EmptyLatentImage", + "pos": [ + 740, + 720 + ], + "size": { + "0": 210, + "1": 110 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 768, + 512, + 1 + ] + }, + { + "id": 1, + "type": "KSampler", + "pos": [ + 1090, + 270 + ], + "size": { + "0": 240, + "1": 262 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 93 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 3 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 4 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 40, + "fixed", + 30, + 5, + "ddim", + "ddim_uniform", + 1 + ] + }, + { + "id": 10, + "type": "IPAdapterModelLoader", + "pos": [ + 490, + -80 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 85 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.bin" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 230, + -80 + ], + "size": [ + 230, + 320 + ], + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 32 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 96 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "clipspace/clipspace-mask-790059.png [input]", + "image" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 51, + "type": "IPAdapterApply", + "pos": [ + 830, + 40 + ], + "size": [ + 210, + 190 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 85 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 86 + }, + { + "name": "image", + "type": "IMAGE", + "link": 87 + }, + { + "name": "model", + "type": "MODEL", + "link": 88 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": 96 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 93 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterApply" + }, + "widgets_values": [ + 0.7000000000000001, + 0, + "linear" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 8, + "type": "PreviewImage", + "pos": [ + 1365, + 369 + ], + "size": [ + 675.9466064453127, + 480.1444152832032 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + } + ], + "links": [ + [ + 2, + 4, + 0, + 1, + 1, + "CONDITIONING" + ], + [ + 3, + 5, + 0, + 1, + 2, + "CONDITIONING" + ], + [ + 4, + 3, + 0, + 1, + 3, + "LATENT" + ], + [ + 5, + 2, + 1, + 4, + 0, + "CLIP" + ], + [ + 6, + 2, + 1, + 5, + 0, + "CLIP" + ], + [ + 7, + 1, + 0, + 6, + 0, + "LATENT" + ], + [ + 8, + 7, + 0, + 6, + 1, + "VAE" + ], + [ + 9, + 6, + 0, + 8, + 0, + "IMAGE" + ], + [ + 32, + 12, + 0, + 13, + 0, + "IMAGE" + ], + [ + 85, + 10, + 0, + 51, + 0, + "IPADAPTER" + ], + [ + 86, + 11, + 0, + 51, + 1, + "CLIP_VISION" + ], + [ + 87, + 13, + 0, + 51, + 2, + "IMAGE" + ], + [ + 88, + 2, + 0, + 51, + 3, + "MODEL" + ], + [ + 93, + 51, + 0, + 1, + 0, + "MODEL" + ], + [ + 96, + 12, + 1, + 51, + 4, + "MASK" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/masking.jpg b/examples/masking.jpg new file mode 100644 index 0000000..dd45bae Binary files /dev/null and b/examples/masking.jpg differ