diff --git a/IPAdapterPlus.py b/IPAdapterPlus.py index af5d090..c79e28f 100644 --- a/IPAdapterPlus.py +++ b/IPAdapterPlus.py @@ -76,7 +76,7 @@ def attention(q, k, v, extra_options): out = out.transpose(1, 2).reshape(b, -1, extra_options["n_heads"] * extra_options["dim_head"]) return out -# TODO: still have to find the best way to add noise to the uncod image +# TODO: still have to find the best way to add noise to the uncond image def image_add_noise(image, noise): image = image.permute([0,3,1,2]) transforms = TT.Compose([ @@ -124,6 +124,7 @@ def encode_neg_image(clip_vision, image=None, noise=0): if t is not None: if k == 'hidden_states': outputs["penultimate_hidden_states"] = t[-2].cpu() + outputs["hidden_states"] = None else: outputs[k] = t.cpu() @@ -243,7 +244,7 @@ class IPAdapterApply: "clip_vision": ("CLIP_VISION",), "image": ("IMAGE",), "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.1 }), + "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }) }, } @@ -337,4 +338,4 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "IPAdapterModelLoader": "Load IPAdapter Model", "IPAdapterApply": "Apply IPAdapter", -} \ No newline at end of file +} diff --git a/README.md b/README.md index fb33fbc..f21b3ec 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ There's a basic workflow included in this repo and a few examples in the [exampl The `noise` paramenter is an exclusive experimental exploitation of the IPAdapter models. You can set it as low as `0.01` for an arguably better result. **Please report your experience with the noise option**!
-Nerd-explain of the noise option +Nerd-explain the noise option I made quite a few tests and I was not sure if it was worth it to include this feature, but the results are interesting enough to warrant at least a test period. Let me know what you think. @@ -58,6 +58,10 @@ Basically the IPAdapter sends two pictures for the conditioning, one is the refe What I'm doing is to send a very noisy image instead of an empty one. The `noise` parameter determines the amount of noise. A value of `0.01` adds a lot of noise and creates an image close to the original created by the IPAdapter (a very noisy image being intelligible could be considered like an empty image); a value of `1.0` removes most of noise so the image gets conditioned more.
+### IMPORTANT: Preparing the reference image + +The reference image needs to be encoded by the CLIP vision model. The encoder resizes the image **and crops it to the center!**. It's not an IPAdapter thing, it's how it works. This means that if you use a portrait or landscape image and the main attention (eg: the face of a character) is not in the middle you'll likely get undesired results. As a rule of thumb try to use square pictures. + ### KSampler configuration suggestions Weirdly enough all `dpmpp_2m` (including `_sde`) samplers seem to perfom poorly with IPAdapter and they generally need 5-10 more steps than usual. `ddmin` and `euler` seem to be pretty good.