From 8b8607281a321c7a0a27d351fa14ca69d3f02bd3 Mon Sep 17 00:00:00 2001 From: matt3o Date: Tue, 26 Mar 2024 08:48:18 +0100 Subject: [PATCH] fix compatibility with new IPAdapter --- CrossAttentionPatch.py | 164 ++++++++++++++++ InstantID.py | 179 +----------------- examples/InstantID_IPAdapter.json | 305 +++++++++++++++--------------- utils.py | 24 +++ 4 files changed, 353 insertions(+), 319 deletions(-) create mode 100644 CrossAttentionPatch.py create mode 100644 utils.py diff --git a/CrossAttentionPatch.py b/CrossAttentionPatch.py new file mode 100644 index 0000000..0e06bc6 --- /dev/null +++ b/CrossAttentionPatch.py @@ -0,0 +1,164 @@ +import torch +import math +import torch.nn.functional as F +from comfy.ldm.modules.attention import optimized_attention +from .utils import tensor_to_size + +class CrossAttentionPatch: + # forward for patching + def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + self.weights = [weight] + self.ipadapters = [ipadapter] + self.conds = [cond] + self.unconds = [uncond] + self.weight_types = [weight_type] + self.masks = [mask] + self.sigma_starts = [sigma_start] + self.sigma_ends = [sigma_end] + self.unfold_batch = [unfold_batch] + self.embeds_scaling = [embeds_scaling] + self.number = number + self.layers = 10 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 15 + + self.k_key = str(self.number*2+1) + "_to_k_ip" + self.v_key = str(self.number*2+1) + "_to_v_ip" + + def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + self.weights.append(weight) + self.ipadapters.append(ipadapter) + self.conds.append(cond) + self.unconds.append(uncond) + self.weight_types.append(weight_type) + self.masks.append(mask) + self.sigma_starts.append(sigma_start) + self.sigma_ends.append(sigma_end) + self.unfold_batch.append(unfold_batch) + self.embeds_scaling.append(embeds_scaling) + + def __call__(self, q, k, v, extra_options): + dtype = q.dtype + cond_or_uncond = extra_options["cond_or_uncond"] + sigma = extra_options["sigmas"].detach().cpu()[0].item() if 'sigmas' in extra_options else 999999999.9 + block_type = extra_options["block"][0] + #block_id = extra_options["block"][1] + t_idx = extra_options["transformer_index"] + + # extra options for AnimateDiff + ad_params = extra_options['ad_params'] if "ad_params" in extra_options else None + + b = q.shape[0] + seq_len = q.shape[1] + batch_prompt = b // len(cond_or_uncond) + out = optimized_attention(q, k, v, extra_options["n_heads"]) + _, _, oh, ow = extra_options["original_shape"] + + for weight, cond, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, embeds_scaling in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.embeds_scaling): + if sigma <= sigma_start and sigma >= sigma_end: + if unfold_batch and cond.shape[0] > 1: + # Check AnimateDiff context window + if ad_params is not None and ad_params["sub_idxs"] is not None: + # if image length matches or exceeds full_length get sub_idx images + if cond.shape[0] >= ad_params["full_length"]: + cond = torch.Tensor(cond[ad_params["sub_idxs"]]) + uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) + # otherwise get sub_idxs images + else: + cond = tensor_to_size(cond, ad_params["full_length"]) + uncond = tensor_to_size(uncond, ad_params["full_length"]) + cond = cond[ad_params["sub_idxs"]] + uncond = uncond[ad_params["sub_idxs"]] + + cond = tensor_to_size(cond, batch_prompt) + uncond = tensor_to_size(uncond, batch_prompt) + + k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond) + k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond) + v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond) + v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond) + else: + k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond).repeat(batch_prompt, 1, 1) + k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond).repeat(batch_prompt, 1, 1) + v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond).repeat(batch_prompt, 1, 1) + v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond).repeat(batch_prompt, 1, 1) + + if weight_type == 'ease in': + weight = weight * (0.05 + 0.95 * (1 - t_idx / self.layers)) + elif weight_type == 'ease out': + weight = weight * (0.05 + 0.95 * (t_idx / self.layers)) + elif weight_type == 'ease in-out': + weight = weight * (0.05 + 0.95 * (1 - abs(t_idx - (self.layers/2)) / (self.layers/2))) + elif weight_type == 'reverse in-out': + weight = weight * (0.05 + 0.95 * (abs(t_idx - (self.layers/2)) / (self.layers/2))) + elif weight_type == 'weak input' and block_type == 'input': + weight = weight * 0.2 + elif weight_type == 'weak middle' and block_type == 'middle': + weight = weight * 0.2 + elif weight_type == 'weak output' and block_type == 'output': + weight = weight * 0.2 + elif weight_type == 'strong middle' and (block_type == 'input' or block_type == 'output'): + weight = weight * 0.2 + + ip_k = torch.cat([(k_cond, k_uncond)[i] for i in cond_or_uncond], dim=0) + ip_v = torch.cat([(v_cond, v_uncond)[i] for i in cond_or_uncond], dim=0) + + if embeds_scaling == 'K+mean(V) w/ C penalty': + scaling = float(ip_k.shape[2]) / 1280.0 + weight = weight * scaling + ip_k = ip_k * weight + ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True) + ip_v = (ip_v - ip_v_mean) + ip_v_mean * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + del ip_v_mean + elif embeds_scaling == 'K+V w/ C penalty': + scaling = float(ip_k.shape[2]) / 1280.0 + weight = weight * scaling + ip_k = ip_k * weight + ip_v = ip_v * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + elif embeds_scaling == 'K+V': + ip_k = ip_k * weight + ip_v = ip_v * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + else: + #ip_v = ip_v * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + out_ip = out_ip * weight # I'm doing this to get the same results as before + + if mask is not None: + mask_h = oh / math.sqrt(oh * ow / seq_len) + mask_h = int(mask_h) + int((seq_len % int(mask_h)) != 0) + mask_w = seq_len // mask_h + + # check if using AnimateDiff and sliding context window + if (mask.shape[0] > 1 and ad_params is not None and ad_params["sub_idxs"] is not None): + # if mask length matches or exceeds full_length, get sub_idx masks + if mask.shape[0] >= ad_params["full_length"]: + mask = torch.Tensor(mask[ad_params["sub_idxs"]]) + mask = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bilinear").squeeze(1) + else: + mask = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bilinear").squeeze(1) + mask = tensor_to_size(mask, ad_params["full_length"]) + mask = mask[ad_params["sub_idxs"]] + else: + mask = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bilinear").squeeze(1) + mask = tensor_to_size(mask, batch_prompt) + + mask = mask.repeat(len(cond_or_uncond), 1, 1) + mask = mask.view(mask.shape[0], -1, 1).repeat(1, 1, out.shape[2]) + + # covers cases where extreme aspect ratios can cause the mask to have a wrong size + mask_len = mask_h * mask_w + if mask_len < seq_len: + pad_len = seq_len - mask_len + pad1 = pad_len // 2 + pad2 = pad_len - pad1 + mask = F.pad(mask, (0, 0, pad1, pad2), value=0.0) + elif mask_len > seq_len: + crop_start = (mask_len - seq_len) // 2 + mask = mask[:, crop_start:crop_start+seq_len, :] + + out_ip = out_ip * mask + + out = out + out_ip + + return out.to(dtype=dtype) diff --git a/InstantID.py b/InstantID.py index fa30b6f..4b31ba3 100644 --- a/InstantID.py +++ b/InstantID.py @@ -8,10 +8,16 @@ import cv2 import PIL.Image from comfy.ldm.modules.attention import optimized_attention from .resampler import Resampler +from .CrossAttentionPatch import CrossAttentionPatch +from .utils import tensor_to_size, tensor_to_image, image_to_tensor from insightface.app import FaceAnalysis -import torchvision.transforms.v2 as T +try: + import torchvision.transforms.v2 as T +except ImportError: + import torchvision.transforms as T + import torch.nn.functional as F MODELS_DIR = os.path.join(folder_paths.models_dir, "instantid") @@ -51,163 +57,6 @@ def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,2 out_img_pil = PIL.Image.fromarray(out_img.astype(np.uint8)) return out_img_pil -# All this mess to keep compatibility with IPAdapter, it will be helpful in case we want AnimateDiff to work with InstantID -class CrossAttentionPatch: - # forward for patching - def __init__(self, weight, ipadapter, number, cond, uncond, weight_type="original", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False): - self.weights = [weight] - self.ipadapters = [ipadapter] - self.conds = [cond] - self.unconds = [uncond] - self.number = number - self.weight_type = [weight_type] - self.masks = [mask] - self.sigma_start = [sigma_start] - self.sigma_end = [sigma_end] - self.unfold_batch = [unfold_batch] - - self.k_key = str(self.number*2+1) + "_to_k_ip" - self.v_key = str(self.number*2+1) + "_to_v_ip" - - def set_new_condition(self, weight, ipadapter, number, cond, uncond, weight_type="original", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False): - self.weights.append(weight) - self.ipadapters.append(ipadapter) - self.conds.append(cond) - self.unconds.append(uncond) - self.masks.append(mask) - self.weight_type.append(weight_type) - self.sigma_start.append(sigma_start) - self.sigma_end.append(sigma_end) - self.unfold_batch.append(unfold_batch) - - def __call__(self, n, context_attn2, value_attn2, extra_options): - org_dtype = n.dtype - cond_or_uncond = extra_options["cond_or_uncond"] - sigma = extra_options["sigmas"][0] if 'sigmas' in extra_options else None - sigma = sigma.item() if sigma is not None else 999999999.9 - - # extra options for AnimateDiff - ad_params = extra_options['ad_params'] if "ad_params" in extra_options else None - - q = n - k = context_attn2 - v = value_attn2 - b = q.shape[0] - qs = q.shape[1] - batch_prompt = b // len(cond_or_uncond) - out = optimized_attention(q, k, v, extra_options["n_heads"]) - _, _, lh, lw = extra_options["original_shape"] - - for weight, cond, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_type, self.sigma_start, self.sigma_end, self.unfold_batch): - if sigma > sigma_start or sigma < sigma_end: - continue - - if unfold_batch and cond.shape[0] > 1: - # Check AnimateDiff context window - if ad_params is not None and ad_params["sub_idxs"] is not None: - # if images length matches or exceeds full_length get sub_idx images - if cond.shape[0] >= ad_params["full_length"]: - cond = torch.Tensor(cond[ad_params["sub_idxs"]]) - uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) - # otherwise, need to do more to get proper sub_idxs masks - else: - # check if images length matches full_length - if not, make it match - if cond.shape[0] < ad_params["full_length"]: - cond = torch.cat((cond, cond[-1:].repeat((ad_params["full_length"]-cond.shape[0], 1, 1))), dim=0) - uncond = torch.cat((uncond, uncond[-1:].repeat((ad_params["full_length"]-uncond.shape[0], 1, 1))), dim=0) - # if we have too many remove the excess (should not happen, but just in case) - if cond.shape[0] > ad_params["full_length"]: - cond = cond[:ad_params["full_length"]] - uncond = uncond[:ad_params["full_length"]] - cond = cond[ad_params["sub_idxs"]] - uncond = uncond[ad_params["sub_idxs"]] - - # if we don't have enough reference images repeat the last one until we reach the right size - if cond.shape[0] < batch_prompt: - cond = torch.cat((cond, cond[-1:].repeat((batch_prompt-cond.shape[0], 1, 1))), dim=0) - uncond = torch.cat((uncond, uncond[-1:].repeat((batch_prompt-uncond.shape[0], 1, 1))), dim=0) - # if we have too many remove the exceeding - elif cond.shape[0] > batch_prompt: - cond = cond[:batch_prompt] - uncond = uncond[:batch_prompt] - - k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond) - k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond) - v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond) - v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond) - else: - k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond).repeat(batch_prompt, 1, 1) - k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond).repeat(batch_prompt, 1, 1) - v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond).repeat(batch_prompt, 1, 1) - v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond).repeat(batch_prompt, 1, 1) - - if weight_type.startswith("linear"): - ip_k = torch.cat([(k_cond, k_uncond)[i] for i in cond_or_uncond], dim=0) * weight - ip_v = torch.cat([(v_cond, v_uncond)[i] for i in cond_or_uncond], dim=0) * weight - else: - ip_k = torch.cat([(k_cond, k_uncond)[i] for i in cond_or_uncond], dim=0) - ip_v = torch.cat([(v_cond, v_uncond)[i] for i in cond_or_uncond], dim=0) - - if weight_type.startswith("channel"): - # code by Lvmin Zhang at Stanford University as also seen on Fooocus IPAdapter implementation - ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True) - ip_v_offset = ip_v - ip_v_mean - _, _, C = ip_k.shape - channel_penalty = float(C) / 1280.0 - W = weight * channel_penalty - ip_k = ip_k * W - ip_v = ip_v_offset + ip_v_mean * W - - out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) - if weight_type.startswith("original"): - out_ip = out_ip * weight - - if mask is not None: - # TODO: needs checking - mask_h = lh / math.sqrt(lh * lw / qs) - mask_h = int(mask_h) + int((qs % int(mask_h)) != 0) - mask_w = qs // mask_h - - # check if using AnimateDiff and sliding context window - if (mask.shape[0] > 1 and ad_params is not None and ad_params["sub_idxs"] is not None): - # if mask length matches or exceeds full_length, just get sub_idx masks, resize, and continue - if mask.shape[0] >= ad_params["full_length"]: - mask_downsample = torch.Tensor(mask[ad_params["sub_idxs"]]) - mask_downsample = F.interpolate(mask_downsample.unsqueeze(1), size=(mask_h, mask_w), mode="bicubic").squeeze(1) - # otherwise, need to do more to get proper sub_idxs masks - else: - # resize to needed attention size (to save on memory) - mask_downsample = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bicubic").squeeze(1) - # check if mask length matches full_length - if not, make it match - if mask_downsample.shape[0] < ad_params["full_length"]: - mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:].repeat((ad_params["full_length"]-mask_downsample.shape[0], 1, 1))), dim=0) - # if we have too many remove the excess (should not happen, but just in case) - if mask_downsample.shape[0] > ad_params["full_length"]: - mask_downsample = mask_downsample[:ad_params["full_length"]] - # now, select sub_idxs masks - mask_downsample = mask_downsample[ad_params["sub_idxs"]] - # otherwise, perform usual mask interpolation - else: - mask_downsample = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bicubic").squeeze(1) - - # if we don't have enough masks repeat the last one until we reach the right size - if mask_downsample.shape[0] < batch_prompt: - mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:, :, :].repeat((batch_prompt-mask_downsample.shape[0], 1, 1))), dim=0) - # if we have too many remove the exceeding - elif mask_downsample.shape[0] > batch_prompt: - mask_downsample = mask_downsample[:batch_prompt, :, :] - - # repeat the masks - mask_downsample = mask_downsample.repeat(len(cond_or_uncond), 1, 1) - mask_downsample = mask_downsample.view(mask_downsample.shape[0], -1, 1).repeat(1, 1, out.shape[2]) - - out_ip = out_ip * mask_downsample - - out = out + out_ip - - return out.to(dtype=org_dtype) - - class InstantID(torch.nn.Module): def __init__(self, instantid_model, cross_attention_dim=1280, output_cross_attention_dim=1024, clip_embeddings_dim=512, clip_extra_context_tokens=16): super().__init__() @@ -305,14 +154,8 @@ class InstantIDModelLoader: return (model,) -def tensorToNP(image): - out = torch.clamp(255. * image.detach().cpu(), 0, 255).to(torch.uint8) - out = out[..., [2, 1, 0]] - out = out.numpy() - return out - def extractFeatures(insightface, image, extract_kps=False): - face_img = tensorToNP(image) + face_img = tensor_to_image(image) out = [] insightface.det_model.input_size = (640,640) # reset the detection size @@ -357,7 +200,7 @@ class InstantIDFaceAnalysis: CATEGORY = "InstantID" def load_insight_face(self, provider): - model = FaceAnalysis(name="antelopev2", root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider',]) # buffalo_l + model = FaceAnalysis(name="antelopev2", root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider',]) # alternative to buffalo_l model.prepare(ctx_id=0, det_size=(640, 640)) return (model,) @@ -484,15 +327,14 @@ class ApplyInstantID: mask = mask.to(self.device) patch_kwargs = { + "ipadapter": self.instantid, "number": 0, "weight": ip_weight, - "ipadapter": self.instantid, "cond": image_prompt_embeds, "uncond": uncond_image_prompt_embeds, "mask": mask, "sigma_start": sigma_start, "sigma_end": sigma_end, - "weight_type": "original", } if not is_sdxl: @@ -664,7 +506,6 @@ class InstantIDAttentionPatch: "mask": mask, "sigma_start": sigma_start, "sigma_end": sigma_end, - "weight_type": "original", } if not is_sdxl: diff --git a/examples/InstantID_IPAdapter.json b/examples/InstantID_IPAdapter.json index 76bce6c..dce0174 100644 --- a/examples/InstantID_IPAdapter.json +++ b/examples/InstantID_IPAdapter.json @@ -1,6 +1,6 @@ { - "last_node_id": 71, - "last_link_id": 226, + "last_node_id": 72, + "last_link_id": 231, "nodes": [ { "id": 11, @@ -361,6 +361,97 @@ "image" ] }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1540, + 200 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 231 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 200 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 201 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 1631591432, + "fixed", + 30, + 4.5, + "ddpm", + "karras", + 1 + ] + }, + { + "id": 68, + "type": "IPAdapterModelLoader", + "pos": [ + 830, + -500 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 227 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sdxl_vit-h.safetensors" + ] + }, { "id": 60, "type": "ApplyInstantID", @@ -427,7 +518,7 @@ "name": "MODEL", "type": "MODEL", "links": [ - 225 + 230 ], "shape": 3, "slot_index": 0 @@ -460,38 +551,6 @@ 1 ] }, - { - "id": 68, - "type": "IPAdapterModelLoader", - "pos": [ - 830, - -500 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 6, - "mode": 0, - "outputs": [ - { - "name": "IPADAPTER", - "type": "IPADAPTER", - "links": [ - 222 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterModelLoader" - }, - "widgets_values": [ - "ip-adapter-plus_sdxl_vit-h.safetensors" - ] - }, { "id": 70, "type": "CLIPVisionLoader", @@ -511,82 +570,17 @@ "name": "CLIP_VISION", "type": "CLIP_VISION", "links": [ - 223 - ], - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "CLIPVisionLoader" - }, - "widgets_values": [ - "IPAdapter_image_encoder_sd15.safetensors" - ] - }, - { - "id": 69, - "type": "IPAdapterApply", - "pos": [ - 1243, - -287 - ], - "size": { - "0": 315, - "1": 258 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": 222 - }, - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 223, - "slot_index": 1 - }, - { - "name": "image", - "type": "IMAGE", - "link": 224, - "slot_index": 2 - }, - { - "name": "model", - "type": "MODEL", - "link": 225 - }, - { - "name": "attn_mask", - "type": "MASK", - "link": null - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 226 + 228 ], "shape": 3, "slot_index": 0 } ], "properties": { - "Node name for S&R": "IPAdapterApply" + "Node name for S&R": "CLIPVisionLoader" }, "widgets_values": [ - 0.45, - 0, - "original", - 0, - 1, - false + "CLIP-ViT-H-14-laion2B-s32B-b79K.safetensors" ] }, { @@ -596,10 +590,10 @@ 830, -280 ], - "size": [ - 315, - 314 - ], + "size": { + "0": 315, + "1": 314 + }, "flags": {}, "order": 8, "mode": 0, @@ -608,9 +602,10 @@ "name": "IMAGE", "type": "IMAGE", "links": [ - 224 + 229 ], - "shape": 3 + "shape": 3, + "slot_index": 0 }, { "name": "MASK", @@ -623,67 +618,77 @@ "Node name for S&R": "LoadImage" }, "widgets_values": [ - "SDXL_00624_.png", + "anime_colorful.png", "image" ] }, { - "id": 3, - "type": "KSampler", + "id": 72, + "type": "IPAdapterAdvanced", "pos": [ - 1540, - 200 + 1226, + -337 ], "size": { "0": 315, - "1": 262 + "1": 278 }, "flags": {}, - "order": 13, + "order": 12, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 226 + "link": 230 }, { - "name": "positive", - "type": "CONDITIONING", - "link": 200 + "name": "ipadapter", + "type": "IPADAPTER", + "link": 227 }, { - "name": "negative", - "type": "CONDITIONING", - "link": 201 + "name": "image", + "type": "IMAGE", + "link": 229 }, { - "name": "latent_image", - "type": "LATENT", - "link": 2 + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 228 } ], "outputs": [ { - "name": "LATENT", - "type": "LATENT", + "name": "MODEL", + "type": "MODEL", "links": [ - 7 + 231 ], + "shape": 3, "slot_index": 0 } ], "properties": { - "Node name for S&R": "KSampler" + "Node name for S&R": "IPAdapterAdvanced" }, "widgets_values": [ - 1631591432, - "fixed", - 30, - 4.5, - "ddpm", - "karras", - 1 + 0.5, + "linear", + "concat", + 0, + 1, + "V only" ] } ], @@ -809,40 +814,40 @@ "IMAGE" ], [ - 222, + 227, 68, 0, - 69, - 0, + 72, + 1, "IPADAPTER" ], [ - 223, + 228, 70, 0, - 69, - 1, + 72, + 5, "CLIP_VISION" ], [ - 224, + 229, 71, 0, - 69, + 72, 2, "IMAGE" ], [ - 225, + 230, 60, 0, - 69, - 3, + 72, + 0, "MODEL" ], [ - 226, - 69, + 231, + 72, 0, 3, 0, diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..94da4a6 --- /dev/null +++ b/utils.py @@ -0,0 +1,24 @@ +import torch + +def tensor_to_size(source, dest_size): + if isinstance(dest_size, torch.Tensor): + dest_size = dest_size.shape[0] + source_size = source.shape[0] + + if source_size < dest_size: + shape = [dest_size - source_size] + [1]*(source.dim()-1) + source = torch.cat((source, source[-1:].repeat(shape)), dim=0) + elif source_size > dest_size: + source = source[:dest_size] + + return source + +def tensor_to_image(tensor): + image = tensor.mul(255).clamp(0, 255).byte().cpu() + image = image[..., [2, 1, 0]].numpy() + return image + +def image_to_tensor(image): + tensor = torch.clamp(torch.from_numpy(image).float() / 255., 0, 1) + tensor = tensor[..., [2, 1, 0]] + return tensor