diff --git a/SteerableMotion.py b/SteerableMotion.py index 78d7de8..6a953c7 100644 --- a/SteerableMotion.py +++ b/SteerableMotion.py @@ -5,17 +5,12 @@ import numpy as np # Third-party library imports import torch import torchvision.transforms as transforms - from PIL import Image import matplotlib.pyplot as plt - # Local application/library specific imports import folder_paths -from .imports.IPAdapterPlus import IPAdapterApplyImport, prep_image, IPAdapterEncoderImport -from .imports.AdvancedControlNet.latent_keyframe_nodes import LatentKeyframeInterpolationNodeImport -from .imports.AdvancedControlNet.weight_nodes import ScaledSoftUniversalWeightsImport +from .imports.ComfyUI_IPAdapter_plus.IPAdapterPlus import IPAdapterTiledImport, PrepImageForClipVisionImport, IPAdapterAdvancedImport from .imports.AdvancedControlNet.nodes_sparsectrl import SparseIndexMethodNodeImport -from .imports.AdvancedControlNet.nodes import ControlNetLoaderAdvancedImport, AdvancedControlNetApplyImport,TimestepKeyframeNodeImport class BatchCreativeInterpolationNode: @@ -32,8 +27,7 @@ class BatchCreativeInterpolationNode: "images": ("IMAGE", ), "model": ("MODEL", ), "ipadapter": ("IPADAPTER", ), - "clip_vision": ("CLIP_VISION",), - "control_net_name": (folder_paths.get_filename_list("controlnet"), ), + "clip_vision": ("CLIP_VISION",), "type_of_frame_distribution": (["linear", "dynamic"],), "linear_frame_distribution_value": ("INT", {"default": 16, "min": 4, "max": 64, "step": 1}), "dynamic_frame_distribution_values": ("STRING", {"multiline": True, "default": "0,10,26,40"}), @@ -42,35 +36,27 @@ class BatchCreativeInterpolationNode: "dynamic_key_frame_influence_values": ("STRING", {"multiline": True, "default": "(1.0,1.0),(1.0,1.5)(1.0,0.5)"}), "type_of_strength_distribution": (["linear", "dynamic"],), "linear_strength_value": ("STRING", {"multiline": False, "default": "(0.3,0.4)"}), - "dynamic_strength_values": ("STRING", {"multiline": True, "default": "(0.0,1.0),(0.0,1.0),(0.0,1.0),(0.0,1.0)"}), - "soft_scaled_cn_weights_multiplier": ("FLOAT", {"default": 0.85, "min": 0.0, "max": 10.0, "step": 0.1}), - "buffer": ("INT", {"default": 4, "min": 1, "max": 16, "step": 1}), - "relative_cn_strength": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 10.0, "step": 0.01}), - "relative_ipadapter_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), - "ipadapter_noise": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}), - "ipadapter_start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "ipadapter_end_at": ("FLOAT", {"default": 0.75, "min": 0.0, "max": 1.0, "step": 0.01}), - "cn_start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), - "cn_end_at": ("FLOAT", {"default": 0.75, "min": 0.0, "max": 1.0, "step": 0.01}), + "dynamic_strength_values": ("STRING", {"multiline": True, "default": "(0.0,1.0),(0.0,1.0),(0.0,1.0),(0.0,1.0)"}), + "buffer": ("INT", {"default": 4, "min": 1, "max": 16, "step": 1}), + "high_detail_mode": ("BOOL", {"default": True}), + "ipa_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}), }, "optional": { } } - RETURN_TYPES = ("IMAGE","CONDITIONING","CONDITIONING","MODEL","SPARSE_METHOD","INT") - # "comparison_diagram, positive, negative, model, sparse_indexes, last_key_frame_position" + RETURN_TYPES = ("IMAGE","CONDITIONING","CONDITIONING","MODEL","SPARSE_METHOD","INT") RETURN_NAMES = ("GRAPH","POSITIVE","NEGATIVE","MODEL","KEYFRAME_POSITIONS","BATCH_SIZE") FUNCTION = "combined_function" CATEGORY = "Steerable-Motion" - def combined_function(self,positive,negative,images,model,ipadapter,clip_vision,control_net_name, + def combined_function(self,positive,negative,images,model,ipadapter,clip_vision, type_of_frame_distribution,linear_frame_distribution_value, dynamic_frame_distribution_values, type_of_key_frame_influence,linear_key_frame_influence_value, dynamic_key_frame_influence_values,type_of_strength_distribution, - linear_strength_value,dynamic_strength_values, soft_scaled_cn_weights_multiplier, - buffer, relative_cn_strength,relative_ipadapter_strength,ipadapter_noise, - ipadapter_start_at=0.0,ipadapter_end_at=0.75, cn_start_at=0.0, cn_end_at=0.75): + linear_strength_value,dynamic_strength_values, + buffer, high_detail_mode,ipa_weight): def get_keyframe_positions(type_of_frame_distribution, dynamic_frame_distribution_values, images, linear_frame_distribution_value): @@ -369,7 +355,6 @@ class BatchCreativeInterpolationNode: image = images[i-1] key_frame_influence_from,key_frame_influence_to = key_frame_influence_values[i-1] start_strength, mid_strength, end_strength = strength_values[i-1] - # strength_from, strength_to = cn_strength_values[i-1] keyframe_position = keyframe_positions[i] previous_key_frame_position = keyframe_positions[i-1] @@ -381,8 +366,10 @@ class BatchCreativeInterpolationNode: # interpolation = "ease-out" elif i == len(keyframe_positions) - 1: + image = images[i-2] strength_from = strength_to = strength_values[i-2][1] + batch_index_from = keyframe_positions[i-1] batch_index_to_excl = last_key_frame_position weights, frame_numbers = calculate_weights(batch_index_from, batch_index_to_excl, strength_from, strength_to, interpolation, False, last_key_frame_position, i, len(keyframe_positions), buffer) @@ -410,43 +397,32 @@ class BatchCreativeInterpolationNode: # COMBINE FIRST AND SECOND HALF weights = np.concatenate([first_half_weights, second_half_weights]) frame_numbers = np.concatenate([first_half_frame_numbers, second_half_frame_numbers]) - - # IMPORT REQUIRED NODES - latent_keyframe_interpolation_node = LatentKeyframeInterpolationNodeImport() - scaled_soft_control_net_weights = ScaledSoftUniversalWeightsImport() - timestep_keyframe_node = TimestepKeyframeNodeImport() - control_net_loader = ControlNetLoaderAdvancedImport() - apply_advanced_control_net = AdvancedControlNetApplyImport() - ipadapter_application = IPAdapterApplyImport() - ipadapter_encoder = IPAdapterEncoderImport() - - # IF CONTROL NET IS USED, APPLY IT - if relative_cn_strength > 0.0: - cn_frame_numbers, cn_weights = process_weights(frame_numbers, weights, relative_cn_strength) - latent_keyframe, = latent_keyframe_interpolation_node.load_keyframe(cn_weights, cn_frame_numbers) - control_net_weights, _ = scaled_soft_control_net_weights.load_weights(soft_scaled_cn_weights_multiplier, False) - timestep_keyframe = timestep_keyframe_node.load_keyframe(start_percent=0.0, control_net_weights=control_net_weights, latent_keyframe=latent_keyframe, prev_timestep_keyframe=None)[0] - control_net = control_net_loader.load_controlnet(control_net_name, timestep_keyframe)[0] - positive, negative = apply_advanced_control_net.apply_controlnet(positive, negative, control_net, image.unsqueeze(0), 1.0, cn_start_at, cn_end_at) - all_cn_frame_numbers.append(cn_frame_numbers) - all_cn_weights.append(cn_weights) + + # PROCESS WEIGHTS + ipa_frame_numbers, ipa_weights = process_weights(frame_numbers, weights, relative_ipadapter_strength) + + prepare_for_clip_vision = PrepImageForClipVisionImport() + prepped_image, = prepare_for_clip_vision.prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.1) + + prepped_image = prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.0)[0] + mask = create_mask_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) + + if high_detail_mode: + normal_ipa_start_at = 0.0 + normal_ipa_end_at = 0.25 else: - all_cn_frame_numbers = None - all_cn_weights = None - - # IF IP ADAPTER IS USED, APPLY IT - if relative_ipadapter_strength > 0.0: - ipa_frame_numbers, ipa_weights = process_weights(frame_numbers, weights, relative_ipadapter_strength) - prepped_image = prep_image(image=image.unsqueeze(0), interpolation="LANCZOS", crop_position="pad", sharpening=0.0)[0] - mask = create_mask_batch(last_key_frame_position, ipa_weights, ipa_frame_numbers) - embed, = ipadapter_encoder.preprocess(clip_vision, prepped_image, True, 0.0, 1.0) - model,_,_ = ipadapter_application.apply_ipadapter(ipadapter=ipadapter, model=model, weight=1.0, image=None, weight_type="original", - noise=ipadapter_noise, embeds=embed, attn_mask=mask, start_at=ipadapter_start_at, end_at=ipadapter_end_at, unfold_batch=True) - all_ipa_frame_numbers.append(ipa_frame_numbers) - all_ipa_weights.append(ipa_weights) - else: - all_ipa_frame_numbers = None - all_ipa_weights = None + normal_ipa_start_at = 0.0 + normal_ipa_end_at = 0.75 + + ipadapter_application = IPAdapterAdvancedImport() + model, = ipadapter_application.apply_ipadapter(model=model, ipadapter=ipadapter, image=prepped_image, weight=ipa_weight, weight_type='ease in-out', start_at=normal_ipa_start_at, end_at=normal_ipa_end_at, clip_vision=clip_vision, attn_mask=mask) + + if high_detail_mode: + tiled_ipa_application = IPAdapterTiledImport() + model, _ = ipadapter_application.apply_tiled(model=model, ipadapter=ipadapter, image=image.unsqueeze(0), weight=ipa_weight, weight_type='ease in-out', start_at=normal_ipa_start_at, end_at=normal_ipa_end_at, clip_vision=clip_vision, attn_mask=mask) + + all_ipa_frame_numbers.append(ipa_frame_numbers) + all_ipa_weights.append(ipa_weights) comparison_diagram, = plot_weight_comparison(all_cn_frame_numbers, all_cn_weights, all_ipa_frame_numbers, all_ipa_weights, buffer) diff --git a/imports/ComfyUI_IPAdapter_plus/.gitignore b/imports/ComfyUI_IPAdapter_plus/.gitignore new file mode 100644 index 0000000..b921ebf --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/.gitignore @@ -0,0 +1,4 @@ +/__pycache__/ +/models/*.bin +/models/*.safetensors +.directory \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py b/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py new file mode 100644 index 0000000..195292d --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py @@ -0,0 +1,167 @@ +import torch +import math +import torch.nn.functional as F +from comfy.ldm.modules.attention import optimized_attention +from .utils import tensor_to_size + +class CrossAttentionPatchImport: + # forward for patching + def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + self.weights = [weight] + self.ipadapters = [ipadapter] + self.conds = [cond] + self.unconds = [uncond] + self.weight_types = [weight_type] + self.masks = [mask] + self.sigma_starts = [sigma_start] + self.sigma_ends = [sigma_end] + self.unfold_batch = [unfold_batch] + self.embeds_scaling = [embeds_scaling] + self.number = number + self.layers = 10 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 15 # TODO: check if this is a valid condition to detect all models + + self.k_key = str(self.number*2+1) + "_to_k_ip" + self.v_key = str(self.number*2+1) + "_to_v_ip" + + def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'): + self.weights.append(weight) + self.ipadapters.append(ipadapter) + self.conds.append(cond) + self.unconds.append(uncond) + self.weight_types.append(weight_type) + self.masks.append(mask) + self.sigma_starts.append(sigma_start) + self.sigma_ends.append(sigma_end) + self.unfold_batch.append(unfold_batch) + self.embeds_scaling.append(embeds_scaling) + + def __call__(self, q, k, v, extra_options): + dtype = q.dtype + cond_or_uncond = extra_options["cond_or_uncond"] + sigma = extra_options["sigmas"].detach().cpu()[0].item() if 'sigmas' in extra_options else 999999999.9 + block_type = extra_options["block"][0] + #block_id = extra_options["block"][1] + t_idx = extra_options["transformer_index"] + + # extra options for AnimateDiff + ad_params = extra_options['ad_params'] if "ad_params" in extra_options else None + + b = q.shape[0] + seq_len = q.shape[1] + batch_prompt = b // len(cond_or_uncond) + out = optimized_attention(q, k, v, extra_options["n_heads"]) + _, _, oh, ow = extra_options["original_shape"] + + for weight, cond, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, embeds_scaling in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.embeds_scaling): + if sigma <= sigma_start and sigma >= sigma_end: + if unfold_batch and cond.shape[0] > 1: + # Check AnimateDiff context window + if ad_params is not None and ad_params["sub_idxs"] is not None: + # if image length matches or exceeds full_length get sub_idx images + if cond.shape[0] >= ad_params["full_length"]: + cond = torch.Tensor(cond[ad_params["sub_idxs"]]) + uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) + # otherwise get sub_idxs images + else: + cond = tensor_to_size(cond, ad_params["full_length"]) + uncond = tensor_to_size(uncond, ad_params["full_length"]) + cond = cond[ad_params["sub_idxs"]] + uncond = uncond[ad_params["sub_idxs"]] + + cond = tensor_to_size(cond, batch_prompt) + uncond = tensor_to_size(uncond, batch_prompt) + + k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond) + k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond) + v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond) + v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond) + else: + k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond).repeat(batch_prompt, 1, 1) + k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond).repeat(batch_prompt, 1, 1) + v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond).repeat(batch_prompt, 1, 1) + v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond).repeat(batch_prompt, 1, 1) + + if weight_type == 'ease in': + weight = weight * (0.05 + 0.95 * (1 - t_idx / self.layers)) + elif weight_type == 'ease out': + weight = weight * (0.05 + 0.95 * (t_idx / self.layers)) + elif weight_type == 'ease in-out': + weight = weight * (0.05 + 0.95 * (1 - abs(t_idx - (self.layers/2)) / (self.layers/2))) + elif weight_type == 'reverse in-out': + weight = weight * (0.05 + 0.95 * (abs(t_idx - (self.layers/2)) / (self.layers/2))) + elif weight_type == 'weak input' and block_type == 'input': + weight = weight * 0.2 + elif weight_type == 'weak middle' and block_type == 'middle': + weight = weight * 0.2 + elif weight_type == 'weak output' and block_type == 'output': + weight = weight * 0.2 + elif weight_type == 'strong middle' and (block_type == 'input' or block_type == 'output'): + weight = weight * 0.2 + elif weight_type.startswith('style transfer'): + if t_idx != 6: + weight = 0.0 + + ip_k = torch.cat([(k_cond, k_uncond)[i] for i in cond_or_uncond], dim=0) + ip_v = torch.cat([(v_cond, v_uncond)[i] for i in cond_or_uncond], dim=0) + + if embeds_scaling == 'K+mean(V) w/ C penalty': + scaling = float(ip_k.shape[2]) / 1280.0 + weight = weight * scaling + ip_k = ip_k * weight + ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True) + ip_v = (ip_v - ip_v_mean) + ip_v_mean * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + del ip_v_mean + elif embeds_scaling == 'K+V w/ C penalty': + scaling = float(ip_k.shape[2]) / 1280.0 + weight = weight * scaling + ip_k = ip_k * weight + ip_v = ip_v * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + elif embeds_scaling == 'K+V': + ip_k = ip_k * weight + ip_v = ip_v * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + else: + #ip_v = ip_v * weight + out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) + out_ip = out_ip * weight # I'm doing this to get the same results as before + + if mask is not None: + mask_h = oh / math.sqrt(oh * ow / seq_len) + mask_h = int(mask_h) + int((seq_len % int(mask_h)) != 0) + mask_w = seq_len // mask_h + + # check if using AnimateDiff and sliding context window + if (mask.shape[0] > 1 and ad_params is not None and ad_params["sub_idxs"] is not None): + # if mask length matches or exceeds full_length, get sub_idx masks + if mask.shape[0] >= ad_params["full_length"]: + mask = torch.Tensor(mask[ad_params["sub_idxs"]]) + mask = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bilinear").squeeze(1) + else: + mask = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bilinear").squeeze(1) + mask = tensor_to_size(mask, ad_params["full_length"]) + mask = mask[ad_params["sub_idxs"]] + else: + mask = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bilinear").squeeze(1) + mask = tensor_to_size(mask, batch_prompt) + + mask = mask.repeat(len(cond_or_uncond), 1, 1) + mask = mask.view(mask.shape[0], -1, 1).repeat(1, 1, out.shape[2]) + + # covers cases where extreme aspect ratios can cause the mask to have a wrong size + mask_len = mask_h * mask_w + if mask_len < seq_len: + pad_len = seq_len - mask_len + pad1 = pad_len // 2 + pad2 = pad_len - pad1 + mask = F.pad(mask, (0, 0, pad1, pad2), value=0.0) + elif mask_len > seq_len: + crop_start = (mask_len - seq_len) // 2 + mask = mask[:, crop_start:crop_start+seq_len, :] + + out_ip = out_ip * mask + + out = out + out_ip + + return out.to(dtype=dtype) diff --git a/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py new file mode 100644 index 0000000..b72b9d5 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py @@ -0,0 +1,605 @@ +import torch +import os +import math +import folder_paths + +import comfy.model_management as model_management +from comfy.clip_vision import load as load_clip_vision +from comfy.sd import load_lora_for_models +import comfy.utils + +import torch.nn as nn +from PIL import Image +try: + import torchvision.transforms.v2 as T +except ImportError: + import torchvision.transforms as T + +from .image_proj_models import MLPProjModelImport, MLPProjModelFaceIdImport, ProjModelFaceIdPlusImport, ResamplerImport, ImageProjModelImport +from .CrossAttentionPatchImport import CrossAttentionPatchImport +from .utils import ( + encode_image_masked, + tensor_to_size, + contrast_adaptive_sharpening, + tensor_to_image, + image_to_tensor, + ipadapter_model_loader, + insightface_loader, + get_clipvision_file, + get_ipadapter_file, + get_lora_file, +) + +# set the models directory +if "ipadapter" not in folder_paths.folder_names_and_paths: + current_paths = [os.path.join(folder_paths.models_dir, "ipadapter")] +else: + current_paths, _ = folder_paths.folder_names_and_paths["ipadapter"] +folder_paths.folder_names_and_paths["ipadapter"] = (current_paths, folder_paths.supported_pt_extensions) + +WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out', 'weak input', 'weak output', 'weak middle', 'strong middle', 'style transfer (SDXL)'] + +""" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Main IPAdapter Class +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +""" +class IPAdapterImport(nn.Module): + def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False): + super().__init__() + + self.clip_embeddings_dim = clip_embeddings_dim + self.cross_attention_dim = cross_attention_dim + self.output_cross_attention_dim = output_cross_attention_dim + self.clip_extra_context_tokens = clip_extra_context_tokens + self.is_sdxl = is_sdxl + self.is_full = is_full + self.is_plus = is_plus + + if is_faceid: + self.image_proj_model = self.init_proj_faceid() + elif is_full: + self.image_proj_model = self.init_proj_full() + elif is_plus: + self.image_proj_model = self.init_proj_plus() + else: + self.image_proj_model = self.init_proj() + + self.image_proj_model.load_state_dict(ipadapter_model["image_proj"]) + self.ip_layers = To_KV(ipadapter_model["ip_adapter"]) + + def init_proj(self): + image_proj_model = ImageProjModelImport( + cross_attention_dim=self.cross_attention_dim, + clip_embeddings_dim=self.clip_embeddings_dim, + clip_extra_context_tokens=self.clip_extra_context_tokens + ) + return image_proj_model + + def init_proj_plus(self): + image_proj_model = ResamplerImport( + dim=self.cross_attention_dim, + depth=4, + dim_head=64, + heads=20 if self.is_sdxl else 12, + num_queries=self.clip_extra_context_tokens, + embedding_dim=self.clip_embeddings_dim, + output_dim=self.output_cross_attention_dim, + ff_mult=4 + ) + return image_proj_model + + def init_proj_full(self): + image_proj_model = MLPProjModelImport( + cross_attention_dim=self.cross_attention_dim, + clip_embeddings_dim=self.clip_embeddings_dim + ) + return image_proj_model + + def init_proj_faceid(self): + if self.is_plus: + image_proj_model = ProjModelFaceIdPlusImport( + cross_attention_dim=self.cross_attention_dim, + id_embeddings_dim=512, + clip_embeddings_dim=self.clip_embeddings_dim, # 1280, + num_tokens=self.clip_extra_context_tokens, # 4, + ) + else: + image_proj_model = MLPProjModelFaceIdImport( + cross_attention_dim=self.cross_attention_dim, + id_embeddings_dim=512, + num_tokens=self.clip_extra_context_tokens, + ) + return image_proj_model + + @torch.inference_mode() + def get_image_embeds(self, clip_embed, clip_embed_zeroed): + image_prompt_embeds = self.image_proj_model(clip_embed) + uncond_image_prompt_embeds = self.image_proj_model(clip_embed_zeroed) + return image_prompt_embeds, uncond_image_prompt_embeds + + @torch.inference_mode() + def get_image_embeds_faceid_plus(self, face_embed, clip_embed, s_scale, shortcut): + embeds = self.image_proj_model(face_embed, clip_embed, scale=s_scale, shortcut=shortcut) + return embeds + +class To_KV(nn.Module): + def __init__(self, state_dict): + super().__init__() + + self.to_kvs = nn.ModuleDict() + for key, value in state_dict.items(): + self.to_kvs[key.replace(".weight", "").replace(".", "_")] = nn.Linear(value.shape[1], value.shape[0], bias=False) + self.to_kvs[key.replace(".weight", "").replace(".", "_")].weight.data = value + +def set_model_patch_replace(model, patch_kwargs, key): + to = model.model_options["transformer_options"] + if "patches_replace" not in to: + to["patches_replace"] = {} + if "attn2" not in to["patches_replace"]: + to["patches_replace"]["attn2"] = {} + if key not in to["patches_replace"]["attn2"]: + to["patches_replace"]["attn2"][key] = CrossAttentionPatchImport(**patch_kwargs) + else: + to["patches_replace"]["attn2"][key].set_new_condition(**patch_kwargs) + +def ipadapter_execute(model, + ipadapter, + clipvision, + insightface=None, + image=None, + image_negative=None, + weight=1.0, + weight_faceidv2=None, + weight_type="linear", + combine_embeds="concat", + start_at=0.0, + end_at=1.0, + attn_mask=None, + pos_embed=None, + neg_embed=None, + unfold_batch=False, + embeds_scaling='V only'): + dtype = torch.float16 if model_management.should_use_fp16() else torch.bfloat16 if model_management.should_use_bf16() else torch.float32 + device = model_management.get_torch_device() + + is_full = "proj.3.weight" in ipadapter["image_proj"] + is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] + is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] + is_plus = is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"] + is_faceidv2 = "faceidplusv2" in ipadapter + output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1] + is_sdxl = output_cross_attention_dim == 2048 + + if weight_type == "style transfer (SDXL)" and not is_sdxl: + weight_type = "linear" + print("\033[33mINFO: 'Style Transfer' weight type is only available for SDXL models, falling back to 'linear'.\033[0m") + + if is_faceid and not insightface: + raise Exception("insightface model is required for FaceID models") + + if is_faceidv2: + weight_faceidv2 = weight_faceidv2 if weight_faceidv2 is not None else weight*2 + + cross_attention_dim = 1280 if is_plus and is_sdxl and not is_faceid else output_cross_attention_dim + clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait else 4 + + if image is not None and image.shape[1] != image.shape[2]: + print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m") + + face_cond_embeds = None + if is_faceid: + if insightface is None: + raise Exception("Insightface model is required for FaceID models") + + from insightface.utils import face_align + + insightface.det_model.input_size = (640,640) # reset the detection size + image_iface = tensor_to_image(image) + face_cond_embeds = [] + image = [] + + for i in range(image_iface.shape[0]): + for size in [(size, size) for size in range(640, 256, -64)]: + insightface.det_model.input_size = size # TODO: hacky but seems to be working + face = insightface.get(image_iface[i]) + if face: + face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0)) + image.append(image_to_tensor(face_align.norm_crop(image_iface[i], landmark=face[0].kps, image_size=256))) + + if 640 not in size: + print(f"\033[33mINFO: InsightFace detection resolution lowered to {size}.\033[0m") + break + else: + raise Exception('InsightFace: No face detected.') + face_cond_embeds = torch.stack(face_cond_embeds).to(device, dtype=dtype) + image = torch.stack(image) + del image_iface, face + + if image is not None: + img_cond_embeds = encode_image_masked(clipvision, image) + + if is_plus: + img_cond_embeds = img_cond_embeds.penultimate_hidden_states + image_negative = image_negative if image_negative is not None else torch.zeros([1, 224, 224, 3]) + img_uncond_embeds = encode_image_masked(clipvision, image_negative).penultimate_hidden_states + else: + img_cond_embeds = img_cond_embeds.image_embeds if not is_faceid else face_cond_embeds + if image_negative is not None: + img_uncond_embeds = encode_image_masked(clipvision, image_negative).image_embeds + else: + img_uncond_embeds = torch.zeros_like(img_cond_embeds) + elif pos_embed is not None: + img_cond_embeds = pos_embed + + if neg_embed is not None: + img_uncond_embeds = neg_embed + else: + if is_plus: + img_uncond_embeds = encode_image_masked(clipvision, torch.zeros([1, 224, 224, 3])).penultimate_hidden_states + else: + img_uncond_embeds = torch.zeros_like(img_cond_embeds) + else: + raise Exception("Images or Embeds are required") + + # ensure that cond and uncond have the same batch size + img_uncond_embeds = tensor_to_size(img_uncond_embeds, img_cond_embeds.shape[0]) + + img_cond_embeds = img_cond_embeds.to(device, dtype=dtype) + img_uncond_embeds = img_uncond_embeds.to(device, dtype=dtype) + + # combine the embeddings if needed + if combine_embeds != "concat" and img_cond_embeds.shape[0] > 1 and not unfold_batch: + if combine_embeds == "add": + img_cond_embeds = torch.sum(img_cond_embeds, dim=0).unsqueeze(0) + if face_cond_embeds is not None: + face_cond_embeds = torch.sum(face_cond_embeds, dim=0).unsqueeze(0) + elif combine_embeds == "subtract": + img_cond_embeds = img_cond_embeds[0] - torch.mean(img_cond_embeds[1:], dim=0) + img_cond_embeds = img_cond_embeds.unsqueeze(0) + if face_cond_embeds is not None: + face_cond_embeds = face_cond_embeds[0] - torch.mean(face_cond_embeds[1:], dim=0) + face_cond_embeds = face_cond_embeds.unsqueeze(0) + elif combine_embeds == "average": + img_cond_embeds = torch.mean(img_cond_embeds, dim=0).unsqueeze(0) + if face_cond_embeds is not None: + face_cond_embeds = torch.mean(face_cond_embeds, dim=0).unsqueeze(0) + elif combine_embeds == "norm average": + img_cond_embeds = torch.mean(img_cond_embeds / torch.norm(img_cond_embeds, dim=0, keepdim=True), dim=0).unsqueeze(0) + if face_cond_embeds is not None: + face_cond_embeds = torch.mean(face_cond_embeds / torch.norm(face_cond_embeds, dim=0, keepdim=True), dim=0).unsqueeze(0) + img_uncond_embeds = img_uncond_embeds[0].unsqueeze(0) # TODO: better strategy for uncond could be to average them + + if attn_mask is not None: + attn_mask = attn_mask.to(device, dtype=dtype) + + ipa = IPAdapterImport( + ipadapter, + cross_attention_dim=cross_attention_dim, + output_cross_attention_dim=output_cross_attention_dim, + clip_embeddings_dim=img_cond_embeds.shape[-1], + clip_extra_context_tokens=clip_extra_context_tokens, + is_sdxl=is_sdxl, + is_plus=is_plus, + is_full=is_full, + is_faceid=is_faceid + ).to(device, dtype=dtype) + + if is_faceid and is_plus: + cond = ipa.get_image_embeds_faceid_plus(face_cond_embeds, img_cond_embeds, weight_faceidv2, is_faceidv2) + # TODO: check if noise helps with the uncod face embeds + uncod = ipa.get_image_embeds_faceid_plus(torch.zeros_like(face_cond_embeds), img_uncond_embeds, weight_faceidv2, is_faceidv2) + else: + cond, uncod = ipa.get_image_embeds(img_cond_embeds, img_uncond_embeds) + + cond = cond.to(device, dtype=dtype) + uncod = uncod.to(device, dtype=dtype) + + del img_cond_embeds, img_uncond_embeds + + sigma_start = model.model.model_sampling.percent_to_sigma(start_at) + sigma_end = model.model.model_sampling.percent_to_sigma(end_at) + + patch_kwargs = { + "ipadapter": ipa, + "number": 0, + "weight": weight, + "cond": cond, + "uncond": uncod, + "weight_type": weight_type, + "mask": attn_mask, + "sigma_start": sigma_start, + "sigma_end": sigma_end, + "unfold_batch": unfold_batch, + "embeds_scaling": embeds_scaling, + } + + if not is_sdxl: + for id in [1,2,4,5,7,8]: # id of input_blocks that have cross attention + set_model_patch_replace(model, patch_kwargs, ("input", id)) + patch_kwargs["number"] += 1 + for id in [3,4,5,6,7,8,9,10,11]: # id of output_blocks that have cross attention + set_model_patch_replace(model, patch_kwargs, ("output", id)) + patch_kwargs["number"] += 1 + set_model_patch_replace(model, patch_kwargs, ("middle", 0)) + else: + for id in [4,5,7,8]: # id of input_blocks that have cross attention + block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth + for index in block_indices: + set_model_patch_replace(model, patch_kwargs, ("input", id, index)) + patch_kwargs["number"] += 1 + for id in range(6): # id of output_blocks that have cross attention + block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth + for index in block_indices: + set_model_patch_replace(model, patch_kwargs, ("output", id, index)) + patch_kwargs["number"] += 1 + for index in range(10): + set_model_patch_replace(model, patch_kwargs, ("middle", 0, index)) + patch_kwargs["number"] += 1 + + return model + + +class IPAdapterAdvancedImport: + def __init__(self): + self.unfold_batch = False + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL", ), + "ipadapter": ("IPADAPTER", ), + "image": ("IMAGE",), + "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), + "weight_type": (WEIGHT_TYPES, ), + "combine_embeds": (["concat", "add", "subtract", "average", "norm average"],), + "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), + "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), + "embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ), + }, + "optional": { + "image_negative": ("IMAGE",), + "attn_mask": ("MASK",), + "clip_vision": ("CLIP_VISION",), + } + } + + RETURN_TYPES = ("MODEL",) + FUNCTION = "apply_ipadapter" + CATEGORY = "ipadapter" + + def apply_ipadapter(self, model, ipadapter, image, weight, weight_type, start_at, end_at, combine_embeds="concat", weight_faceidv2=None, image_negative=None, clip_vision=None, attn_mask=None, insightface=None, embeds_scaling='V only'): + ipa_args = { + "image": image, + "image_negative": image_negative, + "weight": weight, + "weight_faceidv2": weight_faceidv2, + "weight_type": weight_type, + "combine_embeds": combine_embeds, + "start_at": start_at, + "end_at": end_at, + "attn_mask": attn_mask, + "unfold_batch": self.unfold_batch, + "embeds_scaling": embeds_scaling, + "insightface": insightface if insightface is not None else ipadapter['insightface']['model'] if 'insightface' in ipadapter else None + } + + if 'ipadapter' in ipadapter: + ipadapter_model = ipadapter['ipadapter']['model'] + clip_vision = clip_vision if clip_vision is not None else ipadapter['clipvision']['model'] + else: + ipadapter_model = ipadapter + clip_vision = clip_vision + + if clip_vision is None: + raise Exception("Missing CLIPVision model.") + + del ipadapter + + return (ipadapter_execute(model.clone(), ipadapter_model, clip_vision, **ipa_args), ) + + + + +class IPAdapterTiledImport: + def __init__(self): + self.unfold_batch = False + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL", ), + "ipadapter": ("IPADAPTER", ), + "image": ("IMAGE",), + "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), + "weight_type": (WEIGHT_TYPES, ), + "combine_embeds": (["concat", "add", "subtract", "average", "norm average"],), + "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), + "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), + "sharpening": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05 }), + "embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ), + }, + "optional": { + "image_negative": ("IMAGE",), + "attn_mask": ("MASK",), + "clip_vision": ("CLIP_VISION",), + } + } + + RETURN_TYPES = ("MODEL", "IMAGE", "MASK", ) + RETURN_NAMES = ("MODEL", "tiles", "masks", ) + FUNCTION = "apply_tiled" + CATEGORY = "ipadapter" + + def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only'): + # 1. Select the models + if 'ipadapter' in ipadapter: + ipadapter_model = ipadapter['ipadapter']['model'] + clip_vision = clip_vision if clip_vision is not None else ipadapter['clipvision']['model'] + else: + ipadapter_model = ipadapter + clip_vision = clip_vision + + if clip_vision is None: + raise Exception("Missing CLIPVision model.") + + del ipadapter + + # 2. Extract the tiles + tile_size = 256 # I'm using 256 instead of 224 as it is more likely divisible by the latent size, it will be downscaled to 224 by the clip vision encoder + _, oh, ow, _ = image.shape + if attn_mask is None: + attn_mask = torch.ones([1, oh, ow], dtype=image.dtype, device=image.device) + + image = image.permute([0,3,1,2]) + attn_mask = attn_mask.unsqueeze(1) + # the mask should have the same proportions as the reference image and the latent + attn_mask = T.Resize((oh, ow), interpolation=T.InterpolationMode.BICUBIC, antialias=True)(attn_mask) + + # if the image is almost a square, we crop it to a square + if oh / ow > 0.75 and oh / ow < 1.33: + # crop the image to a square + image = T.CenterCrop(min(oh, ow))(image) + resize = (tile_size*2, tile_size*2) + + attn_mask = T.CenterCrop(min(oh, ow))(attn_mask) + # otherwise resize the smallest side and the other proportionally + else: + resize = (int(tile_size * ow / oh), tile_size) if oh < ow else (tile_size, int(tile_size * oh / ow)) + + # using PIL for better results + imgs = [] + for img in image: + img = T.ToPILImage()(img) + img = img.resize(resize, resample=Image.Resampling['LANCZOS']) + imgs.append(T.ToTensor()(img)) + image = torch.stack(imgs) + del imgs, img + + # we don't need a high quality resize for the mask + attn_mask = T.Resize(resize[::-1], interpolation=T.InterpolationMode.BICUBIC, antialias=True)(attn_mask) + + # we allow a maximum of 4 tiles + if oh / ow > 4 or oh / ow < 0.25: + crop = (tile_size, tile_size*4) if oh < ow else (tile_size*4, tile_size) + image = T.CenterCrop(crop)(image) + attn_mask = T.CenterCrop(crop)(attn_mask) + + attn_mask = attn_mask.squeeze(1) + + if sharpening > 0: + image = contrast_adaptive_sharpening(image, sharpening) + + image = image.permute([0,2,3,1]) + + _, oh, ow, _ = image.shape + + # find the number of tiles for each side + tiles_x = math.ceil(ow / tile_size) + tiles_y = math.ceil(oh / tile_size) + overlap_x = max(0, (tiles_x * tile_size - ow) / (tiles_x - 1 if tiles_x > 1 else 1)) + overlap_y = max(0, (tiles_y * tile_size - oh) / (tiles_y - 1 if tiles_y > 1 else 1)) + + base_mask = torch.zeros([attn_mask.shape[0], oh, ow], dtype=image.dtype, device=image.device) + + # extract all the tiles from the image and create the masks + tiles = [] + masks = [] + for y in range(tiles_y): + for x in range(tiles_x): + start_x = int(x * (tile_size - overlap_x)) + start_y = int(y * (tile_size - overlap_y)) + tiles.append(image[:, start_y:start_y+tile_size, start_x:start_x+tile_size, :]) + mask = base_mask.clone() + mask[:, start_y:start_y+tile_size, start_x:start_x+tile_size] = attn_mask[:, start_y:start_y+tile_size, start_x:start_x+tile_size] + masks.append(mask) + del mask + + # 3. Apply the ipadapter to each group of tiles + model = model.clone() + for i in range(len(tiles)): + ipa_args = { + "image": tiles[i], + "image_negative": image_negative, + "weight": weight, + "weight_type": weight_type, + "combine_embeds": combine_embeds, + "start_at": start_at, + "end_at": end_at, + "attn_mask": masks[i], + "unfold_batch": self.unfold_batch, + "embeds_scaling": embeds_scaling, + } + # apply the ipadapter to the model without cloning it + model = ipadapter_execute(model, ipadapter_model, clip_vision, **ipa_args) + + return (model, torch.cat(tiles), torch.cat(masks), ) + + + +class PrepImageForClipVisionImport: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "image": ("IMAGE",), + "interpolation": (["LANCZOS", "BICUBIC", "HAMMING", "BILINEAR", "BOX", "NEAREST"],), + "crop_position": (["top", "bottom", "left", "right", "center", "pad"],), + "sharpening": ("FLOAT", {"default": 0.0, "min": 0, "max": 1, "step": 0.05}), + }, + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "prep_image" + + CATEGORY = "ipadapter" + + def prep_image(self, image, interpolation="LANCZOS", crop_position="center", sharpening=0.0): + size = (224, 224) + _, oh, ow, _ = image.shape + output = image.permute([0,3,1,2]) + + if crop_position == "pad": + if oh != ow: + if oh > ow: + pad = (oh - ow) // 2 + pad = (pad, 0, pad, 0) + elif ow > oh: + pad = (ow - oh) // 2 + pad = (0, pad, 0, pad) + output = T.functional.pad(output, pad, fill=0) + else: + crop_size = min(oh, ow) + x = (ow-crop_size) // 2 + y = (oh-crop_size) // 2 + if "top" in crop_position: + y = 0 + elif "bottom" in crop_position: + y = oh-crop_size + elif "left" in crop_position: + x = 0 + elif "right" in crop_position: + x = ow-crop_size + + x2 = x+crop_size + y2 = y+crop_size + + output = output[:, :, y:y2, x:x2] + + imgs = [] + for img in output: + img = T.ToPILImage()(img) # using PIL for better results + img = img.resize(size, resample=Image.Resampling[interpolation]) + imgs.append(T.ToTensor()(img)) + output = torch.stack(imgs, dim=0) + del imgs, img + + if sharpening > 0: + output = contrast_adaptive_sharpening(output, sharpening) + + output = output.permute([0,2,3,1]) + + return (output, ) + + diff --git a/imports/ComfyUI_IPAdapter_plus/LICENSE b/imports/ComfyUI_IPAdapter_plus/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/imports/ComfyUI_IPAdapter_plus/README.md b/imports/ComfyUI_IPAdapter_plus/README.md new file mode 100644 index 0000000..f7fed54 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/README.md @@ -0,0 +1,126 @@ +# ComfyUI IPAdapter plus +[ComfyUI](https://github.com/comfyanonymous/ComfyUI) reference implementation for [IPAdapter](https://github.com/tencent-ailab/IP-Adapter/) models. + +IPAdapter implementation that follows the ComfyUI way of doing things. The code is memory efficient, fast, and shouldn't break with Comfy updates. + +# Open source for you but not free for me... + +I started working on IPAdapter because I needed it for my work. As the project evolved I'm inevitably receiving feature requests, bug reports and support requests. + +I'm an open source advocate and I'm happy to share all my code for free but maintaining the IPAdapter, the [Essentials](https://github.com/cubiq/ComfyUI_essentials), [InstantID](https://github.com/cubiq/ComfyUI_InstantID) and [Face Analysis](https://github.com/cubiq/ComfyUI_FaceAnalysis) takes time. + +**I'm not expecting donations but if you are making a profit from my projects it is only fair that you give something back.** I'm talking especially to companies here, I know the struggles of being a freelancer. + +Please contact me if you are interested in a sponsorship at _matt3o@gmail_ or consider a contribution via [PayPal](https://paypal.me/matt3o) (Matteo "matt3o" Spinelli, Firenze, IT). That will help maintaining the code, adding new features and working on better documentation. + +And in that regard I really need to thank [Nathan Shipley](https://www.nathanshipley.com/) for his generous donation. Go check his website, he's terribly talented. + +## :warning: IPAdapter V2: complete Code rewrite warning + +A code cleanup was long overdue and with the occasion I also added a few new important features. The code should be faster and should take less resources but with such an important code rewrite it's inevitable to have introduced some new bugs. + +**At the moment I'm releasing this completely undocumented!** I will post better documentation and video tutorials in the coming days. In the meantime you can check the `example` directory for most of the old and new features. + +## Important updates + +**2024/03/23**: Complete code rewrite!. **This is a breaking update!** Your previous workflows won't work and you'll need to recreate them. You've been warned! After the update, refresh your browser, delete the old IPAdapter nodes and create the new ones. + +**2024/02/02**: Added experimental [tiled IPAdapter](#tiled-ipadapter). It lets you easily handle reference images that are not square. Can be useful for upscaling. + +**2024/01/19**: Support for FaceID Portrait models. + +**2024/01/16**: Notably increased quality of FaceID Plus/v2 models. Check the [comparison](https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/195) of all face models. + +*(previous updates removed for better readability)* + +## What is it? + +The IPAdapter are very powerful models for image-to-image conditioning. Given one or more reference images you can do variations augmented by text prompt, controlnets and masks. Think of it as a 1-image lora. + +## Example workflow + +The [example directory](./examples/) has many workflows that cover all IPAdapter functionalities. + +![IPAdapter Example workflow](./examples/demo_workflow.jpg) + +## Video Tutorials + + + Watch the video + + +**:star: [New IPAdapter features](https://youtu.be/_JzDcgKgghY)** + +The following videos are about the previous version of IPAdapter, but they still contain valuable information. + +**:nerd_face: [Basic usage video](https://youtu.be/7m9ZZFU3HWo)** + +**:rocket: [Advanced features video](https://www.youtube.com/watch?v=mJQ62ly7jrg)** + +**:japanese_goblin: [Attention Masking video](https://www.youtube.com/watch?v=vqG1VXKteQg)** + +**:movie_camera: [Animation Features video](https://www.youtube.com/watch?v=ddYbhv3WgWw)** + +## Installation + +Download or git clone this repository inside `ComfyUI/custom_nodes/` directory or use the Manager. Beware that the automatic update of the manager sometimes doesn't work and you may need to upgrade manually. + +IPAdapter always requires the latest version of ComfyUI. If something doesn't work be sure to upgrade! + +There's now an *Unified Model Loader*, for it to work you need to name the files exactly how it is described below. + +The pre-trained models are available on [huggingface](https://huggingface.co/h94/IP-Adapter), download and place them in the `ComfyUI/models/ipadapter` directory (create it if not present). You can also use any custom location setting an `ipadapter` entry in the `extra_model_paths.yaml` file. + +IPAdapter also needs the image encoders. You need the [CLIP-ViT-H-14-laion2B-s32B-b79K.safetensors](https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors) and [CLIP-ViT-bigG-14-laion2B-39B-b160k.safetensors](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/image_encoder/model.safetensors) image encoders, you may already have them. If you don't, download them but **be careful because the file name is the same for both!** Rename them and place them in the `ComfyUI/models/clip_vision/` directory. + +The following table shows the combination of Checkpoint and Image encoder to use for each IPAdapter Model. Any Tensor size mismatch you may get it is likely caused by a wrong combination. + +| SD v. | IPadapter | Img encoder | Notes | +|---|---|---|---| +| v1.5 | [ip-adapter_sd15](https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter_sd15.safetensors) | ViT-H | Basic model, average strength | +| v1.5 | [ip-adapter_sd15_light](https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter_sd15_light.safetensors) | ViT-H | Light model, very light impact | +| v1.5 | [ip-adapter_sd15_light_v11](https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter_sd15_light_v11.bin) | ViT-H | Updated light model | +| v1.5 | [ip-adapter-plus_sd15](https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter-plus_sd15.safetensors) | ViT-H | Plus model, very strong | +| v1.5 | [ip-adapter-plus-face_sd15](https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter-plus-face_sd15.safetensors) | ViT-H | Face model, use only for faces | +| v1.5 | [ip-adapter-full-face_sd15](https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter-full-face_sd15.safetensors) | ViT-H | Stronger face model, not necessarily better | +| v1.5 | [ip-adapter_sd15_vit-G](https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter_sd15_vit-G.safetensors) | ViT-bigG | Base model trained with a bigG encoder | +| SDXL | [ip-adapter_sdxl](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/ip-adapter_sdxl.safetensors) | ViT-bigG | Base SDXL model, mostly deprecated | +| SDXL | [ip-adapter_sdxl_vit-h](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/ip-adapter_sdxl_vit-h.safetensors) | ViT-H | New base SDXL model | +| SDXL | [ip-adapter-plus_sdxl_vit-h](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors) | ViT-H | SDXL plus model, stronger | +| SDXL | [ip-adapter-plus-face_sdxl_vit-h](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/ip-adapter-plus-face_sdxl_vit-h.safetensors) | ViT-H | SDXL face model | + +**FaceID** requires `insightface`, you need to install them in your ComfyUI environment. Check [this issue](https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/162) for help. + +When the dependencies are satisfied you need: + +| SD v. | IPadapter | Img encoder | Lora | +|---|---|---|---| +| v1.5 | [FaceID](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid_sd15.bin) | (not used¹) | [FaceID Lora](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid_sd15_lora.safetensors) | +| v1.5 | [FaceID Plus](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plus_sd15.bin) | ViT-H | [FaceID Plus Lora](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plus_sd15_lora.safetensors) | +| v1.5 | [FaceID Plus v2](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plusv2_sd15.bin) | ViT-H | [FaceID Plus v2 Lora](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plusv2_sd15_lora.safetensors) | +| v1.5 | [FaceID Portrait](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-portrait_sd15.bin) | (not used¹)| not needed | +| SDXL | [FaceID](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid_sdxl.bin) | (not used¹) | [FaceID SDXL Lora](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid_sdxl_lora.safetensors) | +| SDXL | [FaceID Plus v2](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plusv2_sdxl.bin) | ViT-H | [FaceID SDXL Lora](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plusv2_sdxl_lora.safetensors) | + + +¹ The base FaceID model doesn't make use of a CLIP vision encoder. Remember to pair any FaceID model together with any other Face model to make it more effective. + +The loras need to be placed into `ComfyUI/models/loras/` directory. + +## Generic suggestions + +There's a basic workflow included in this repo and a few examples in the [examples](./examples/) directory. Usually it's a good idea to lower the `weight` to at least `0.8` and increase the steps a little. + +## Documentation soon to come... + +Working on it! + +## Troubleshooting + +Please check the [troubleshooting](https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/108) before posting a new issue. Alse remember to check the previous closed issues. + +## Credits + +- [IPAdapter](https://github.com/tencent-ailab/IP-Adapter/) +- [ComfyUI](https://github.com/comfyanonymous/ComfyUI) +- [laksjdjf](https://github.com/laksjdjf/IPAdapter-ComfyUI/) diff --git a/imports/ComfyUI_IPAdapter_plus/__init__.py b/imports/ComfyUI_IPAdapter_plus/__init__.py new file mode 100644 index 0000000..03a9149 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/__init__.py @@ -0,0 +1,19 @@ +""" + ██▓ ██▓███ ▄▄▄ ▓█████▄ ▄▄▄ ██▓███ ▄▄▄█████▓▓█████ ██▀███ +▓██▒▓██░ ██▒▒████▄ ▒██▀ ██▌▒████▄ ▓██░ ██▒▓ ██▒ ▓▒▓█ ▀ ▓██ ▒ ██▒ +▒██▒▓██░ ██▓▒▒██ ▀█▄ ░██ █▌▒██ ▀█▄ ▓██░ ██▓▒▒ ▓██░ ▒░▒███ ▓██ ░▄█ ▒ +░██░▒██▄█▓▒ ▒░██▄▄▄▄██ ░▓█▄ ▌░██▄▄▄▄██ ▒██▄█▓▒ ▒░ ▓██▓ ░ ▒▓█ ▄ ▒██▀▀█▄ +░██░▒██▒ ░ ░ ▓█ ▓██▒░▒████▓ ▓█ ▓██▒▒██▒ ░ ░ ▒██▒ ░ ░▒████▒░██▓ ▒██▒ +░▓ ▒▓▒░ ░ ░ ▒▒ ▓▒█░ ▒▒▓ ▒ ▒▒ ▓▒█░▒▓▒░ ░ ░ ▒ ░░ ░░ ▒░ ░░ ▒▓ ░▒▓░ + ▒ ░░▒ ░ ▒ ▒▒ ░ ░ ▒ ▒ ▒ ▒▒ ░░▒ ░ ░ ░ ░ ░ ░▒ ░ ▒░ + ▒ ░░░ ░ ▒ ░ ░ ░ ░ ▒ ░░ ░ ░ ░░ ░ + ░ ░ ░ ░ ░ ░ ░ ░ ░ + ░ + ·-—+ IPAdapter Plus Extension for ComfyUI +—-· · + Brought to you by Matteo "Matt3o/Cubiq" Spinelli + https://github.com/cubiq/ComfyUI_IPAdapter_plus/ +""" + +from .IPAdapterPlus import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] diff --git a/imports/ComfyUI_IPAdapter_plus/examples/demo_workflow.jpg b/imports/ComfyUI_IPAdapter_plus/examples/demo_workflow.jpg new file mode 100644 index 0000000..d30bb7c Binary files /dev/null and b/imports/ComfyUI_IPAdapter_plus/examples/demo_workflow.jpg differ diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_advanced.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_advanced.json new file mode 100644 index 0000000..ae712dd --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_advanced.json @@ -0,0 +1,618 @@ +{ + "last_node_id": 17, + "last_link_id": 26, + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 50, + 730 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 20 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1770, + 710 + ], + "size": [ + 529.7760009765616, + 582.3048192804504 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 690, + 840 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1570, + 700 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 16, + "type": "CLIPVisionLoader", + "pos": [ + 308, + 161 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 24 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ] + }, + { + "id": 15, + "type": "IPAdapterModelLoader", + "pos": [ + 308, + 52 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 21 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.safetensors" + ] + }, + { + "id": 14, + "type": "IPAdapterAdvanced", + "pos": [ + 793, + 304 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 20 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 21, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 26 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 24, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 17, + "type": "PrepImageForClipVision", + "pos": [ + 798, + 145 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 25 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 26 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PrepImageForClipVision" + }, + "widgets_values": [ + "LANCZOS", + "top", + 0.15 + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 311, + 270 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "girl_sitting.png", + "image" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 690, + 610 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "in a peaceful spring morning a woman wearing a white shirt is sitting in a park on a bench\n\nhigh quality, detailed, diffuse light" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1210, + 700 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 23 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "ddpm", + "karras", + 1 + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 20, + 4, + 0, + 14, + 0, + "MODEL" + ], + [ + 21, + 15, + 0, + 14, + 1, + "IPADAPTER" + ], + [ + 23, + 14, + 0, + 3, + 0, + "MODEL" + ], + [ + 24, + 16, + 0, + 14, + 5, + "CLIP_VISION" + ], + [ + 25, + 12, + 0, + 17, + 0, + "IMAGE" + ], + [ + 26, + 17, + 0, + 14, + 2, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_combine_embeds.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_combine_embeds.json new file mode 100644 index 0000000..f961f67 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_combine_embeds.json @@ -0,0 +1,1538 @@ +{ + "last_node_id": 51, + "last_link_id": 118, + "nodes": [ + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2, + 89, + 100, + 111 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 16, + "type": "CLIPVisionLoader", + "pos": [ + 650, + 80 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 24, + 96, + 107, + 118 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1580, + 300 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 23 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 300, + 700 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 20, + 93, + 104, + 115 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8, + 91, + 102, + 113 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 240, + -60 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 83 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "warrior_woman.png", + "image" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 740, + 780 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6, + 88, + 99, + 110 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed" + ] + }, + { + "id": 14, + "type": "IPAdapterAdvanced", + "pos": [ + 1220, + 260 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 20 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 21, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 85 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 24, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 42, + "type": "SaveImage", + "pos": [ + 1950, + 1070 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 92 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 40, + "type": "KSampler", + "pos": [ + 1580, + 1070 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 86 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 87 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 88 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 89 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 90 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1950, + 420 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 740, + 550 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4, + 87, + 98, + 109 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup of a fierce warrior woman wearing a full armor at the end of a battle\n\nhigh quality, detailed" + ] + }, + { + "id": 15, + "type": "IPAdapterModelLoader", + "pos": [ + 650, + -20 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 21, + 94, + 105, + 116 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.safetensors" + ] + }, + { + "id": 39, + "type": "ImageBatch", + "pos": [ + 714, + 287 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 83 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 84 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 85, + 95, + 106, + 117 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageBatch" + } + }, + { + "id": 38, + "type": "LoadImage", + "pos": [ + 240, + 310 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 84 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "anime_illustration.png", + "image" + ] + }, + { + "id": 43, + "type": "IPAdapterAdvanced", + "pos": [ + 1220, + 1030 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 93 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 94, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 95 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 96, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 86 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "linear", + "add", + 0, + 1 + ] + }, + { + "id": 44, + "type": "KSampler", + "pos": [ + 3470, + 390 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 97 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 98 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 99 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 100 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 101 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 45, + "type": "VAEDecode", + "pos": [ + 3840, + 400 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 101 + }, + { + "name": "vae", + "type": "VAE", + "link": 102 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 103 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 48, + "type": "KSampler", + "pos": [ + 3480, + 1170 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 108 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 109 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 110 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 111 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 112 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 49, + "type": "VAEDecode", + "pos": [ + 3850, + 1180 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 112 + }, + { + "name": "vae", + "type": "VAE", + "link": 113 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 114 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 51, + "type": "IPAdapterAdvanced", + "pos": [ + 3120, + 1130 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 115 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 116, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 117 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 118, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 108 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "linear", + "norm average", + 0, + 1 + ] + }, + { + "id": 47, + "type": "IPAdapterAdvanced", + "pos": [ + 3110, + 350 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 104 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 105, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 106 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 107, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 97 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "linear", + "average", + 0, + 1 + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1748, + 613 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 41, + "type": "VAEDecode", + "pos": [ + 1746, + 1386 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 90 + }, + { + "name": "vae", + "type": "VAE", + "link": 91 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 92 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 50, + "type": "SaveImage", + "pos": [ + 2530, + 1073 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 114 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 46, + "type": "SaveImage", + "pos": [ + 2518, + 419 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 103 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 20, + 4, + 0, + 14, + 0, + "MODEL" + ], + [ + 21, + 15, + 0, + 14, + 1, + "IPADAPTER" + ], + [ + 23, + 14, + 0, + 3, + 0, + "MODEL" + ], + [ + 24, + 16, + 0, + 14, + 5, + "CLIP_VISION" + ], + [ + 83, + 12, + 0, + 39, + 0, + "IMAGE" + ], + [ + 84, + 38, + 0, + 39, + 1, + "IMAGE" + ], + [ + 85, + 39, + 0, + 14, + 2, + "IMAGE" + ], + [ + 86, + 43, + 0, + 40, + 0, + "MODEL" + ], + [ + 87, + 6, + 0, + 40, + 1, + "CONDITIONING" + ], + [ + 88, + 7, + 0, + 40, + 2, + "CONDITIONING" + ], + [ + 89, + 5, + 0, + 40, + 3, + "LATENT" + ], + [ + 90, + 40, + 0, + 41, + 0, + "LATENT" + ], + [ + 91, + 4, + 2, + 41, + 1, + "VAE" + ], + [ + 92, + 41, + 0, + 42, + 0, + "IMAGE" + ], + [ + 93, + 4, + 0, + 43, + 0, + "MODEL" + ], + [ + 94, + 15, + 0, + 43, + 1, + "IPADAPTER" + ], + [ + 95, + 39, + 0, + 43, + 2, + "IMAGE" + ], + [ + 96, + 16, + 0, + 43, + 5, + "CLIP_VISION" + ], + [ + 97, + 47, + 0, + 44, + 0, + "MODEL" + ], + [ + 98, + 6, + 0, + 44, + 1, + "CONDITIONING" + ], + [ + 99, + 7, + 0, + 44, + 2, + "CONDITIONING" + ], + [ + 100, + 5, + 0, + 44, + 3, + "LATENT" + ], + [ + 101, + 44, + 0, + 45, + 0, + "LATENT" + ], + [ + 102, + 4, + 2, + 45, + 1, + "VAE" + ], + [ + 103, + 45, + 0, + 46, + 0, + "IMAGE" + ], + [ + 104, + 4, + 0, + 47, + 0, + "MODEL" + ], + [ + 105, + 15, + 0, + 47, + 1, + "IPADAPTER" + ], + [ + 106, + 39, + 0, + 47, + 2, + "IMAGE" + ], + [ + 107, + 16, + 0, + 47, + 5, + "CLIP_VISION" + ], + [ + 108, + 51, + 0, + 48, + 0, + "MODEL" + ], + [ + 109, + 6, + 0, + 48, + 1, + "CONDITIONING" + ], + [ + 110, + 7, + 0, + 48, + 2, + "CONDITIONING" + ], + [ + 111, + 5, + 0, + 48, + 3, + "LATENT" + ], + [ + 112, + 48, + 0, + 49, + 0, + "LATENT" + ], + [ + 113, + 4, + 2, + 49, + 1, + "VAE" + ], + [ + 114, + 49, + 0, + 50, + 0, + "IMAGE" + ], + [ + 115, + 4, + 0, + 51, + 0, + "MODEL" + ], + [ + 116, + 15, + 0, + 51, + 1, + "IPADAPTER" + ], + [ + 117, + 39, + 0, + 51, + 2, + "IMAGE" + ], + [ + 118, + 16, + 0, + 51, + 5, + "CLIP_VISION" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_faceid.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_faceid.json new file mode 100644 index 0000000..99d0bbe --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_faceid.json @@ -0,0 +1,566 @@ +{ + "last_node_id": 20, + "last_link_id": 36, + "nodes": [ + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1640, + 710 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 870, + 1100 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1280, + 710 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 32 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "ddpm", + "karras", + 1 + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1830, + 700 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 450, + 240 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 29 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "rosario_4.jpg", + "image" + ] + }, + { + "id": 20, + "type": "IPAdapterUnifiedLoaderFaceID", + "pos": [ + 460, + 60 + ], + "size": { + "0": 315, + "1": 126 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 36 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 35 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 34 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoaderFaceID" + }, + "widgets_values": [ + "FACEID PLUS V2", + 0.6, + "CPU" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 760, + 850 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed, naked" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 760, + 620 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup of a beautiful woman wearing a black dress on the seaside\n\nserene, sunset, spring, high quality, detailed, diffuse light" + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 10, + 680 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 36 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 18, + "type": "IPAdapterFaceID", + "pos": [ + 850, + 190 + ], + "size": { + "0": 315, + "1": 298 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 35 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 34, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 29 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + }, + { + "name": "insightface", + "type": "INSIGHTFACE", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 32 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterFaceID" + }, + "widgets_values": [ + 1, + 2, + "linear", + "concat", + 0, + 1 + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 29, + 12, + 0, + 18, + 2, + "IMAGE" + ], + [ + 32, + 18, + 0, + 3, + 0, + "MODEL" + ], + [ + 34, + 20, + 1, + 18, + 1, + "IPADAPTER" + ], + [ + 35, + 20, + 0, + 18, + 0, + "MODEL" + ], + [ + 36, + 4, + 0, + 20, + 0, + "MODEL" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_faceid_batch.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_faceid_batch.json new file mode 100644 index 0000000..3dd6685 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_faceid_batch.json @@ -0,0 +1,1168 @@ +{ + "last_node_id": 32, + "last_link_id": 54, + "nodes": [ + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1640, + 710 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 870, + 1100 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1280, + 710 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 42 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "ddpm", + "karras", + 1 + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1830, + 700 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 760, + 850 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed, naked" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 760, + 620 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup of a beautiful woman wearing a black dress on the seaside\n\nserene, sunset, spring, high quality, detailed, diffuse light" + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 10, + 680 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 36 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 22, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 855, + 51 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 43 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 38 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 40 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 37 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "FULL FACE - SD1.5 only (portraits stronger)" + ] + }, + { + "id": 21, + "type": "IPAdapter", + "pos": [ + 1280, + 170 + ], + "size": { + "0": 315, + "1": 166 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 40 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 37, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 53 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 42 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapter" + }, + "widgets_values": [ + 0.4, + 0, + 1 + ] + }, + { + "id": 20, + "type": "IPAdapterUnifiedLoaderFaceID", + "pos": [ + 457, + 57 + ], + "size": { + "0": 315, + "1": 126 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 36 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 35 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 34, + 38 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoaderFaceID" + }, + "widgets_values": [ + "FACEID PLUS V2", + 0.6, + "CPU" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + -220, + -370 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 44 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "rosario-1.jpg", + "image" + ] + }, + { + "id": 25, + "type": "LoadImage", + "pos": [ + -580, + -380 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 45 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "rosario_4.jpg", + "image" + ] + }, + { + "id": 24, + "type": "LoadImage", + "pos": [ + -230, + 0 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 46 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "rosario_5.jpg", + "image" + ] + }, + { + "id": 26, + "type": "LoadImage", + "pos": [ + -580, + 0 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 47 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "rosario-2.jpg", + "image" + ] + }, + { + "id": 27, + "type": "ImageBatch", + "pos": [ + 180, + -210 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 44 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 45 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 48 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageBatch" + } + }, + { + "id": 28, + "type": "ImageBatch", + "pos": [ + 180, + -110 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 46 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 47 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageBatch" + } + }, + { + "id": 18, + "type": "IPAdapterFaceID", + "pos": [ + 850, + 190 + ], + "size": { + "0": 315, + "1": 298 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 35 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 34, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 50 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + }, + { + "name": "insightface", + "type": "INSIGHTFACE", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 43 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterFaceID" + }, + "widgets_values": [ + 0.8, + 2, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 29, + "type": "ImageBatch", + "pos": [ + 420, + -170 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 48 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 49 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 50, + 51 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageBatch" + } + }, + { + "id": 31, + "type": "ImageListToBatch+", + "pos": [ + 1218, + -208 + ], + "size": { + "0": 210, + "1": 26 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 52 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 53 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageListToBatch+" + } + }, + { + "id": 30, + "type": "FaceBoundingBox", + "pos": [ + 848, + -211 + ], + "size": { + "0": 315, + "1": 162 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "analysis_models", + "type": "ANALYSIS_MODELS", + "link": 54, + "slot_index": 0 + }, + { + "name": "image", + "type": "IMAGE", + "link": 51 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 52 + ], + "shape": 6, + "slot_index": 0 + }, + { + "name": "x", + "type": "INT", + "links": null, + "shape": 6 + }, + { + "name": "y", + "type": "INT", + "links": null, + "shape": 6 + }, + { + "name": "width", + "type": "INT", + "links": null, + "shape": 6 + }, + { + "name": "height", + "type": "INT", + "links": null, + "shape": 6 + } + ], + "properties": { + "Node name for S&R": "FaceBoundingBox" + }, + "widgets_values": [ + 30, + -1 + ] + }, + { + "id": 32, + "type": "FaceAnalysisModels", + "pos": [ + 602, + -369 + ], + "size": [ + 210, + 82 + ], + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "ANALYSIS_MODELS", + "type": "ANALYSIS_MODELS", + "links": [ + 54 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FaceAnalysisModels" + }, + "widgets_values": [ + "dlib", + "CPU" + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 34, + 20, + 1, + 18, + 1, + "IPADAPTER" + ], + [ + 35, + 20, + 0, + 18, + 0, + "MODEL" + ], + [ + 36, + 4, + 0, + 20, + 0, + "MODEL" + ], + [ + 37, + 22, + 1, + 21, + 1, + "IPADAPTER" + ], + [ + 38, + 20, + 1, + 22, + 1, + "IPADAPTER" + ], + [ + 40, + 22, + 0, + 21, + 0, + "MODEL" + ], + [ + 42, + 21, + 0, + 3, + 0, + "MODEL" + ], + [ + 43, + 18, + 0, + 22, + 0, + "MODEL" + ], + [ + 44, + 12, + 0, + 27, + 0, + "IMAGE" + ], + [ + 45, + 25, + 0, + 27, + 1, + "IMAGE" + ], + [ + 46, + 24, + 0, + 28, + 0, + "IMAGE" + ], + [ + 47, + 26, + 0, + 28, + 1, + "IMAGE" + ], + [ + 48, + 27, + 0, + 29, + 0, + "IMAGE" + ], + [ + 49, + 28, + 0, + 29, + 1, + "IMAGE" + ], + [ + 50, + 29, + 0, + 18, + 2, + "IMAGE" + ], + [ + 51, + 29, + 0, + 30, + 1, + "IMAGE" + ], + [ + 52, + 30, + 0, + 31, + 0, + "IMAGE" + ], + [ + 53, + 31, + 0, + 21, + 2, + "IMAGE" + ], + [ + 54, + 32, + 0, + 30, + 0, + "ANALYSIS_MODELS" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_ideal_faceid_config.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_ideal_faceid_config.json new file mode 100644 index 0000000..70df91a --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_ideal_faceid_config.json @@ -0,0 +1,756 @@ +{ + "last_node_id": 23, + "last_link_id": 43, + "nodes": [ + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1640, + 710 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 870, + 1100 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1280, + 710 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 42 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "ddpm", + "karras", + 1 + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1830, + 700 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 450, + 240 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 29 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "rosario_4.jpg", + "image" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 760, + 850 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed, naked" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 760, + 620 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup of a beautiful woman wearing a black dress on the seaside\n\nserene, sunset, spring, high quality, detailed, diffuse light" + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 10, + 680 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 36 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 20, + "type": "IPAdapterUnifiedLoaderFaceID", + "pos": [ + 460, + 60 + ], + "size": { + "0": 315, + "1": 126 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 36 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 35 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 34, + 38 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoaderFaceID" + }, + "widgets_values": [ + "FACEID PLUS V2", + 0.6, + "CPU" + ] + }, + { + "id": 22, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 855, + 51 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 43 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 38 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 40 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 37 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "FULL FACE - SD1.5 only (portraits stronger)" + ] + }, + { + "id": 21, + "type": "IPAdapter", + "pos": [ + 1280, + 170 + ], + "size": { + "0": 315, + "1": 166 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 40 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 37, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 41 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 42 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapter" + }, + "widgets_values": [ + 0.4, + 0, + 1 + ] + }, + { + "id": 18, + "type": "IPAdapterFaceID", + "pos": [ + 850, + 190 + ], + "size": { + "0": 315, + "1": 298 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 35 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 34, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 29 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + }, + { + "name": "insightface", + "type": "INSIGHTFACE", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 43 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterFaceID" + }, + "widgets_values": [ + 0.8, + 2, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 23, + "type": "LoadImage", + "pos": [ + 1280, + -230 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 41 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "rosario.png", + "image" + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 29, + 12, + 0, + 18, + 2, + "IMAGE" + ], + [ + 34, + 20, + 1, + 18, + 1, + "IPADAPTER" + ], + [ + 35, + 20, + 0, + 18, + 0, + "MODEL" + ], + [ + 36, + 4, + 0, + 20, + 0, + "MODEL" + ], + [ + 37, + 22, + 1, + 21, + 1, + "IPADAPTER" + ], + [ + 38, + 20, + 1, + 22, + 1, + "IPADAPTER" + ], + [ + 40, + 22, + 0, + 21, + 0, + "MODEL" + ], + [ + 41, + 23, + 0, + 21, + 2, + "IMAGE" + ], + [ + 42, + 21, + 0, + 3, + 0, + "MODEL" + ], + [ + 43, + 18, + 0, + 22, + 0, + "MODEL" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_negative_image.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_negative_image.json new file mode 100644 index 0000000..2c783bd --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_negative_image.json @@ -0,0 +1,956 @@ +{ + "last_node_id": 24, + "last_link_id": 49, + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 50, + 730 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 20, + 44 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8, + 42 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1770, + 710 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 690, + 840 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6, + 39 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed" + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2, + 40 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 16, + "type": "CLIPVisionLoader", + "pos": [ + 308, + 161 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 24, + 48 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ] + }, + { + "id": 15, + "type": "IPAdapterModelLoader", + "pos": [ + 308, + 52 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 33, + 45 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.safetensors" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1210, + 700 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 23 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m_sde_gpu", + "exponential", + 1 + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1575, + 705 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 690, + 610 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4, + 38 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "a castle on a cliff\n\nhigh quality, detailed, diffuse light" + ] + }, + { + "id": 24, + "type": "IPAdapterAdvanced", + "pos": [ + 1800, + 330 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 44 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 45, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 46 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": 49 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 48, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 37 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.7000000000000001, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 20, + "type": "PrepImageForClipVision", + "pos": [ + 775, + 347 + ], + "size": [ + 210, + 106 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 35 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 36, + 46 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PrepImageForClipVision" + }, + "widgets_values": [ + "LANCZOS", + "top", + 0 + ] + }, + { + "id": 21, + "type": "KSampler", + "pos": [ + 2190, + 330 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 37 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 38 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 39 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 40 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 41 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m_sde_gpu", + "exponential", + 1 + ] + }, + { + "id": 14, + "type": "IPAdapterAdvanced", + "pos": [ + 1199, + 346 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 20 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 33, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 36 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 24, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.7000000000000001, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 23, + "type": "SaveImage", + "pos": [ + 2333, + 711 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 43 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 19, + "type": "LoadImage", + "pos": [ + 1206, + -41 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "trees.jpg", + "image" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 313, + 291 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 35 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "castle.jpg", + "image" + ] + }, + { + "id": 22, + "type": "VAEDecode", + "pos": [ + 2581, + 331 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 41 + }, + { + "name": "vae", + "type": "VAE", + "link": 42 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 43 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 20, + 4, + 0, + 14, + 0, + "MODEL" + ], + [ + 23, + 14, + 0, + 3, + 0, + "MODEL" + ], + [ + 24, + 16, + 0, + 14, + 5, + "CLIP_VISION" + ], + [ + 33, + 15, + 0, + 14, + 1, + "IPADAPTER" + ], + [ + 35, + 12, + 0, + 20, + 0, + "IMAGE" + ], + [ + 36, + 20, + 0, + 14, + 2, + "IMAGE" + ], + [ + 37, + 24, + 0, + 21, + 0, + "MODEL" + ], + [ + 38, + 6, + 0, + 21, + 1, + "CONDITIONING" + ], + [ + 39, + 7, + 0, + 21, + 2, + "CONDITIONING" + ], + [ + 40, + 5, + 0, + 21, + 3, + "LATENT" + ], + [ + 41, + 21, + 0, + 22, + 0, + "LATENT" + ], + [ + 42, + 4, + 2, + 22, + 1, + "VAE" + ], + [ + 43, + 22, + 0, + 23, + 0, + "IMAGE" + ], + [ + 44, + 4, + 0, + 24, + 0, + "MODEL" + ], + [ + 45, + 15, + 0, + 24, + 1, + "IPADAPTER" + ], + [ + 46, + 20, + 0, + 24, + 2, + "IMAGE" + ], + [ + 48, + 16, + 0, + 24, + 5, + "CLIP_VISION" + ], + [ + 49, + 19, + 0, + 24, + 3, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_noise_injection.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_noise_injection.json new file mode 100644 index 0000000..0146c03 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_noise_injection.json @@ -0,0 +1,676 @@ +{ + "last_node_id": 18, + "last_link_id": 30, + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 50, + 730 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 20 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1770, + 710 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 690, + 840 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed" + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 16, + "type": "CLIPVisionLoader", + "pos": [ + 308, + 161 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 24 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ] + }, + { + "id": 15, + "type": "IPAdapterModelLoader", + "pos": [ + 308, + 52 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 21 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.safetensors" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 690, + 610 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "in a peaceful spring morning a woman wearing a white shirt is sitting in a park on a bench\n\nhigh quality, detailed, diffuse light" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 311, + 270 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "girl_sitting.png", + "image" + ] + }, + { + "id": 17, + "type": "PrepImageForClipVision", + "pos": [ + 728, + 290 + ], + "size": [ + 210, + 106 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 25 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 26, + 29 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PrepImageForClipVision" + }, + "widgets_values": [ + "LANCZOS", + "top", + 0.15 + ] + }, + { + "id": 14, + "type": "IPAdapterAdvanced", + "pos": [ + 1351, + 214 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 20 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 21, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 26 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": 30 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 24, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.7000000000000001, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1210, + 700 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 23 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m_sde_gpu", + "exponential", + 1 + ] + }, + { + "id": 18, + "type": "IPAdapterNoise", + "pos": [ + 1019, + 405 + ], + "size": [ + 210, + 106 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "image_optional", + "type": "IMAGE", + "link": 29 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 30 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterNoise" + }, + "widgets_values": [ + "fade", + 0.3, + 5 + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1575, + 705 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 20, + 4, + 0, + 14, + 0, + "MODEL" + ], + [ + 21, + 15, + 0, + 14, + 1, + "IPADAPTER" + ], + [ + 23, + 14, + 0, + 3, + 0, + "MODEL" + ], + [ + 24, + 16, + 0, + 14, + 5, + "CLIP_VISION" + ], + [ + 25, + 12, + 0, + 17, + 0, + "IMAGE" + ], + [ + 26, + 17, + 0, + 14, + 2, + "IMAGE" + ], + [ + 29, + 17, + 0, + 18, + 0, + "IMAGE" + ], + [ + 30, + 18, + 0, + 14, + 3, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_simple.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_simple.json new file mode 100644 index 0000000..e43b058 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_simple.json @@ -0,0 +1,546 @@ +{ + "last_node_id": 13, + "last_link_id": 17, + "nodes": [ + { + "id": 11, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 440, + 440 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 10 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 11 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 12 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "PLUS (high strength)" + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 50, + 730 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 10 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 440, + 60 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "warrior_woman.png", + "image" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1210, + 700 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 13 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 690, + 610 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup of a fierce warrior woman wearing a full armor at the end of a battle\n\nhigh quality, detailed" + ] + }, + { + "id": 10, + "type": "IPAdapter", + "pos": [ + 820, + 350 + ], + "size": { + "0": 315, + "1": 166 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 11 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 12 + }, + { + "name": "image", + "type": "IMAGE", + "link": 17, + "slot_index": 2 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapter" + }, + "widgets_values": [ + 0.8, + 0, + 1 + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1770, + 710 + ], + "size": [ + 529.7760009765616, + 582.3048192804504 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 690, + 840 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1570, + 700 + ], + "size": [ + 140, + 46 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 10, + 4, + 0, + 11, + 0, + "MODEL" + ], + [ + 11, + 11, + 0, + 10, + 0, + "MODEL" + ], + [ + 12, + 11, + 1, + 10, + 1, + "IPADAPTER" + ], + [ + 13, + 10, + 0, + 3, + 0, + "MODEL" + ], + [ + 17, + 12, + 0, + 10, + 2, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_tiled.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_tiled.json new file mode 100644 index 0000000..48a02c8 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_tiled.json @@ -0,0 +1,582 @@ +{ + "last_node_id": 18, + "last_link_id": 32, + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 50, + 730 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 29 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 690, + 840 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1570, + 700 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 250, + 290 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 27 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "girl_sitting.png", + "image" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 690, + 610 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "in a peaceful spring morning a woman wearing a white shirt is sitting in a park on a bench\n\nhigh quality, detailed, diffuse light" + ] + }, + { + "id": 16, + "type": "CLIPVisionLoader", + "pos": [ + 250, + 180 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 32 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 768, + 1 + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1210, + 700 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 30 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 2, + "fixed", + 30, + 6.5, + "ddpm", + "karras", + 1 + ] + }, + { + "id": 15, + "type": "IPAdapterModelLoader", + "pos": [ + 250, + 70 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 31 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.safetensors" + ] + }, + { + "id": 18, + "type": "IPAdapterTiled", + "pos": [ + 700, + 230 + ], + "size": { + "0": 315, + "1": 278 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 29 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 31 + }, + { + "name": "image", + "type": "IMAGE", + "link": 27 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 32 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 30 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "tiles", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "masks", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterTiled" + }, + "widgets_values": [ + 0.7000000000000001, + "ease in", + "concat", + 0, + 1, + 0 + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1768, + 700 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 27, + 12, + 0, + 18, + 2, + "IMAGE" + ], + [ + 29, + 4, + 0, + 18, + 0, + "MODEL" + ], + [ + 30, + 18, + 0, + 3, + 0, + "MODEL" + ], + [ + 31, + 15, + 0, + 18, + 1, + "IPADAPTER" + ], + [ + 32, + 16, + 0, + 18, + 5, + "CLIP_VISION" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_weight_types.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_weight_types.json new file mode 100644 index 0000000..7efe9aa --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_weight_types.json @@ -0,0 +1,1733 @@ +{ + "last_node_id": 37, + "last_link_id": 82, + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 50, + 730 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 20, + 35, + 46, + 57, + 68 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8, + 33, + 44, + 55, + 66 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 690, + 840 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6, + 30, + 41, + 52, + 63 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed" + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2, + 31, + 42, + 53, + 64 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 16, + "type": "CLIPVisionLoader", + "pos": [ + 308, + 161 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 24, + 38, + 49, + 60, + 71 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IPAdapter_image_encoder_sd15.safetensors" + ] + }, + { + "id": 15, + "type": "IPAdapterModelLoader", + "pos": [ + 308, + 52 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 21, + 36, + 47, + 58, + 69 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.safetensors" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 311, + 270 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 27, + 37, + 48, + 59, + 70 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "warrior_woman.png", + "image" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1660, + 290 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 23 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 20, + "type": "SaveImage", + "pos": [ + 2030, + 940 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 34 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 18, + "type": "KSampler", + "pos": [ + 1660, + 940 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 28 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 29 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 30 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 31 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 32 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 19, + "type": "VAEDecode", + "pos": [ + 1830, + 1280 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 32 + }, + { + "name": "vae", + "type": "VAE", + "link": 33 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 34 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1820, + 610 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 2040, + 300 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 14, + "type": "IPAdapterAdvanced", + "pos": [ + 1290, + 250 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 20 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 21, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 27 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 24, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "linear", + "concat", + 0, + 1 + ] + }, + { + "id": 21, + "type": "IPAdapterAdvanced", + "pos": [ + 1280, + 900 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 35 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 36, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 37 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 38, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 28 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "ease in", + "concat", + 0, + 1 + ] + }, + { + "id": 22, + "type": "KSampler", + "pos": [ + 1660, + 1580 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 39 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 40 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 41 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 42 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 43 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 23, + "type": "VAEDecode", + "pos": [ + 1830, + 1920 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 43 + }, + { + "name": "vae", + "type": "VAE", + "link": 44 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 45 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 25, + "type": "IPAdapterAdvanced", + "pos": [ + 1280, + 1540 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 46 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 47, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 48 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 49, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 39 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "ease out", + "concat", + 0, + 1 + ] + }, + { + "id": 24, + "type": "SaveImage", + "pos": [ + 2030, + 1580 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 45 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 27, + "type": "VAEDecode", + "pos": [ + 3750, + 670 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 54 + }, + { + "name": "vae", + "type": "VAE", + "link": 55 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 56 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 28, + "type": "SaveImage", + "pos": [ + 2620, + 300 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 56 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 30, + "type": "KSampler", + "pos": [ + 3590, + 1000 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 61 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 62 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 63 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 64 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 65 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 31, + "type": "VAEDecode", + "pos": [ + 3760, + 1340 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 65 + }, + { + "name": "vae", + "type": "VAE", + "link": 66 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 67 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 32, + "type": "SaveImage", + "pos": [ + 2610, + 940 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 67 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 26, + "type": "KSampler", + "pos": [ + 3590, + 350 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 50 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 51 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 52 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 53 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 54 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 29, + "type": "IPAdapterAdvanced", + "pos": [ + 3220, + 310 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 57 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 58, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 59 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 60, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 50 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "ease in-out", + "concat", + 0, + 1 + ] + }, + { + "id": 33, + "type": "IPAdapterAdvanced", + "pos": [ + 3210, + 960 + ], + "size": { + "0": 315, + "1": 254 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 68 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 69, + "slot_index": 1 + }, + { + "name": "image", + "type": "IMAGE", + "link": 70 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 71, + "slot_index": 5 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 61 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterAdvanced" + }, + "widgets_values": [ + 0.8, + "reverse in-out", + "concat", + 0, + 1 + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 690, + 610 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4, + 29, + 40, + 51, + 62 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup of a fierce warrior woman wearing a full armor at the end of a battle. cherry blossoms\n\nhigh quality, detailed" + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 20, + 4, + 0, + 14, + 0, + "MODEL" + ], + [ + 21, + 15, + 0, + 14, + 1, + "IPADAPTER" + ], + [ + 23, + 14, + 0, + 3, + 0, + "MODEL" + ], + [ + 24, + 16, + 0, + 14, + 5, + "CLIP_VISION" + ], + [ + 27, + 12, + 0, + 14, + 2, + "IMAGE" + ], + [ + 28, + 21, + 0, + 18, + 0, + "MODEL" + ], + [ + 29, + 6, + 0, + 18, + 1, + "CONDITIONING" + ], + [ + 30, + 7, + 0, + 18, + 2, + "CONDITIONING" + ], + [ + 31, + 5, + 0, + 18, + 3, + "LATENT" + ], + [ + 32, + 18, + 0, + 19, + 0, + "LATENT" + ], + [ + 33, + 4, + 2, + 19, + 1, + "VAE" + ], + [ + 34, + 19, + 0, + 20, + 0, + "IMAGE" + ], + [ + 35, + 4, + 0, + 21, + 0, + "MODEL" + ], + [ + 36, + 15, + 0, + 21, + 1, + "IPADAPTER" + ], + [ + 37, + 12, + 0, + 21, + 2, + "IMAGE" + ], + [ + 38, + 16, + 0, + 21, + 5, + "CLIP_VISION" + ], + [ + 39, + 25, + 0, + 22, + 0, + "MODEL" + ], + [ + 40, + 6, + 0, + 22, + 1, + "CONDITIONING" + ], + [ + 41, + 7, + 0, + 22, + 2, + "CONDITIONING" + ], + [ + 42, + 5, + 0, + 22, + 3, + "LATENT" + ], + [ + 43, + 22, + 0, + 23, + 0, + "LATENT" + ], + [ + 44, + 4, + 2, + 23, + 1, + "VAE" + ], + [ + 45, + 23, + 0, + 24, + 0, + "IMAGE" + ], + [ + 46, + 4, + 0, + 25, + 0, + "MODEL" + ], + [ + 47, + 15, + 0, + 25, + 1, + "IPADAPTER" + ], + [ + 48, + 12, + 0, + 25, + 2, + "IMAGE" + ], + [ + 49, + 16, + 0, + 25, + 5, + "CLIP_VISION" + ], + [ + 50, + 29, + 0, + 26, + 0, + "MODEL" + ], + [ + 51, + 6, + 0, + 26, + 1, + "CONDITIONING" + ], + [ + 52, + 7, + 0, + 26, + 2, + "CONDITIONING" + ], + [ + 53, + 5, + 0, + 26, + 3, + "LATENT" + ], + [ + 54, + 26, + 0, + 27, + 0, + "LATENT" + ], + [ + 55, + 4, + 2, + 27, + 1, + "VAE" + ], + [ + 56, + 27, + 0, + 28, + 0, + "IMAGE" + ], + [ + 57, + 4, + 0, + 29, + 0, + "MODEL" + ], + [ + 58, + 15, + 0, + 29, + 1, + "IPADAPTER" + ], + [ + 59, + 12, + 0, + 29, + 2, + "IMAGE" + ], + [ + 60, + 16, + 0, + 29, + 5, + "CLIP_VISION" + ], + [ + 61, + 33, + 0, + 30, + 0, + "MODEL" + ], + [ + 62, + 6, + 0, + 30, + 1, + "CONDITIONING" + ], + [ + 63, + 7, + 0, + 30, + 2, + "CONDITIONING" + ], + [ + 64, + 5, + 0, + 30, + 3, + "LATENT" + ], + [ + 65, + 30, + 0, + 31, + 0, + "LATENT" + ], + [ + 66, + 4, + 2, + 31, + 1, + "VAE" + ], + [ + 67, + 31, + 0, + 32, + 0, + "IMAGE" + ], + [ + 68, + 4, + 0, + 33, + 0, + "MODEL" + ], + [ + 69, + 15, + 0, + 33, + 1, + "IPADAPTER" + ], + [ + 70, + 12, + 0, + 33, + 2, + "IMAGE" + ], + [ + 71, + 16, + 0, + 33, + 5, + "CLIP_VISION" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_weighted_embeds.json b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_weighted_embeds.json new file mode 100644 index 0000000..e248a44 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/examples/ipadapter_weighted_embeds.json @@ -0,0 +1,836 @@ +{ + "last_node_id": 18, + "last_link_id": 29, + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 50, + 730 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 10 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd15/realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 690, + 610 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 4 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup of a fierce warrior woman wearing a full armor at the end of a battle\n\nhigh quality, detailed" + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 1770, + 710 + ], + "size": { + "0": 529.7760009765625, + "1": 582.3048095703125 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1570, + 700 + ], + "size": { + "0": 140, + "1": 46 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 801, + 1097 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 453, + -296 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 21 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "warrior_woman.png", + "image" + ] + }, + { + "id": 15, + "type": "LoadImage", + "pos": [ + 458, + 70 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "anime_illustration.png", + "image" + ] + }, + { + "id": 11, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 440, + 440 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 10 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 19 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 20, + 22, + 27 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "PLUS (high strength)" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1210, + 700 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 28 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 30, + 6.5, + "dpmpp_2m", + "karras", + 1 + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 690, + 840 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, messy, lowres, jpeg, artifacts, ill, distorted, malformed, hat, hood, scars, blood" + ] + }, + { + "id": 17, + "type": "IPAdapterEncoder", + "pos": [ + 859, + 69 + ], + "size": [ + 210, + 118 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 22 + }, + { + "name": "image", + "type": "IMAGE", + "link": 23 + }, + { + "name": "mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + } + ], + "outputs": [ + { + "name": "pos_embed", + "type": "EMBEDS", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "neg_embed", + "type": "EMBEDS", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "IPAdapterEncoder" + }, + "widgets_values": [ + 1.5 + ] + }, + { + "id": 18, + "type": "IPAdapterCombineEmbeds", + "pos": [ + 1136, + -95 + ], + "size": [ + 210, + 138 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "embed1", + "type": "EMBEDS", + "link": 24 + }, + { + "name": "embed2", + "type": "EMBEDS", + "link": 25 + }, + { + "name": "embed3", + "type": "EMBEDS", + "link": null + }, + { + "name": "embed4", + "type": "EMBEDS", + "link": null + }, + { + "name": "embed5", + "type": "EMBEDS", + "link": null + } + ], + "outputs": [ + { + "name": "EMBEDS", + "type": "EMBEDS", + "links": [ + 26 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterCombineEmbeds" + }, + "widgets_values": [ + "average" + ] + }, + { + "id": 14, + "type": "IPAdapterEmbeds", + "pos": [ + 1143, + 160 + ], + "size": { + "0": 315, + "1": 230 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 19 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 27 + }, + { + "name": "pos_embed", + "type": "EMBEDS", + "link": 26 + }, + { + "name": "neg_embed", + "type": "EMBEDS", + "link": 29 + }, + { + "name": "attn_mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 28 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterEmbeds" + }, + "widgets_values": [ + 0.8, + "linear", + 0, + 1 + ] + }, + { + "id": 16, + "type": "IPAdapterEncoder", + "pos": [ + 863, + -285 + ], + "size": [ + 210, + 118 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 20 + }, + { + "name": "image", + "type": "IMAGE", + "link": 21 + }, + { + "name": "mask", + "type": "MASK", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + } + ], + "outputs": [ + { + "name": "pos_embed", + "type": "EMBEDS", + "links": [ + 24 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "neg_embed", + "type": "EMBEDS", + "links": [ + 29 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterEncoder" + }, + "widgets_values": [ + 0.6 + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 4, + 6, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 7, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 10, + 4, + 0, + 11, + 0, + "MODEL" + ], + [ + 19, + 11, + 0, + 14, + 0, + "MODEL" + ], + [ + 20, + 11, + 1, + 16, + 0, + "IPADAPTER" + ], + [ + 21, + 12, + 0, + 16, + 1, + "IMAGE" + ], + [ + 22, + 11, + 1, + 17, + 0, + "IPADAPTER" + ], + [ + 23, + 15, + 0, + 17, + 1, + "IMAGE" + ], + [ + 24, + 16, + 0, + 18, + 0, + "EMBEDS" + ], + [ + 25, + 17, + 0, + 18, + 1, + "EMBEDS" + ], + [ + 26, + 18, + 0, + 14, + 2, + "EMBEDS" + ], + [ + 27, + 11, + 1, + 14, + 1, + "IPADAPTER" + ], + [ + 28, + 14, + 0, + 3, + 0, + "MODEL" + ], + [ + 29, + 16, + 1, + 14, + 3, + "EMBEDS" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/imports/ComfyUI_IPAdapter_plus/image_proj_models.py b/imports/ComfyUI_IPAdapter_plus/image_proj_models.py new file mode 100644 index 0000000..06df68b --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/image_proj_models.py @@ -0,0 +1,275 @@ +import math +import torch +import torch.nn as nn +from einops import rearrange +from einops.layers.torch import Rearrange + + +# FFN +def FeedForwardImport(dim, mult=4): + inner_dim = int(dim * mult) + return nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, inner_dim, bias=False), + nn.GELU(), + nn.Linear(inner_dim, dim, bias=False), + ) + + +def reshape_tensor(x, heads): + bs, length, width = x.shape + # (bs, length, width) --> (bs, length, n_heads, dim_per_head) + x = x.view(bs, length, heads, -1) + # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head) + x = x.transpose(1, 2) + # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head) + x = x.reshape(bs, heads, length, -1) + return x + + +class PerceiverAttentionImport(nn.Module): + def __init__(self, *, dim, dim_head=64, heads=8): + super().__init__() + self.scale = dim_head**-0.5 + self.dim_head = dim_head + self.heads = heads + inner_dim = dim_head * heads + + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False) + self.to_out = nn.Linear(inner_dim, dim, bias=False) + + def forward(self, x, latents): + """ + Args: + x (torch.Tensor): image features + shape (b, n1, D) + latent (torch.Tensor): latent features + shape (b, n2, D) + """ + x = self.norm1(x) + latents = self.norm2(latents) + + b, l, _ = latents.shape + + q = self.to_q(latents) + kv_input = torch.cat((x, latents), dim=-2) + k, v = self.to_kv(kv_input).chunk(2, dim=-1) + + q = reshape_tensor(q, self.heads) + k = reshape_tensor(k, self.heads) + v = reshape_tensor(v, self.heads) + + # attention + scale = 1 / math.sqrt(math.sqrt(self.dim_head)) + weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards + weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) + out = weight @ v + + out = out.permute(0, 2, 1, 3).reshape(b, l, -1) + + return self.to_out(out) + + +class ResamplerImport(nn.Module): + def __init__( + self, + dim=1024, + depth=8, + dim_head=64, + heads=16, + num_queries=8, + embedding_dim=768, + output_dim=1024, + ff_mult=4, + max_seq_len: int = 257, # CLIP tokens + CLS token + apply_pos_emb: bool = False, + num_latents_mean_pooled: int = 0, # number of latents derived from mean pooled representation of the sequence + ): + super().__init__() + self.pos_emb = nn.Embedding(max_seq_len, embedding_dim) if apply_pos_emb else None + + self.latents = nn.Parameter(torch.randn(1, num_queries, dim) / dim**0.5) + + self.proj_in = nn.Linear(embedding_dim, dim) + + self.proj_out = nn.Linear(dim, output_dim) + self.norm_out = nn.LayerNorm(output_dim) + + self.to_latents_from_mean_pooled_seq = ( + nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, dim * num_latents_mean_pooled), + Rearrange("b (n d) -> b n d", n=num_latents_mean_pooled), + ) + if num_latents_mean_pooled > 0 + else None + ) + + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append( + nn.ModuleList( + [ + PerceiverAttentionImport(dim=dim, dim_head=dim_head, heads=heads), + FeedForwardImport(dim=dim, mult=ff_mult), + ] + ) + ) + + def forward(self, x): + if self.pos_emb is not None: + n, device = x.shape[1], x.device + pos_emb = self.pos_emb(torch.arange(n, device=device)) + x = x + pos_emb + + latents = self.latents.repeat(x.size(0), 1, 1) + + x = self.proj_in(x) + + if self.to_latents_from_mean_pooled_seq: + meanpooled_seq = masked_mean(x, dim=1, mask=torch.ones(x.shape[:2], device=x.device, dtype=torch.bool)) + meanpooled_latents = self.to_latents_from_mean_pooled_seq(meanpooled_seq) + latents = torch.cat((meanpooled_latents, latents), dim=-2) + + for attn, ff in self.layers: + latents = attn(x, latents) + latents + latents = ff(latents) + latents + + latents = self.proj_out(latents) + return self.norm_out(latents) + + +def masked_mean(t, *, dim, mask=None): + if mask is None: + return t.mean(dim=dim) + + denom = mask.sum(dim=dim, keepdim=True) + mask = rearrange(mask, "b n -> b n 1") + masked_t = t.masked_fill(~mask, 0.0) + + return masked_t.sum(dim=dim) / denom.clamp(min=1e-5) + + +class FacePerceiverResamplerImport(nn.Module): + def __init__( + self, + *, + dim=768, + depth=4, + dim_head=64, + heads=16, + embedding_dim=1280, + output_dim=768, + ff_mult=4, + ): + super().__init__() + + self.proj_in = nn.Linear(embedding_dim, dim) + self.proj_out = nn.Linear(dim, output_dim) + self.norm_out = nn.LayerNorm(output_dim) + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append( + nn.ModuleList( + [ + PerceiverAttentionImport(dim=dim, dim_head=dim_head, heads=heads), + FeedForwardImport(dim=dim, mult=ff_mult), + ] + ) + ) + + def forward(self, latents, x): + x = self.proj_in(x) + for attn, ff in self.layers: + latents = attn(x, latents) + latents + latents = ff(latents) + latents + latents = self.proj_out(latents) + return self.norm_out(latents) + + +class MLPProjModelImport(nn.Module): + def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024): + super().__init__() + + self.proj = nn.Sequential( + nn.Linear(clip_embeddings_dim, clip_embeddings_dim), + nn.GELU(), + nn.Linear(clip_embeddings_dim, cross_attention_dim), + nn.LayerNorm(cross_attention_dim) + ) + + def forward(self, image_embeds): + clip_extra_context_tokens = self.proj(image_embeds) + return clip_extra_context_tokens + +class MLPProjModelFaceIdImport(nn.Module): + def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, num_tokens=4): + super().__init__() + + self.cross_attention_dim = cross_attention_dim + self.num_tokens = num_tokens + + self.proj = nn.Sequential( + nn.Linear(id_embeddings_dim, id_embeddings_dim*2), + nn.GELU(), + nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens), + ) + self.norm = nn.LayerNorm(cross_attention_dim) + + def forward(self, id_embeds): + x = self.proj(id_embeds) + x = x.reshape(-1, self.num_tokens, self.cross_attention_dim) + x = self.norm(x) + return x + +class ProjModelFaceIdPlusImport(nn.Module): + def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, clip_embeddings_dim=1280, num_tokens=4): + super().__init__() + + self.cross_attention_dim = cross_attention_dim + self.num_tokens = num_tokens + + self.proj = nn.Sequential( + nn.Linear(id_embeddings_dim, id_embeddings_dim*2), + nn.GELU(), + nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens), + ) + self.norm = nn.LayerNorm(cross_attention_dim) + + self.perceiver_resampler = FacePerceiverResamplerImport( + dim=cross_attention_dim, + depth=4, + dim_head=64, + heads=cross_attention_dim // 64, + embedding_dim=clip_embeddings_dim, + output_dim=cross_attention_dim, + ff_mult=4, + ) + + def forward(self, id_embeds, clip_embeds, scale=1.0, shortcut=False): + x = self.proj(id_embeds) + x = x.reshape(-1, self.num_tokens, self.cross_attention_dim) + x = self.norm(x) + out = self.perceiver_resampler(x, clip_embeds) + if shortcut: + out = x + scale * out + return out + +class ImageProjModelImport(nn.Module): + def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4): + super().__init__() + + self.cross_attention_dim = cross_attention_dim + self.clip_extra_context_tokens = clip_extra_context_tokens + self.proj = nn.Linear(clip_embeddings_dim, self.clip_extra_context_tokens * cross_attention_dim) + self.norm = nn.LayerNorm(cross_attention_dim) + + def forward(self, image_embeds): + embeds = image_embeds + x = self.proj(embeds).reshape(-1, self.clip_extra_context_tokens, self.cross_attention_dim) + x = self.norm(x) + return x diff --git a/imports/ComfyUI_IPAdapter_plus/models/legacy_directory_do_not_use.txt b/imports/ComfyUI_IPAdapter_plus/models/legacy_directory_do_not_use.txt new file mode 100644 index 0000000..e69de29 diff --git a/imports/ComfyUI_IPAdapter_plus/utils.py b/imports/ComfyUI_IPAdapter_plus/utils.py new file mode 100644 index 0000000..b1fab58 --- /dev/null +++ b/imports/ComfyUI_IPAdapter_plus/utils.py @@ -0,0 +1,228 @@ +import re +import torch +import os +import folder_paths +from comfy.clip_vision import clip_preprocess, Output +import comfy.utils +import comfy.model_management as model_management +try: + import torchvision.transforms.v2 as T +except ImportError: + import torchvision.transforms as T + +def get_clipvision_file(preset): + preset = preset.lower() + clipvision_list = folder_paths.get_filename_list("clip_vision") + + if preset.startswith("vit-g"): + pattern = '(ViT.bigG.14.*39B.b160k|ipadapter.*sdxl|sdxl.*model\.(bin|safetensors))' + else: + pattern = '(ViT.H.14.*s32B.b79K|ipadapter.*sd15|sd1.?5.*model\.(bin|safetensors))' + clipvision_file = [e for e in clipvision_list if re.search(pattern, e, re.IGNORECASE)] + + clipvision_file = folder_paths.get_full_path("clip_vision", clipvision_file[0]) if clipvision_file else None + + return clipvision_file + +def get_ipadapter_file(preset, is_sdxl): + preset = preset.lower() + ipadapter_list = folder_paths.get_filename_list("ipadapter") + is_insightface = False + lora_pattern = None + + if preset.startswith("light"): + if is_sdxl: + raise Exception("light model is not supported for SDXL") + pattern = 'sd15.light.v11\.(safetensors|bin)$' + # if light model v11 is not found, try with the old version + if not [e for e in ipadapter_list if re.search(pattern, e, re.IGNORECASE)]: + pattern = 'sd15.light\.(safetensors|bin)$' + elif preset.startswith("standard"): + if is_sdxl: + pattern = 'ip.adapter.sdxl.vit.h\.(safetensors|bin)$' + else: + pattern = 'ip.adapter.sd15\.(safetensors|bin)$' + elif preset.startswith("vit-g"): + if is_sdxl: + pattern = 'ip.adapter.sdxl\.(safetensors|bin)$' + else: + pattern = 'sd15.vit.g\.(safetensors|bin)$' + elif preset.startswith("plus ("): + if is_sdxl: + pattern = 'plus.sdxl.vit.h\.(safetensors|bin)$' + else: + pattern = 'ip.adapter.plus.sd15\.(safetensors|bin)$' + elif preset.startswith("plus face"): + if is_sdxl: + pattern = 'plus.face.sdxl.vit.h\.(safetensors|bin)$' + else: + pattern = 'plus.face.sd15\.(safetensors|bin)$' + elif preset.startswith("full"): + if is_sdxl: + raise Exception("full face model is not supported for SDXL") + pattern = 'full.face.sd15\.(safetensors|bin)$' + elif preset.startswith("faceid portrait"): + if is_sdxl: + raise Exception("portrait model is not supported for SDXL") + pattern = 'portrait.sd15\.(safetensors|bin)$' + is_insightface = True + elif preset == "faceid": + if is_sdxl: + pattern = 'faceid.sdxl\.(safetensors|bin)$' + lora_pattern = 'faceid.sdxl.lora\.safetensors$' + else: + pattern = 'faceid.sd15\.(safetensors|bin)$' + lora_pattern = 'faceid.sd15.lora\.safetensors$' + is_insightface = True + elif preset.startswith("faceid plus -"): + if is_sdxl: + raise Exception("faceid plus model is not supported for SDXL") + pattern = 'faceid.plus.sd15\.(safetensors|bin)$' + lora_pattern = 'faceid.plus.sd15.lora\.safetensors$' + is_insightface = True + elif preset.startswith("faceid plus v2"): + if is_sdxl: + pattern = 'faceid.plusv2.sdxl\.(safetensors|bin)$' + lora_pattern = 'faceid.plusv2.sdxl.lora\.safetensors$' + else: + pattern = 'faceid.plusv2.sd15\.(safetensors|bin)$' + lora_pattern = 'faceid.plusv2.sd15.lora\.safetensors$' + is_insightface = True + else: + raise Exception(f"invalid type '{preset}'") + + ipadapter_file = [e for e in ipadapter_list if re.search(pattern, e, re.IGNORECASE)] + ipadapter_file = folder_paths.get_full_path("ipadapter", ipadapter_file[0]) if ipadapter_file else None + + return ipadapter_file, is_insightface, lora_pattern + +def get_lora_file(pattern): + lora_list = folder_paths.get_filename_list("loras") + lora_file = [e for e in lora_list if re.search(pattern, e, re.IGNORECASE)] + lora_file = folder_paths.get_full_path("loras", lora_file[0]) if lora_file else None + + return lora_file + +def ipadapter_model_loader(file): + model = comfy.utils.load_torch_file(file, safe_load=True) + + if file.lower().endswith(".safetensors"): + st_model = {"image_proj": {}, "ip_adapter": {}} + for key in model.keys(): + if key.startswith("image_proj."): + st_model["image_proj"][key.replace("image_proj.", "")] = model[key] + elif key.startswith("ip_adapter."): + st_model["ip_adapter"][key.replace("ip_adapter.", "")] = model[key] + model = st_model + del st_model + + if not "ip_adapter" in model.keys() or not model["ip_adapter"]: + raise Exception("invalid IPAdapter model {}".format(file)) + + if 'plusv2' in file.lower(): + model["faceidplusv2"] = True + + return model + +def insightface_loader(provider): + try: + from insightface.app import FaceAnalysis + except ImportError as e: + raise Exception(e) + + path = os.path.join(folder_paths.models_dir, "insightface") + model = FaceAnalysis(name="buffalo_l", root=path, providers=[provider + 'ExecutionProvider',]) + model.prepare(ctx_id=0, det_size=(640, 640)) + return model + +def encode_image_masked(clip_vision, image, mask=None): + model_management.load_model_gpu(clip_vision.patcher) + image = image.to(clip_vision.load_device) + + pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float() + + if mask is not None: + pixel_values = pixel_values * mask.to(clip_vision.load_device) + + out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) + + outputs = Output() + outputs["last_hidden_state"] = out[0].to(model_management.intermediate_device()) + outputs["image_embeds"] = out[2].to(model_management.intermediate_device()) + outputs["penultimate_hidden_states"] = out[1].to(model_management.intermediate_device()) + return outputs + +def tensor_to_size(source, dest_size): + if isinstance(dest_size, torch.Tensor): + dest_size = dest_size.shape[0] + source_size = source.shape[0] + + if source_size < dest_size: + shape = [dest_size - source_size] + [1]*(source.dim()-1) + source = torch.cat((source, source[-1:].repeat(shape)), dim=0) + elif source_size > dest_size: + source = source[:dest_size] + + return source + +def min_(tensor_list): + # return the element-wise min of the tensor list. + x = torch.stack(tensor_list) + mn = x.min(axis=0)[0] + return torch.clamp(mn, min=0) + +def max_(tensor_list): + # return the element-wise max of the tensor list. + x = torch.stack(tensor_list) + mx = x.max(axis=0)[0] + return torch.clamp(mx, max=1) + +# From https://github.com/Jamy-L/Pytorch-Contrast-Adaptive-Sharpening/ +def contrast_adaptive_sharpening(image, amount): + img = T.functional.pad(image, (1, 1, 1, 1)).cpu() + + a = img[..., :-2, :-2] + b = img[..., :-2, 1:-1] + c = img[..., :-2, 2:] + d = img[..., 1:-1, :-2] + e = img[..., 1:-1, 1:-1] + f = img[..., 1:-1, 2:] + g = img[..., 2:, :-2] + h = img[..., 2:, 1:-1] + i = img[..., 2:, 2:] + + # Computing contrast + cross = (b, d, e, f, h) + mn = min_(cross) + mx = max_(cross) + + diag = (a, c, g, i) + mn2 = min_(diag) + mx2 = max_(diag) + mx = mx + mx2 + mn = mn + mn2 + + # Computing local weight + inv_mx = torch.reciprocal(mx) + amp = inv_mx * torch.minimum(mn, (2 - mx)) + + # scaling + amp = torch.sqrt(amp) + w = - amp * (amount * (1/5 - 1/8) + 1/8) + div = torch.reciprocal(1 + 4*w) + + output = ((b + d + f + h)*w + e) * div + output = torch.nan_to_num(output) + output = output.clamp(0, 1) + + return output + +def tensor_to_image(tensor): + image = tensor.mul(255).clamp(0, 255).byte().cpu() + image = image[..., [2, 1, 0]].numpy() + return image + +def image_to_tensor(image): + tensor = torch.clamp(torch.from_numpy(image).float() / 255., 0, 1) + tensor = tensor[..., [2, 1, 0]] + return tensor \ No newline at end of file diff --git a/imports/IPAdapterPlus.py b/imports/IPAdapterPlus.py deleted file mode 100644 index a9aab68..0000000 --- a/imports/IPAdapterPlus.py +++ /dev/null @@ -1,1352 +0,0 @@ -import torch -import os -import math - -import comfy.utils -import comfy.model_management -from comfy.clip_vision import clip_preprocess, Output -from comfy.ldm.modules.attention import optimized_attention -from nodes import MAX_RESOLUTION -import folder_paths - -import torch.nn as nn -from PIL import Image -import torch.nn.functional as F -import torchvision.transforms as TT - - - -# set the models directory backward compatible -GLOBAL_MODELS_DIR = os.path.join(folder_paths.models_dir, "ipadapter") -MODELS_DIR = GLOBAL_MODELS_DIR if os.path.isdir(GLOBAL_MODELS_DIR) else os.path.join(os.path.dirname(os.path.realpath(__file__)), "models") -if "ipadapter" not in folder_paths.folder_names_and_paths: - current_paths = [MODELS_DIR] -else: - current_paths, _ = folder_paths.folder_names_and_paths["ipadapter"] -folder_paths.folder_names_and_paths["ipadapter"] = (current_paths, folder_paths.supported_pt_extensions) - -INSIGHTFACE_DIR = os.path.join(folder_paths.models_dir, "insightface") - -class FacePerceiverResamplerImport(torch.nn.Module): - def __init__( - self, - *, - dim=768, - depth=4, - dim_head=64, - heads=16, - embedding_dim=1280, - output_dim=768, - ff_mult=4, - ): - super().__init__() - - self.proj_in = torch.nn.Linear(embedding_dim, dim) - self.proj_out = torch.nn.Linear(dim, output_dim) - self.norm_out = torch.nn.LayerNorm(output_dim) - self.layers = torch.nn.ModuleList([]) - for _ in range(depth): - self.layers.append( - torch.nn.ModuleList( - [ - PerceiverAttentionImport(dim=dim, dim_head=dim_head, heads=heads), - FeedForwardImport(dim=dim, mult=ff_mult), - ] - ) - ) - - def forward(self, latents, x): - x = self.proj_in(x) - for attn, ff in self.layers: - latents = attn(x, latents) + latents - latents = ff(latents) + latents - latents = self.proj_out(latents) - return self.norm_out(latents) - -class MLPProjModelImport(torch.nn.Module): - def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024): - super().__init__() - - self.proj = torch.nn.Sequential( - torch.nn.Linear(clip_embeddings_dim, clip_embeddings_dim), - torch.nn.GELU(), - torch.nn.Linear(clip_embeddings_dim, cross_attention_dim), - torch.nn.LayerNorm(cross_attention_dim) - ) - - def forward(self, image_embeds): - clip_extra_context_tokens = self.proj(image_embeds) - return clip_extra_context_tokens - -class MLPProjModelFaceIdImport(torch.nn.Module): - def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, num_tokens=4): - super().__init__() - - self.cross_attention_dim = cross_attention_dim - self.num_tokens = num_tokens - - self.proj = torch.nn.Sequential( - torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2), - torch.nn.GELU(), - torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens), - ) - self.norm = torch.nn.LayerNorm(cross_attention_dim) - - def forward(self, id_embeds): - clip_extra_context_tokens = self.proj(id_embeds) - clip_extra_context_tokens = clip_extra_context_tokens.reshape(-1, self.num_tokens, self.cross_attention_dim) - clip_extra_context_tokens = self.norm(clip_extra_context_tokens) - return clip_extra_context_tokens - -class ProjModelFaceIdPlusImport(torch.nn.Module): - def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, clip_embeddings_dim=1280, num_tokens=4): - super().__init__() - - self.cross_attention_dim = cross_attention_dim - self.num_tokens = num_tokens - - self.proj = torch.nn.Sequential( - torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2), - torch.nn.GELU(), - torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens), - ) - self.norm = torch.nn.LayerNorm(cross_attention_dim) - - self.perceiver_resampler = FacePerceiverResamplerImport( - dim=cross_attention_dim, - depth=4, - dim_head=64, - heads=cross_attention_dim // 64, - embedding_dim=clip_embeddings_dim, - output_dim=cross_attention_dim, - ff_mult=4, - ) - - def forward(self, id_embeds, clip_embeds, scale=1.0, shortcut=False): - x = self.proj(id_embeds) - x = x.reshape(-1, self.num_tokens, self.cross_attention_dim) - x = self.norm(x) - out = self.perceiver_resampler(x, clip_embeds) - if shortcut: - out = x + scale * out - return out - -class ImageProjModelImport(nn.Module): - def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4): - super().__init__() - - self.cross_attention_dim = cross_attention_dim - self.clip_extra_context_tokens = clip_extra_context_tokens - self.proj = nn.Linear(clip_embeddings_dim, self.clip_extra_context_tokens * cross_attention_dim) - self.norm = nn.LayerNorm(cross_attention_dim) - - def forward(self, image_embeds): - embeds = image_embeds - clip_extra_context_tokens = self.proj(embeds).reshape(-1, self.clip_extra_context_tokens, self.cross_attention_dim) - clip_extra_context_tokens = self.norm(clip_extra_context_tokens) - return clip_extra_context_tokens - -class To_KVImport(nn.Module): - def __init__(self, state_dict): - super().__init__() - - self.to_kvs = nn.ModuleDict() - for key, value in state_dict.items(): - self.to_kvs[key.replace(".weight", "").replace(".", "_")] = nn.Linear(value.shape[1], value.shape[0], bias=False) - self.to_kvs[key.replace(".weight", "").replace(".", "_")].weight.data = value - -def set_model_patch_replace(model, patch_kwargs, key): - to = model.model_options["transformer_options"] - if "patches_replace" not in to: - to["patches_replace"] = {} - if "attn2" not in to["patches_replace"]: - to["patches_replace"]["attn2"] = {} - if key not in to["patches_replace"]["attn2"]: - to["patches_replace"]["attn2"][key] = CrossAttentionPatchImport(**patch_kwargs) - else: - to["patches_replace"]["attn2"][key].set_new_condition(**patch_kwargs) - -def masked_tiling(image, short_side_tiles, weight=0.6, blur=0): - _, orig_height, orig_width, _ = image.shape - tile_size = 224 - - if orig_width < orig_height: - num_tiles_x = short_side_tiles - new_width = tile_size * num_tiles_x - new_height = orig_height * new_width // orig_width - num_tiles_y = new_height // tile_size - else: - num_tiles_y = short_side_tiles - new_height = tile_size * num_tiles_y - new_width = orig_width * new_height // orig_height - num_tiles_x = new_width // tile_size - - start_x = start_y = 0 - if (new_height % tile_size) >= tile_size//4: - num_tiles_y += 1 - else: - start_y = (new_height - tile_size*num_tiles_y) // 2 - - if (new_width % tile_size) >= tile_size//4: - num_tiles_x += 1 - else: - start_x = (new_width - tile_size*num_tiles_x) // 2 - - weight = 1.0 if num_tiles_x == 1 or num_tiles_y == 1 else weight - - ref_image = F.interpolate(image.permute([0,3,1,2]), size=(new_height, new_width), mode="bicubic").permute([0,2,3,1]) - - tiles = [] - attn_mask = [] - - for i in range(num_tiles_y): - for j in range(num_tiles_x): - start_height = i * tile_size + start_y - end_height = start_height + tile_size + start_y - start_width = j * tile_size + start_x - end_width = start_width + tile_size + start_x - - if end_height > new_height: - start_height = new_height - tile_size - end_height = new_height - if end_width > new_width: - start_width = new_width - tile_size - end_width = new_width - - tile = ref_image[:1, start_height:end_height, start_width:end_width, :] - tiles.append(tile.squeeze(0)) - - # create mask - mask = torch.zeros([new_height, new_width], dtype=image.dtype, device=image.device) - mask[start_height:end_height, start_width:end_width] = weight - - attn_mask.append(mask) - - image = torch.stack(tiles, dim=0) - attn_mask = torch.stack(attn_mask, dim=0) - - # If we have a lot of tiles we add bigger tiles with a higher weight to give clipvision a better idea of the overall image - if num_tiles_x != 1 and num_tiles_y != 1: - comp_tiles, comp_attn_mask = masked_tiling(ref_image, 1, 1.0, blur) - - if image.shape[1:3] != comp_tiles.shape[1:3]: - comp_tiles = F.interpolate(comp_tiles.permute([0,3,1,2]), size=(image.shape[1], image.shape[2]), mode="bicubic").permute([0,2,3,1]) - image = torch.cat([comp_tiles, image], dim=0) - - if attn_mask.shape[1:3] != comp_attn_mask.shape[1:3]: - comp_attn_mask = F.interpolate(comp_attn_mask.unsqueeze(1), size=(attn_mask.shape[1], attn_mask.shape[2]), mode="bicubic").squeeze(1) - attn_mask = torch.cat([comp_attn_mask, attn_mask], dim=0) - - if blur > 0: - attn_mask = TT.GaussianBlur(int(6*blur+1), blur)(attn_mask.unsqueeze(1)).permute([0,2,3,1]).squeeze(-1) - - return image, attn_mask - -def image_add_noise(image, noise): - image = image.permute([0,3,1,2]) - torch.manual_seed(0) # use a fixed random for reproducible results - transforms = TT.Compose([ - TT.CenterCrop(min(image.shape[2], image.shape[3])), - TT.Resize((224, 224), interpolation=TT.InterpolationMode.BICUBIC, antialias=True), - TT.ElasticTransform(alpha=75.0, sigma=noise*3.5), # shuffle the image - TT.RandomVerticalFlip(p=1.0), # flip the image to change the geometry even more - TT.RandomHorizontalFlip(p=1.0), - ]) - image = transforms(image.cpu()) - image = image.permute([0,2,3,1]) - image = image + ((0.25*(1-noise)+0.05) * torch.randn_like(image) ) # add further random noise - return image - -def zeroed_hidden_states(clip_vision, batch_size): - image = torch.zeros([batch_size, 224, 224, 3]) - comfy.model_management.load_model_gpu(clip_vision.patcher) - pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float() - outputs = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) - - # we only need the penultimate hidden states - return outputs[1].to(comfy.model_management.intermediate_device()) - -def encode_image_masked(clip_vision, image, mask=None): - comfy.model_management.load_model_gpu(clip_vision.patcher) - pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float() - - if mask is not None: - pixel_values = pixel_values * mask.to(clip_vision.load_device) - - out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) - - outputs = Output() - outputs["last_hidden_state"] = out[0].to(comfy.model_management.intermediate_device()) - outputs["image_embeds"] = out[2].to(comfy.model_management.intermediate_device()) - outputs["penultimate_hidden_states"] = out[1].to(comfy.model_management.intermediate_device()) - return outputs - -def min_(tensor_list): - # return the element-wise min of the tensor list. - x = torch.stack(tensor_list) - mn = x.min(axis=0)[0] - return torch.clamp(mn, min=0) - -def max_(tensor_list): - # return the element-wise max of the tensor list. - x = torch.stack(tensor_list) - mx = x.max(axis=0)[0] - return torch.clamp(mx, max=1) - -# From https://github.com/Jamy-L/Pytorch-Contrast-Adaptive-Sharpening/ -def contrast_adaptive_sharpening(image, amount): - img = F.pad(image, pad=(1, 1, 1, 1)).cpu() - - a = img[..., :-2, :-2] - b = img[..., :-2, 1:-1] - c = img[..., :-2, 2:] - d = img[..., 1:-1, :-2] - e = img[..., 1:-1, 1:-1] - f = img[..., 1:-1, 2:] - g = img[..., 2:, :-2] - h = img[..., 2:, 1:-1] - i = img[..., 2:, 2:] - - # Computing contrast - cross = (b, d, e, f, h) - mn = min_(cross) - mx = max_(cross) - - diag = (a, c, g, i) - mn2 = min_(diag) - mx2 = max_(diag) - mx = mx + mx2 - mn = mn + mn2 - - # Computing local weight - inv_mx = torch.reciprocal(mx) - amp = inv_mx * torch.minimum(mn, (2 - mx)) - - # scaling - amp = torch.sqrt(amp) - w = - amp * (amount * (1/5 - 1/8) + 1/8) - div = torch.reciprocal(1 + 4*w) - - output = ((b + d + f + h)*w + e) * div - output = torch.nan_to_num(output) - output = output.clamp(0, 1) - - return (output) - -def tensorToNP(image): - out = torch.clamp(255. * image.detach().cpu(), 0, 255).to(torch.uint8) - out = out[..., [2, 1, 0]] - out = out.numpy() - - return out - -def NPToTensor(image): - out = torch.from_numpy(image) - out = torch.clamp(out.to(torch.float)/255., 0.0, 1.0) - out = out[..., [2, 1, 0]] - - return out - -class IPAdapterImport(nn.Module): - def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False): - super().__init__() - - self.clip_embeddings_dim = clip_embeddings_dim - self.cross_attention_dim = cross_attention_dim - self.output_cross_attention_dim = output_cross_attention_dim - self.clip_extra_context_tokens = clip_extra_context_tokens - self.is_sdxl = is_sdxl - self.is_full = is_full - self.is_plus = is_plus - - if is_faceid: - self.image_proj_model = self.init_proj_faceid() - elif is_plus: - self.image_proj_model = self.init_proj_plus() - else: - self.image_proj_model = self.init_proj() - - self.image_proj_model.load_state_dict(ipadapter_model["image_proj"]) - self.ip_layers = To_KVImport(ipadapter_model["ip_adapter"]) - - def init_proj(self): - image_proj_model = ImageProjModelImport( - cross_attention_dim=self.cross_attention_dim, - clip_embeddings_dim=self.clip_embeddings_dim, - clip_extra_context_tokens=self.clip_extra_context_tokens - ) - return image_proj_model - - def init_proj_plus(self): - if self.is_full: - image_proj_model = MLPProjModelImport( - cross_attention_dim=self.cross_attention_dim, - clip_embeddings_dim=self.clip_embeddings_dim - ) - else: - image_proj_model = ResamplerImport( - dim=self.cross_attention_dim, - depth=4, - dim_head=64, - heads=20 if self.is_sdxl else 12, - num_queries=self.clip_extra_context_tokens, - embedding_dim=self.clip_embeddings_dim, - output_dim=self.output_cross_attention_dim, - ff_mult=4 - ) - return image_proj_model - - def init_proj_faceid(self): - if self.is_plus: - image_proj_model = ProjModelFaceIdPlusImport( - cross_attention_dim=self.cross_attention_dim, - id_embeddings_dim=512, - clip_embeddings_dim=1280, - num_tokens=4, - ) - else: - image_proj_model = MLPProjModelFaceIdImport( - cross_attention_dim=self.cross_attention_dim, - id_embeddings_dim=512, - num_tokens=self.clip_extra_context_tokens, - ) - return image_proj_model - - @torch.inference_mode() - def get_image_embeds(self, clip_embed, clip_embed_zeroed): - image_prompt_embeds = self.image_proj_model(clip_embed) - uncond_image_prompt_embeds = self.image_proj_model(clip_embed_zeroed) - return image_prompt_embeds, uncond_image_prompt_embeds - - @torch.inference_mode() - def get_image_embeds_faceid_plus(self, face_embed, clip_embed, s_scale, shortcut): - embeds = self.image_proj_model(face_embed, clip_embed, scale=s_scale, shortcut=shortcut) - return embeds - -class CrossAttentionPatchImport: - # forward for patching - def __init__(self, weight, ipadapter, number, cond, uncond, weight_type="original", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False): - self.weights = [weight] - self.ipadapters = [ipadapter] - self.conds = [cond] - self.unconds = [uncond] - self.number = number - self.weight_type = [weight_type] - self.masks = [mask] - self.sigma_start = [sigma_start] - self.sigma_end = [sigma_end] - self.unfold_batch = [unfold_batch] - - self.k_key = str(self.number*2+1) + "_to_k_ip" - self.v_key = str(self.number*2+1) + "_to_v_ip" - - def set_new_condition(self, weight, ipadapter, number, cond, uncond, weight_type="original", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False): - self.weights.append(weight) - self.ipadapters.append(ipadapter) - self.conds.append(cond) - self.unconds.append(uncond) - self.masks.append(mask) - self.weight_type.append(weight_type) - self.sigma_start.append(sigma_start) - self.sigma_end.append(sigma_end) - self.unfold_batch.append(unfold_batch) - - def __call__(self, n, context_attn2, value_attn2, extra_options): - org_dtype = n.dtype - cond_or_uncond = extra_options["cond_or_uncond"] - sigma = extra_options["sigmas"][0] if 'sigmas' in extra_options else None - sigma = sigma.item() if sigma is not None else 999999999.9 - - # extra options for AnimateDiff - ad_params = extra_options['ad_params'] if "ad_params" in extra_options else None - - q = n - k = context_attn2 - v = value_attn2 - b = q.shape[0] - qs = q.shape[1] - batch_prompt = b // len(cond_or_uncond) - out = optimized_attention(q, k, v, extra_options["n_heads"]) - _, _, lh, lw = extra_options["original_shape"] - - for weight, cond, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_type, self.sigma_start, self.sigma_end, self.unfold_batch): - if sigma > sigma_start or sigma < sigma_end: - continue - - if unfold_batch and cond.shape[0] > 1: - # Check AnimateDiff context window - if ad_params is not None and ad_params["sub_idxs"] is not None: - # if images length matches or exceeds full_length get sub_idx images - if cond.shape[0] >= ad_params["full_length"]: - cond = torch.Tensor(cond[ad_params["sub_idxs"]]) - uncond = torch.Tensor(uncond[ad_params["sub_idxs"]]) - # otherwise, need to do more to get proper sub_idxs masks - else: - # check if images length matches full_length - if not, make it match - if cond.shape[0] < ad_params["full_length"]: - cond = torch.cat((cond, cond[-1:].repeat((ad_params["full_length"]-cond.shape[0], 1, 1))), dim=0) - uncond = torch.cat((uncond, uncond[-1:].repeat((ad_params["full_length"]-uncond.shape[0], 1, 1))), dim=0) - # if we have too many remove the excess (should not happen, but just in case) - if cond.shape[0] > ad_params["full_length"]: - cond = cond[:ad_params["full_length"]] - uncond = uncond[:ad_params["full_length"]] - cond = cond[ad_params["sub_idxs"]] - uncond = uncond[ad_params["sub_idxs"]] - - # if we don't have enough reference images repeat the last one until we reach the right size - if cond.shape[0] < batch_prompt: - cond = torch.cat((cond, cond[-1:].repeat((batch_prompt-cond.shape[0], 1, 1))), dim=0) - uncond = torch.cat((uncond, uncond[-1:].repeat((batch_prompt-uncond.shape[0], 1, 1))), dim=0) - # if we have too many remove the exceeding - elif cond.shape[0] > batch_prompt: - cond = cond[:batch_prompt] - uncond = uncond[:batch_prompt] - - k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond) - k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond) - v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond) - v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond) - else: - k_cond = ipadapter.ip_layers.to_kvs[self.k_key](cond).repeat(batch_prompt, 1, 1) - k_uncond = ipadapter.ip_layers.to_kvs[self.k_key](uncond).repeat(batch_prompt, 1, 1) - v_cond = ipadapter.ip_layers.to_kvs[self.v_key](cond).repeat(batch_prompt, 1, 1) - v_uncond = ipadapter.ip_layers.to_kvs[self.v_key](uncond).repeat(batch_prompt, 1, 1) - - if weight_type.startswith("linear"): - ip_k = torch.cat([(k_cond, k_uncond)[i] for i in cond_or_uncond], dim=0) * weight - ip_v = torch.cat([(v_cond, v_uncond)[i] for i in cond_or_uncond], dim=0) * weight - else: - ip_k = torch.cat([(k_cond, k_uncond)[i] for i in cond_or_uncond], dim=0) - ip_v = torch.cat([(v_cond, v_uncond)[i] for i in cond_or_uncond], dim=0) - - if weight_type.startswith("channel"): - # code by Lvmin Zhang at Stanford University as also seen on Fooocus IPAdapter implementation - ip_v_mean = torch.mean(ip_v, dim=1, keepdim=True) - ip_v_offset = ip_v - ip_v_mean - _, _, C = ip_k.shape - channel_penalty = float(C) / 1280.0 - W = weight * channel_penalty - ip_k = ip_k * W - ip_v = ip_v_offset + ip_v_mean * W - - out_ip = optimized_attention(q, ip_k, ip_v, extra_options["n_heads"]) - if weight_type.startswith("original"): - out_ip = out_ip * weight - - if mask is not None: - # TODO: needs checking - mask_h = lh / math.sqrt(lh * lw / qs) - mask_h = int(mask_h) + int((qs % int(mask_h)) != 0) - mask_w = qs // mask_h - - # check if using AnimateDiff and sliding context window - if (mask.shape[0] > 1 and ad_params is not None and ad_params["sub_idxs"] is not None): - # if mask length matches or exceeds full_length, just get sub_idx masks, resize, and continue - if mask.shape[0] >= ad_params["full_length"]: - mask_downsample = torch.Tensor(mask[ad_params["sub_idxs"]]) - mask_downsample = F.interpolate(mask_downsample.unsqueeze(1), size=(mask_h, mask_w), mode="bicubic").squeeze(1) - # otherwise, need to do more to get proper sub_idxs masks - else: - # resize to needed attention size (to save on memory) - mask_downsample = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bicubic").squeeze(1) - # check if mask length matches full_length - if not, make it match - if mask_downsample.shape[0] < ad_params["full_length"]: - mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:].repeat((ad_params["full_length"]-mask_downsample.shape[0], 1, 1))), dim=0) - # if we have too many remove the excess (should not happen, but just in case) - if mask_downsample.shape[0] > ad_params["full_length"]: - mask_downsample = mask_downsample[:ad_params["full_length"]] - # now, select sub_idxs masks - mask_downsample = mask_downsample[ad_params["sub_idxs"]] - # otherwise, perform usual mask interpolation - else: - mask_downsample = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bicubic").squeeze(1) - - # if we don't have enough masks repeat the last one until we reach the right size - if mask_downsample.shape[0] < batch_prompt: - mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:, :, :].repeat((batch_prompt-mask_downsample.shape[0], 1, 1))), dim=0) - # if we have too many remove the exceeding - elif mask_downsample.shape[0] > batch_prompt: - mask_downsample = mask_downsample[:batch_prompt, :, :] - - # repeat the masks - mask_downsample = mask_downsample.repeat(len(cond_or_uncond), 1, 1) - mask_downsample = mask_downsample.view(mask_downsample.shape[0], -1, 1).repeat(1, 1, out.shape[2]) - - out_ip = out_ip * mask_downsample - - out = out + out_ip - - return out.to(dtype=org_dtype) - -class IPAdapterModelLoaderImport: - @classmethod - def INPUT_TYPES(s): - return {"required": { "ipadapter_file": (folder_paths.get_filename_list("ipadapter"), )}} - - RETURN_TYPES = ("IPADAPTER",) - FUNCTION = "load_ipadapter_model" - CATEGORY = "ipadapter" - - def load_ipadapter_model(self, ipadapter_file): - ckpt_path = folder_paths.get_full_path("ipadapter", ipadapter_file) - - model = comfy.utils.load_torch_file(ckpt_path, safe_load=True) - - if ckpt_path.lower().endswith(".safetensors"): - st_model = {"image_proj": {}, "ip_adapter": {}} - for key in model.keys(): - if key.startswith("image_proj."): - st_model["image_proj"][key.replace("image_proj.", "")] = model[key] - elif key.startswith("ip_adapter."): - st_model["ip_adapter"][key.replace("ip_adapter.", "")] = model[key] - model = st_model - - if not "ip_adapter" in model.keys() or not model["ip_adapter"]: - raise Exception("invalid IPAdapter model {}".format(ckpt_path)) - - return (model,) - -insightface_face_align = None -class InsightFaceLoaderImport: - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "provider": (["CPU", "CUDA", "ROCM"], ), - }, - } - - RETURN_TYPES = ("INSIGHTFACE",) - FUNCTION = "load_insight_face" - CATEGORY = "ipadapter" - - def load_insight_face(self, provider): - try: - from insightface.app import FaceAnalysis - except ImportError as e: - raise Exception(e) - - from insightface.utils import face_align - global insightface_face_align - insightface_face_align = face_align - - model = FaceAnalysis(name="buffalo_l", root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider',]) - model.prepare(ctx_id=0, det_size=(640, 640)) - - return (model,) - -class IPAdapterApplyImport: - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "ipadapter": ("IPADAPTER", ), - "clip_vision": ("CLIP_VISION",), - "image": ("IMAGE",), - "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), - "weight_type": (["original", "linear", "channel penalty"], ), - "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "unfold_batch": ("BOOLEAN", { "default": False }), - }, - "optional": { - "attn_mask": ("MASK",), - } - } - - RETURN_TYPES = ("MODEL",) - FUNCTION = "apply_ipadapter" - CATEGORY = "ipadapter" - - def apply_ipadapter(self, - ipadapter, - model, - weight, - clip_vision=None, - image=None, - weight_type="original", - noise=None, - embeds=None, - attn_mask=None, - start_at=0.0, - end_at=1.0, - unfold_batch=False, - insightface=None, - faceid_v2=False, - weight_v2=False, - clip_vision_mask=None, - short_side_tiles=0, - tile_weight=0.0, - tile_blur=0, - ): - - self.dtype = torch.float16 if comfy.model_management.should_use_fp16() else torch.float32 - self.device = comfy.model_management.get_torch_device() - self.weight = weight - self.is_full = "proj.3.weight" in ipadapter["image_proj"] - self.is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] - self.is_faceid = self.is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] - self.is_plus = (self.is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]) - is_tiled = True if short_side_tiles > 0 else False - - if is_tiled: - image, attn_mask = masked_tiling(image, short_side_tiles, tile_weight, tile_blur) - - if self.is_faceid and not insightface: - raise Exception('InsightFace must be provided for FaceID models.') - - output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1] - self.is_sdxl = output_cross_attention_dim == 2048 - cross_attention_dim = 1280 if self.is_plus and self.is_sdxl and not self.is_faceid else output_cross_attention_dim - clip_extra_context_tokens = 16 if self.is_plus or self.is_portrait else 4 - - if embeds is not None: - embeds = torch.unbind(embeds) - clip_embed = embeds[0].cpu() - clip_embed_zeroed = embeds[1].cpu() - else: - if self.is_faceid: - insightface.det_model.input_size = (640,640) # reset the detection size - face_img = tensorToNP(image) - face_embed = [] - face_clipvision = [] - - for i in range(face_img.shape[0]): - for size in [(size, size) for size in range(640, 128, -64)]: - insightface.det_model.input_size = size # TODO: hacky but seems to be working - face = insightface.get(face_img[i]) - if face: - face_embed.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0)) - face_clipvision.append(NPToTensor(insightface_face_align.norm_crop(face_img[i], landmark=face[0].kps, image_size=256))) - - if 640 not in size: - print(f"\033[33mINFO: InsightFace detection resolution lowered to {size}.\033[0m") - break - else: - raise Exception('InsightFace: No face detected.') - - face_embed = torch.stack(face_embed, dim=0) - image = torch.stack(face_clipvision, dim=0) - - neg_image = image_add_noise(image, noise) if noise > 0 else None - - if self.is_plus: - clip_embed = clip_vision.encode_image(image).penultimate_hidden_states - if noise > 0: - clip_embed_zeroed = clip_vision.encode_image(neg_image).penultimate_hidden_states - else: - clip_embed_zeroed = zeroed_hidden_states(clip_vision, image.shape[0]) - - # TODO: check noise to the uncods too - face_embed_zeroed = torch.zeros_like(face_embed) - else: - clip_embed = face_embed - clip_embed_zeroed = torch.zeros_like(clip_embed) - else: - if image.shape[1] != image.shape[2]: - print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m") - - clip_embed = encode_image_masked(clip_vision, image, clip_vision_mask) - neg_image = image_add_noise(image, noise) if noise > 0 else None - - if self.is_plus: - clip_embed = clip_embed.penultimate_hidden_states - if noise > 0: - clip_embed_zeroed = clip_vision.encode_image(neg_image).penultimate_hidden_states - else: - clip_embed_zeroed = zeroed_hidden_states(clip_vision, image.shape[0]) - else: - clip_embed = clip_embed.image_embeds - if noise > 0: - clip_embed_zeroed = clip_vision.encode_image(neg_image).image_embeds - else: - clip_embed_zeroed = torch.zeros_like(clip_embed) - - clip_embeddings_dim = clip_embed.shape[-1] - - self.ipadapter = IPAdapterImport( - ipadapter, - cross_attention_dim=cross_attention_dim, - output_cross_attention_dim=output_cross_attention_dim, - clip_embeddings_dim=clip_embeddings_dim, - clip_extra_context_tokens=clip_extra_context_tokens, - is_sdxl=self.is_sdxl, - is_plus=self.is_plus, - is_full=self.is_full, - is_faceid=self.is_faceid, - ) - - self.ipadapter.to(self.device, dtype=self.dtype) - - if self.is_faceid and self.is_plus: - image_prompt_embeds = self.ipadapter.get_image_embeds_faceid_plus(face_embed.to(self.device, dtype=self.dtype), clip_embed.to(self.device, dtype=self.dtype), weight_v2, faceid_v2) - uncond_image_prompt_embeds = self.ipadapter.get_image_embeds_faceid_plus(face_embed_zeroed.to(self.device, dtype=self.dtype), clip_embed_zeroed.to(self.device, dtype=self.dtype), weight_v2, faceid_v2) - else: - image_prompt_embeds, uncond_image_prompt_embeds = self.ipadapter.get_image_embeds(clip_embed.to(self.device, dtype=self.dtype), clip_embed_zeroed.to(self.device, dtype=self.dtype)) - - image_prompt_embeds = image_prompt_embeds.to(self.device, dtype=self.dtype) - uncond_image_prompt_embeds = uncond_image_prompt_embeds.to(self.device, dtype=self.dtype) - - self.work_model = model.clone() - - if attn_mask is not None: - attn_mask = attn_mask.to(self.device) - - sigma_start = self.work_model.model.model_sampling.percent_to_sigma(start_at) - sigma_end = self.work_model.model.model_sampling.percent_to_sigma(end_at) - - if is_tiled: - for i in range(image_prompt_embeds.shape[0]): #TODO: check if we can apply one mask per image in the attention patch phase - patch_kwargs = { - "number": 0, - "weight": self.weight, - "ipadapter": self.ipadapter, - "cond": image_prompt_embeds[i].unsqueeze(0), - "uncond": uncond_image_prompt_embeds[i].unsqueeze(0), - "weight_type": weight_type, - "mask": attn_mask[i].unsqueeze(0), - "sigma_start": sigma_start, - "sigma_end": sigma_end, - "unfold_batch": unfold_batch, - } - self.apply_patch(patch_kwargs) - else: - patch_kwargs = { - "number": 0, - "weight": self.weight, - "ipadapter": self.ipadapter, - "cond": image_prompt_embeds, - "uncond": uncond_image_prompt_embeds, - "weight_type": weight_type, - "mask": attn_mask, - "sigma_start": sigma_start, - "sigma_end": sigma_end, - "unfold_batch": unfold_batch, - } - self.apply_patch(patch_kwargs) - - return (self.work_model, attn_mask, image,) - - def apply_patch(self, patch_kwargs): - if not self.is_sdxl: - for id in [1,2,4,5,7,8]: # id of input_blocks that have cross attention - set_model_patch_replace(self.work_model, patch_kwargs, ("input", id)) - patch_kwargs["number"] += 1 - for id in [3,4,5,6,7,8,9,10,11]: # id of output_blocks that have cross attention - set_model_patch_replace(self.work_model, patch_kwargs, ("output", id)) - patch_kwargs["number"] += 1 - set_model_patch_replace(self.work_model, patch_kwargs, ("middle", 0)) - else: - for id in [4,5,7,8]: # id of input_blocks that have cross attention - block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth - for index in block_indices: - set_model_patch_replace(self.work_model, patch_kwargs, ("input", id, index)) - patch_kwargs["number"] += 1 - for id in range(6): # id of output_blocks that have cross attention - block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth - for index in block_indices: - set_model_patch_replace(self.work_model, patch_kwargs, ("output", id, index)) - patch_kwargs["number"] += 1 - for index in range(10): - set_model_patch_replace(self.work_model, patch_kwargs, ("middle", 0, index)) - patch_kwargs["number"] += 1 - -class IPAdapterTilesMaskedImport(IPAdapterApplyImport): - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "ipadapter": ("IPADAPTER", ), - "clip_vision": ("CLIP_VISION",), - "image": ("IMAGE",), - "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 0.7, "min": -1, "max": 3, "step": 0.05 }), - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), - "weight_type": (["original", "linear", "channel penalty"], ), - "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "short_side_tiles": ("INT", { "default": 1, "min": 0, "max": 12, "step": 1 }), - "tile_weight": ("FLOAT", { "default": 0.6, "min": 0.0, "max": 1.0, "step": 0.05 }), - #"tile_blur": ("INT", { "default": 0, "min": 0, "max": 112, "step": 1 }), - }, - } - - RETURN_TYPES = ("MODEL",) - -class IPAdapterApplyFaceIDImport(IPAdapterApplyImport): - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "ipadapter": ("IPADAPTER", ), - "clip_vision": ("CLIP_VISION",), - "insightface": ("INSIGHTFACE",), - "image": ("IMAGE",), - "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), - "weight_type": (["original", "linear", "channel penalty"], ), - "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "faceid_v2": ("BOOLEAN", { "default": False }), - "weight_v2": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "unfold_batch": ("BOOLEAN", { "default": False }), - }, - "optional": { - "attn_mask": ("MASK",), - } - } - -def prepImage(image, interpolation="LANCZOS", crop_position="center", size=(224,224), sharpening=0.0, padding=0): - _, oh, ow, _ = image.shape - output = image.permute([0,3,1,2]) - - if "pad" in crop_position: - target_length = max(oh, ow) - pad_l = (target_length - ow) // 2 - pad_r = (target_length - ow) - pad_l - pad_t = (target_length - oh) // 2 - pad_b = (target_length - oh) - pad_t - output = F.pad(output, (pad_l, pad_r, pad_t, pad_b), value=0, mode="constant") - else: - crop_size = min(oh, ow) - x = (ow-crop_size) // 2 - y = (oh-crop_size) // 2 - if "top" in crop_position: - y = 0 - elif "bottom" in crop_position: - y = oh-crop_size - elif "left" in crop_position: - x = 0 - elif "right" in crop_position: - x = ow-crop_size - - x2 = x+crop_size - y2 = y+crop_size - - # crop - output = output[:, :, y:y2, x:x2] - - # resize (apparently PIL resize is better than tourchvision interpolate) - imgs = [] - for i in range(output.shape[0]): - img = TT.ToPILImage()(output[i]) - img = img.resize(size, resample=Image.Resampling[interpolation]) - imgs.append(TT.ToTensor()(img)) - output = torch.stack(imgs, dim=0) - imgs = None # zelous GC - - if sharpening > 0: - output = contrast_adaptive_sharpening(output, sharpening) - - if padding > 0: - output = F.pad(output, (padding, padding, padding, padding), value=255, mode="constant") - - output = output.permute([0,2,3,1]) - - return output - -class PrepImageForInsightFaceImport: - @classmethod - def INPUT_TYPES(s): - return {"required": { - "image": ("IMAGE",), - "crop_position": (["center", "top", "bottom", "left", "right"],), - "sharpening": ("FLOAT", {"default": 0.0, "min": 0, "max": 1, "step": 0.05}), - "pad_around": ("BOOLEAN", { "default": True }), - }, - } - - RETURN_TYPES = ("IMAGE",) - FUNCTION = "prep_image" - - CATEGORY = "ipadapter" - - def prep_image(self, image, crop_position, sharpening=0.0, pad_around=True): - if pad_around: - padding = 30 - size = (580, 580) - else: - padding = 0 - size = (640, 640) - output = prepImage(image, "LANCZOS", crop_position, size, sharpening, padding) - - return (output, ) - -class PrepImageForClipVisionImport: - @classmethod - def INPUT_TYPES(s): - return {"required": { - "image": ("IMAGE",), - "interpolation": (["LANCZOS", "BICUBIC", "HAMMING", "BILINEAR", "BOX", "NEAREST"],), - "crop_position": (["top", "bottom", "left", "right", "center", "pad"],), - "sharpening": ("FLOAT", {"default": 0.0, "min": 0, "max": 1, "step": 0.05}), - }, - } - - RETURN_TYPES = ("IMAGE",) - FUNCTION = "prep_image" - - CATEGORY = "ipadapter" - - def prep_image(self, image, interpolation="LANCZOS", crop_position="center", sharpening=0.0): - size = (224, 224) - output = prepImage(image, interpolation, crop_position, size, sharpening, 0) - return (output, ) - -class IPAdapterEncoderImport: - @classmethod - def INPUT_TYPES(s): - return {"required": { - "clip_vision": ("CLIP_VISION",), - "image_1": ("IMAGE",), - "ipadapter_plus": ("BOOLEAN", { "default": False }), - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), - "weight_1": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), - }, - "optional": { - "image_2": ("IMAGE",), - "image_3": ("IMAGE",), - "image_4": ("IMAGE",), - "weight_2": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), - "weight_3": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), - "weight_4": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), - } - } - - RETURN_TYPES = ("EMBEDS",) - FUNCTION = "preprocess" - CATEGORY = "ipadapter" - - def preprocess(self, clip_vision, image_1, ipadapter_plus, noise, weight_1, image_2=None, image_3=None, image_4=None, weight_2=1.0, weight_3=1.0, weight_4=1.0): - weight_1 *= (0.1 + (weight_1 - 0.1)) - weight_2 *= (0.1 + (weight_2 - 0.1)) - weight_3 *= (0.1 + (weight_3 - 0.1)) - weight_4 *= (0.1 + (weight_4 - 0.1)) - - image = image_1 - weight = [weight_1]*image_1.shape[0] - - if image_2 is not None: - if image_1.shape[1:] != image_2.shape[1:]: - image_2 = comfy.utils.common_upscale(image_2.movedim(-1,1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1,-1) - image = torch.cat((image, image_2), dim=0) - weight += [weight_2]*image_2.shape[0] - if image_3 is not None: - if image.shape[1:] != image_3.shape[1:]: - image_3 = comfy.utils.common_upscale(image_3.movedim(-1,1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1,-1) - image = torch.cat((image, image_3), dim=0) - weight += [weight_3]*image_3.shape[0] - if image_4 is not None: - if image.shape[1:] != image_4.shape[1:]: - image_4 = comfy.utils.common_upscale(image_4.movedim(-1,1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1,-1) - image = torch.cat((image, image_4), dim=0) - weight += [weight_4]*image_4.shape[0] - - clip_embed = clip_vision.encode_image(image) - neg_image = image_add_noise(image, noise) if noise > 0 else None - - if ipadapter_plus: - clip_embed = clip_embed.penultimate_hidden_states - if noise > 0: - clip_embed_zeroed = clip_vision.encode_image(neg_image).penultimate_hidden_states - else: - clip_embed_zeroed = zeroed_hidden_states(clip_vision, image.shape[0]) - else: - clip_embed = clip_embed.image_embeds - if noise > 0: - clip_embed_zeroed = clip_vision.encode_image(neg_image).image_embeds - else: - clip_embed_zeroed = torch.zeros_like(clip_embed) - - if any(e != 1.0 for e in weight): - weight = torch.tensor(weight).unsqueeze(-1) if not ipadapter_plus else torch.tensor(weight).unsqueeze(-1).unsqueeze(-1) - clip_embed = clip_embed * weight - - output = torch.stack((clip_embed, clip_embed_zeroed)) - - return( output, ) - -class IPAdapterApplyEncodedImport(IPAdapterApplyImport): - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "ipadapter": ("IPADAPTER", ), - "embeds": ("EMBEDS",), - "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "weight_type": (["original", "linear", "channel penalty"], ), - "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "unfold_batch": ("BOOLEAN", { "default": False }), - }, - "optional": { - "attn_mask": ("MASK",), - } - } - -class IPAdapterSaveEmbedsImport: - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - - @classmethod - def INPUT_TYPES(s): - return {"required": { - "embeds": ("EMBEDS",), - "filename_prefix": ("STRING", {"default": "embeds/IPAdapter"}) - }, - } - - RETURN_TYPES = () - FUNCTION = "save" - OUTPUT_NODE = True - CATEGORY = "ipadapter" - - def save(self, embeds, filename_prefix): - full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) - file = f"{filename}_{counter:05}_.ipadpt" - file = os.path.join(full_output_folder, file) - - torch.save(embeds, file) - return (None, ) - - -class IPAdapterLoadEmbedsImport: - @classmethod - def INPUT_TYPES(s): - input_dir = folder_paths.get_input_directory() - files = [os.path.relpath(os.path.join(root, file), input_dir) for root, dirs, files in os.walk(input_dir) for file in files if file.endswith('.ipadpt')] - return {"required": {"embeds": [sorted(files), ]}, } - - RETURN_TYPES = ("EMBEDS", ) - FUNCTION = "load" - CATEGORY = "ipadapter" - - def load(self, embeds): - path = folder_paths.get_annotated_filepath(embeds) - output = torch.load(path).cpu() - - return (output, ) - - -class IPAdapterBatchEmbedsImport: - @classmethod - def INPUT_TYPES(s): - return {"required": { - "embed1": ("EMBEDS",), - "embed2": ("EMBEDS",), - }} - - RETURN_TYPES = ("EMBEDS",) - FUNCTION = "batch" - CATEGORY = "ipadapter" - - def batch(self, embed1, embed2): - return (torch.cat((embed1, embed2), dim=1), ) - - - -import math - -import torch -import torch.nn as nn -from einops import rearrange -from einops.layers.torch import Rearrange - - -# FFN -def FeedForwardImport(dim, mult=4): - inner_dim = int(dim * mult) - return nn.Sequential( - nn.LayerNorm(dim), - nn.Linear(dim, inner_dim, bias=False), - nn.GELU(), - nn.Linear(inner_dim, dim, bias=False), - ) - - -def reshape_tensor(x, heads): - bs, length, width = x.shape - # (bs, length, width) --> (bs, length, n_heads, dim_per_head) - x = x.view(bs, length, heads, -1) - # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head) - x = x.transpose(1, 2) - # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head) - x = x.reshape(bs, heads, length, -1) - return x - - -class PerceiverAttentionImport(nn.Module): - def __init__(self, *, dim, dim_head=64, heads=8): - super().__init__() - self.scale = dim_head**-0.5 - self.dim_head = dim_head - self.heads = heads - inner_dim = dim_head * heads - - self.norm1 = nn.LayerNorm(dim) - self.norm2 = nn.LayerNorm(dim) - - self.to_q = nn.Linear(dim, inner_dim, bias=False) - self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False) - self.to_out = nn.Linear(inner_dim, dim, bias=False) - - def forward(self, x, latents): - """ - Args: - x (torch.Tensor): image features - shape (b, n1, D) - latent (torch.Tensor): latent features - shape (b, n2, D) - """ - x = self.norm1(x) - latents = self.norm2(latents) - - b, l, _ = latents.shape - - q = self.to_q(latents) - kv_input = torch.cat((x, latents), dim=-2) - k, v = self.to_kv(kv_input).chunk(2, dim=-1) - - q = reshape_tensor(q, self.heads) - k = reshape_tensor(k, self.heads) - v = reshape_tensor(v, self.heads) - - # attention - scale = 1 / math.sqrt(math.sqrt(self.dim_head)) - weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards - weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) - out = weight @ v - - out = out.permute(0, 2, 1, 3).reshape(b, l, -1) - - return self.to_out(out) - - -class ResamplerImport(nn.Module): - def __init__( - self, - dim=1024, - depth=8, - dim_head=64, - heads=16, - num_queries=8, - embedding_dim=768, - output_dim=1024, - ff_mult=4, - max_seq_len: int = 257, # CLIP tokens + CLS token - apply_pos_emb: bool = False, - num_latents_mean_pooled: int = 0, # number of latents derived from mean pooled representation of the sequence - ): - super().__init__() - self.pos_emb = nn.Embedding(max_seq_len, embedding_dim) if apply_pos_emb else None - - self.latents = nn.Parameter(torch.randn(1, num_queries, dim) / dim**0.5) - - self.proj_in = nn.Linear(embedding_dim, dim) - - self.proj_out = nn.Linear(dim, output_dim) - self.norm_out = nn.LayerNorm(output_dim) - - self.to_latents_from_mean_pooled_seq = ( - nn.Sequential( - nn.LayerNorm(dim), - nn.Linear(dim, dim * num_latents_mean_pooled), - Rearrange("b (n d) -> b n d", n=num_latents_mean_pooled), - ) - if num_latents_mean_pooled > 0 - else None - ) - - self.layers = nn.ModuleList([]) - for _ in range(depth): - self.layers.append( - nn.ModuleList( - [ - PerceiverAttentionImport(dim=dim, dim_head=dim_head, heads=heads), - FeedForwardImport(dim=dim, mult=ff_mult), - ] - ) - ) - - def forward(self, x): - if self.pos_emb is not None: - n, device = x.shape[1], x.device - pos_emb = self.pos_emb(torch.arange(n, device=device)) - x = x + pos_emb - - latents = self.latents.repeat(x.size(0), 1, 1) - - x = self.proj_in(x) - - if self.to_latents_from_mean_pooled_seq: - meanpooled_seq = masked_mean(x, dim=1, mask=torch.ones(x.shape[:2], device=x.device, dtype=torch.bool)) - meanpooled_latents = self.to_latents_from_mean_pooled_seq(meanpooled_seq) - latents = torch.cat((meanpooled_latents, latents), dim=-2) - - for attn, ff in self.layers: - latents = attn(x, latents) + latents - latents = ff(latents) + latents - - latents = self.proj_out(latents) - return self.norm_out(latents) - - -def masked_mean(t, *, dim, mask=None): - if mask is None: - return t.mean(dim=dim) - - denom = mask.sum(dim=dim, keepdim=True) - mask = rearrange(mask, "b n -> b n 1") - masked_t = t.masked_fill(~mask, 0.0) - - return masked_t.sum(dim=dim) / denom.clamp(min=1e-5) - - -def prep_image(image, interpolation="LANCZOS", crop_position="center", sharpening=0.0): - _, oh, ow, _ = image.shape - output = image.permute([0,3,1,2]) - - if "pad" in crop_position: - target_length = max(oh, ow) - pad_l = (target_length - ow) // 2 - pad_r = (target_length - ow) - pad_l - pad_t = (target_length - oh) // 2 - pad_b = (target_length - oh) - pad_t - output = F.pad(output, (pad_l, pad_r, pad_t, pad_b), value=0, mode="constant") - else: - crop_size = min(oh, ow) - x = (ow-crop_size) // 2 - y = (oh-crop_size) // 2 - if "top" in crop_position: - y = 0 - elif "bottom" in crop_position: - y = oh-crop_size - elif "left" in crop_position: - x = 0 - elif "right" in crop_position: - x = ow-crop_size - - x2 = x+crop_size - y2 = y+crop_size - - # crop - output = output[:, :, y:y2, x:x2] - - # resize (apparently PIL resize is better than tourchvision interpolate) - imgs = [] - for i in range(output.shape[0]): - img = TT.ToPILImage()(output[i]) - img = img.resize((224,224), resample=Image.Resampling[interpolation]) - imgs.append(TT.ToTensor()(img)) - output = torch.stack(imgs, dim=0) - - if sharpening > 0: - output = contrast_adaptive_sharpening(output, sharpening) - - output = output.permute([0,2,3,1]) - - return (output,) \ No newline at end of file