jakechai
53ae277d77
1. Add SAI API Replace Background and Relight. Add missing parameters of SAI API nodes. 2. Add Detail Daemon Custom Nodes to most of image|mesh generation workflows and group nodes. 3. Add SDXL Target Res JK🐉 node to fix SDXL Text Encode Target Resolution not working. 4. Add Image | Latent Crop by Mask, Resize, Crop by Mask and Resize, Stitch nodes. Add Crop and Stitch operation for Image Gen and Inpaint Group Nodes. 5. Update img2mesh MV upscale method to achieve better results: SD15 img2img + SDXL Refine + Ultimate Upscale. 6. Add Bool_Or and Bool_And nodes for workflows need to be group. Replace Bool Binary Operation node (Or as default) with Bool_Or. 7. Add Color Grading Node. 8. Seperate Multi-ControlNet Param Stack node to ControlNet Loader and Multi-ControlNet Param Stack. 9. Add "None" selection as default to IPAdapter Unified Loader for better workflow control, especially in API mode. 10. Add Image Refine Group Node. 11. Krita image generation workflows updated. 12. Add Krita Refine, Upscale and Refine, Hand fix, CN preprocessor, remove bg and SAI API module series. 13. Remove JK🐉::Pad Image for Outpainting. Remove JK🐉::CLIPSegMask group node, replace it with Essentials CLIPSeg node. 14. Remove [WAS Node Suite] and [Layer Style] Custom Node from must-have-list for their complexity. 15. Remove useless Group Nodes: JK🐉::Image to Prompt (LLava Local), JK🐉::Image to Prompt Advanced (LLava Local), JK🐉::SegAnythingMask, JK🐉::txt2img. 16. SD3 | Flux image generation workflow | group nodes support Stop At Clip Layer (needs ComfyUI v0.3.8up).
2033 lines
89 KiB
Python
2033 lines
89 KiB
Python
import torch
|
|
import os
|
|
import math
|
|
import folder_paths
|
|
|
|
import comfy.model_management as model_management
|
|
from node_helpers import conditioning_set_values
|
|
from comfy.clip_vision import load as load_clip_vision
|
|
from comfy.sd import load_lora_for_models
|
|
import comfy.utils
|
|
|
|
import torch.nn as nn
|
|
from PIL import Image
|
|
try:
|
|
import torchvision.transforms.v2 as T
|
|
except ImportError:
|
|
import torchvision.transforms as T
|
|
|
|
from .image_proj_models import MLPProjModel, MLPProjModelFaceId, ProjModelFaceIdPlus, Resampler, ImageProjModel
|
|
from .CrossAttentionPatch import Attn2Replace, ipadapter_attention
|
|
from .utils import (
|
|
encode_image_masked,
|
|
tensor_to_size,
|
|
contrast_adaptive_sharpening,
|
|
tensor_to_image,
|
|
image_to_tensor,
|
|
ipadapter_model_loader,
|
|
insightface_loader,
|
|
get_clipvision_file,
|
|
get_ipadapter_file,
|
|
get_lora_file,
|
|
)
|
|
|
|
# set the models directory
|
|
if "ipadapter" not in folder_paths.folder_names_and_paths:
|
|
current_paths = [os.path.join(folder_paths.models_dir, "ipadapter")]
|
|
else:
|
|
current_paths, _ = folder_paths.folder_names_and_paths["ipadapter"]
|
|
folder_paths.folder_names_and_paths["ipadapter"] = (current_paths, folder_paths.supported_pt_extensions)
|
|
|
|
WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out', 'weak input', 'weak output', 'weak middle', 'strong middle', 'style transfer', 'composition', 'strong style transfer', 'style and composition', 'style transfer precise', 'composition precise']
|
|
|
|
"""
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
Main IPAdapter Class
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
"""
|
|
class IPAdapter(nn.Module):
|
|
def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False, is_portrait_unnorm=False, is_kwai_kolors=False, encoder_hid_proj=None, weight_kolors=1.0):
|
|
super().__init__()
|
|
|
|
self.clip_embeddings_dim = clip_embeddings_dim
|
|
self.cross_attention_dim = cross_attention_dim
|
|
self.output_cross_attention_dim = output_cross_attention_dim
|
|
self.clip_extra_context_tokens = clip_extra_context_tokens
|
|
self.is_sdxl = is_sdxl
|
|
self.is_full = is_full
|
|
self.is_plus = is_plus
|
|
self.is_portrait_unnorm = is_portrait_unnorm
|
|
self.is_kwai_kolors = is_kwai_kolors
|
|
|
|
if is_faceid and not is_portrait_unnorm:
|
|
self.image_proj_model = self.init_proj_faceid()
|
|
elif is_full:
|
|
self.image_proj_model = self.init_proj_full()
|
|
elif is_plus or is_portrait_unnorm:
|
|
self.image_proj_model = self.init_proj_plus()
|
|
else:
|
|
self.image_proj_model = self.init_proj()
|
|
|
|
self.image_proj_model.load_state_dict(ipadapter_model["image_proj"])
|
|
self.ip_layers = To_KV(ipadapter_model["ip_adapter"], encoder_hid_proj=encoder_hid_proj, weight_kolors=weight_kolors)
|
|
|
|
def init_proj(self):
|
|
image_proj_model = ImageProjModel(
|
|
cross_attention_dim=self.cross_attention_dim,
|
|
clip_embeddings_dim=self.clip_embeddings_dim,
|
|
clip_extra_context_tokens=self.clip_extra_context_tokens
|
|
)
|
|
return image_proj_model
|
|
|
|
def init_proj_plus(self):
|
|
image_proj_model = Resampler(
|
|
dim=self.cross_attention_dim,
|
|
depth=4,
|
|
dim_head=64,
|
|
heads=20 if self.is_sdxl and not self.is_kwai_kolors else 12,
|
|
num_queries=self.clip_extra_context_tokens,
|
|
embedding_dim=self.clip_embeddings_dim,
|
|
output_dim=self.output_cross_attention_dim,
|
|
ff_mult=4
|
|
)
|
|
return image_proj_model
|
|
|
|
def init_proj_full(self):
|
|
image_proj_model = MLPProjModel(
|
|
cross_attention_dim=self.cross_attention_dim,
|
|
clip_embeddings_dim=self.clip_embeddings_dim
|
|
)
|
|
return image_proj_model
|
|
|
|
def init_proj_faceid(self):
|
|
if self.is_plus:
|
|
image_proj_model = ProjModelFaceIdPlus(
|
|
cross_attention_dim=self.cross_attention_dim,
|
|
id_embeddings_dim=512,
|
|
clip_embeddings_dim=self.clip_embeddings_dim,
|
|
num_tokens=self.clip_extra_context_tokens,
|
|
)
|
|
else:
|
|
image_proj_model = MLPProjModelFaceId(
|
|
cross_attention_dim=self.cross_attention_dim,
|
|
id_embeddings_dim=512,
|
|
num_tokens=self.clip_extra_context_tokens,
|
|
)
|
|
return image_proj_model
|
|
|
|
@torch.inference_mode()
|
|
def get_image_embeds(self, clip_embed, clip_embed_zeroed, batch_size):
|
|
torch_device = model_management.get_torch_device()
|
|
intermediate_device = model_management.intermediate_device()
|
|
|
|
if batch_size == 0:
|
|
batch_size = clip_embed.shape[0]
|
|
intermediate_device = torch_device
|
|
elif batch_size > clip_embed.shape[0]:
|
|
batch_size = clip_embed.shape[0]
|
|
|
|
clip_embed = torch.split(clip_embed, batch_size, dim=0)
|
|
clip_embed_zeroed = torch.split(clip_embed_zeroed, batch_size, dim=0)
|
|
|
|
image_prompt_embeds = []
|
|
uncond_image_prompt_embeds = []
|
|
|
|
for ce, cez in zip(clip_embed, clip_embed_zeroed):
|
|
image_prompt_embeds.append(self.image_proj_model(ce.to(torch_device)).to(intermediate_device))
|
|
uncond_image_prompt_embeds.append(self.image_proj_model(cez.to(torch_device)).to(intermediate_device))
|
|
|
|
del clip_embed, clip_embed_zeroed
|
|
|
|
image_prompt_embeds = torch.cat(image_prompt_embeds, dim=0)
|
|
uncond_image_prompt_embeds = torch.cat(uncond_image_prompt_embeds, dim=0)
|
|
|
|
torch.cuda.empty_cache()
|
|
|
|
#image_prompt_embeds = self.image_proj_model(clip_embed)
|
|
#uncond_image_prompt_embeds = self.image_proj_model(clip_embed_zeroed)
|
|
return image_prompt_embeds, uncond_image_prompt_embeds
|
|
|
|
@torch.inference_mode()
|
|
def get_image_embeds_faceid_plus(self, face_embed, clip_embed, s_scale, shortcut, batch_size):
|
|
torch_device = model_management.get_torch_device()
|
|
intermediate_device = model_management.intermediate_device()
|
|
|
|
if batch_size == 0:
|
|
batch_size = clip_embed.shape[0]
|
|
intermediate_device = torch_device
|
|
elif batch_size > clip_embed.shape[0]:
|
|
batch_size = clip_embed.shape[0]
|
|
|
|
face_embed_batch = torch.split(face_embed, batch_size, dim=0)
|
|
clip_embed_batch = torch.split(clip_embed, batch_size, dim=0)
|
|
|
|
embeds = []
|
|
for face_embed, clip_embed in zip(face_embed_batch, clip_embed_batch):
|
|
embeds.append(self.image_proj_model(face_embed.to(torch_device), clip_embed.to(torch_device), scale=s_scale, shortcut=shortcut).to(intermediate_device))
|
|
|
|
embeds = torch.cat(embeds, dim=0)
|
|
del face_embed_batch, clip_embed_batch
|
|
torch.cuda.empty_cache()
|
|
#embeds = self.image_proj_model(face_embed, clip_embed, scale=s_scale, shortcut=shortcut)
|
|
return embeds
|
|
|
|
class To_KV(nn.Module):
|
|
def __init__(self, state_dict, encoder_hid_proj=None, weight_kolors=1.0):
|
|
super().__init__()
|
|
|
|
if encoder_hid_proj is not None:
|
|
hid_proj = nn.Linear(encoder_hid_proj["weight"].shape[1], encoder_hid_proj["weight"].shape[0], bias=True)
|
|
hid_proj.weight.data = encoder_hid_proj["weight"] * weight_kolors
|
|
hid_proj.bias.data = encoder_hid_proj["bias"] * weight_kolors
|
|
|
|
self.to_kvs = nn.ModuleDict()
|
|
for key, value in state_dict.items():
|
|
if encoder_hid_proj is not None:
|
|
linear_proj = nn.Linear(value.shape[1], value.shape[0], bias=False)
|
|
linear_proj.weight.data = value
|
|
self.to_kvs[key.replace(".weight", "").replace(".", "_")] = nn.Sequential(hid_proj, linear_proj)
|
|
else:
|
|
self.to_kvs[key.replace(".weight", "").replace(".", "_")] = nn.Linear(value.shape[1], value.shape[0], bias=False)
|
|
self.to_kvs[key.replace(".weight", "").replace(".", "_")].weight.data = value
|
|
|
|
def set_model_patch_replace(model, patch_kwargs, key):
|
|
to = model.model_options["transformer_options"].copy()
|
|
if "patches_replace" not in to:
|
|
to["patches_replace"] = {}
|
|
else:
|
|
to["patches_replace"] = to["patches_replace"].copy()
|
|
|
|
if "attn2" not in to["patches_replace"]:
|
|
to["patches_replace"]["attn2"] = {}
|
|
else:
|
|
to["patches_replace"]["attn2"] = to["patches_replace"]["attn2"].copy()
|
|
|
|
if key not in to["patches_replace"]["attn2"]:
|
|
to["patches_replace"]["attn2"][key] = Attn2Replace(ipadapter_attention, **patch_kwargs)
|
|
model.model_options["transformer_options"] = to
|
|
else:
|
|
to["patches_replace"]["attn2"][key].add(ipadapter_attention, **patch_kwargs)
|
|
|
|
def ipadapter_execute(model,
|
|
ipadapter,
|
|
clipvision,
|
|
insightface=None,
|
|
image=None,
|
|
image_composition=None,
|
|
image_negative=None,
|
|
weight=1.0,
|
|
weight_composition=1.0,
|
|
weight_faceidv2=None,
|
|
weight_kolors=1.0,
|
|
weight_type="linear",
|
|
combine_embeds="concat",
|
|
start_at=0.0,
|
|
end_at=1.0,
|
|
attn_mask=None,
|
|
pos_embed=None,
|
|
neg_embed=None,
|
|
unfold_batch=False,
|
|
embeds_scaling='V only',
|
|
layer_weights=None,
|
|
encode_batch_size=0,
|
|
style_boost=None,
|
|
composition_boost=None,
|
|
enhance_tiles=1,
|
|
enhance_ratio=1.0,):
|
|
device = model_management.get_torch_device()
|
|
dtype = model_management.unet_dtype()
|
|
if dtype not in [torch.float32, torch.float16, torch.bfloat16]:
|
|
dtype = torch.float16 if model_management.should_use_fp16() else torch.float32
|
|
|
|
is_full = "proj.3.weight" in ipadapter["image_proj"]
|
|
is_portrait_unnorm = "portraitunnorm" in ipadapter
|
|
is_plus = (is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]) and not is_portrait_unnorm
|
|
output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1]
|
|
is_sdxl = output_cross_attention_dim == 2048
|
|
is_kwai_kolors_faceid = "perceiver_resampler.layers.0.0.to_out.weight" in ipadapter["image_proj"] and ipadapter["image_proj"]["perceiver_resampler.layers.0.0.to_out.weight"].shape[0] == 4096
|
|
is_faceidv2 = "faceidplusv2" in ipadapter or is_kwai_kolors_faceid
|
|
is_kwai_kolors = (is_sdxl and "layers.0.0.to_out.weight" in ipadapter["image_proj"] and ipadapter["image_proj"]["layers.0.0.to_out.weight"].shape[0] == 2048) or is_kwai_kolors_faceid
|
|
is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] and not is_kwai_kolors_faceid
|
|
is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] or is_portrait_unnorm or is_kwai_kolors_faceid
|
|
|
|
if is_faceid and not insightface:
|
|
raise Exception("insightface model is required for FaceID models")
|
|
|
|
if is_faceidv2:
|
|
weight_faceidv2 = weight_faceidv2 if weight_faceidv2 is not None else weight*2
|
|
|
|
if is_kwai_kolors_faceid:
|
|
cross_attention_dim = 4096
|
|
elif is_kwai_kolors:
|
|
cross_attention_dim = 2048
|
|
elif (is_plus and is_sdxl and not is_faceid) or is_portrait_unnorm:
|
|
cross_attention_dim = 1280
|
|
else:
|
|
cross_attention_dim = output_cross_attention_dim
|
|
|
|
if is_kwai_kolors_faceid:
|
|
clip_extra_context_tokens = 6
|
|
elif (is_plus and not is_faceid) or is_portrait or is_portrait_unnorm:
|
|
clip_extra_context_tokens = 16
|
|
else:
|
|
clip_extra_context_tokens = 4
|
|
|
|
if image is not None and image.shape[1] != image.shape[2]:
|
|
print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m")
|
|
|
|
if isinstance(weight, list):
|
|
weight = torch.tensor(weight).unsqueeze(-1).unsqueeze(-1).to(device, dtype=dtype) if unfold_batch else weight[0]
|
|
|
|
if style_boost is not None:
|
|
weight_type = "style transfer precise"
|
|
elif composition_boost is not None:
|
|
weight_type = "composition precise"
|
|
|
|
# special weight types
|
|
if layer_weights is not None and layer_weights != '':
|
|
weight = { int(k): float(v)*weight for k, v in [x.split(":") for x in layer_weights.split(",")] }
|
|
weight_type = weight_type if weight_type == "style transfer precise" or weight_type == "composition precise" else "linear"
|
|
elif weight_type == "style transfer":
|
|
weight = { 6:weight } if is_sdxl else { 0:weight, 1:weight, 2:weight, 3:weight, 9:weight, 10:weight, 11:weight, 12:weight, 13:weight, 14:weight, 15:weight }
|
|
elif weight_type == "composition":
|
|
weight = { 3:weight } if is_sdxl else { 4:weight*0.25, 5:weight }
|
|
elif weight_type == "strong style transfer":
|
|
if is_sdxl:
|
|
weight = { 0:weight, 1:weight, 2:weight, 4:weight, 5:weight, 6:weight, 7:weight, 8:weight, 9:weight, 10:weight }
|
|
else:
|
|
weight = { 0:weight, 1:weight, 2:weight, 3:weight, 6:weight, 7:weight, 8:weight, 9:weight, 10:weight, 11:weight, 12:weight, 13:weight, 14:weight, 15:weight }
|
|
elif weight_type == "style and composition":
|
|
if is_sdxl:
|
|
weight = { 3:weight_composition, 6:weight }
|
|
else:
|
|
weight = { 0:weight, 1:weight, 2:weight, 3:weight, 4:weight_composition*0.25, 5:weight_composition, 9:weight, 10:weight, 11:weight, 12:weight, 13:weight, 14:weight, 15:weight }
|
|
elif weight_type == "strong style and composition":
|
|
if is_sdxl:
|
|
weight = { 0:weight, 1:weight, 2:weight, 3:weight_composition, 4:weight, 5:weight, 6:weight, 7:weight, 8:weight, 9:weight, 10:weight }
|
|
else:
|
|
weight = { 0:weight, 1:weight, 2:weight, 3:weight, 4:weight_composition, 5:weight_composition, 6:weight, 7:weight, 8:weight, 9:weight, 10:weight, 11:weight, 12:weight, 13:weight, 14:weight, 15:weight }
|
|
elif weight_type == "style transfer precise":
|
|
weight_composition = style_boost if style_boost is not None else weight
|
|
if is_sdxl:
|
|
weight = { 3:weight_composition, 6:weight }
|
|
else:
|
|
weight = { 0:weight, 1:weight, 2:weight, 3:weight, 4:weight_composition*0.25, 5:weight_composition, 9:weight, 10:weight, 11:weight, 12:weight, 13:weight, 14:weight, 15:weight }
|
|
elif weight_type == "composition precise":
|
|
weight_composition = weight
|
|
weight = composition_boost if composition_boost is not None else weight
|
|
if is_sdxl:
|
|
weight = { 0:weight*.1, 1:weight*.1, 2:weight*.1, 3:weight_composition, 4:weight*.1, 5:weight*.1, 6:weight, 7:weight*.1, 8:weight*.1, 9:weight*.1, 10:weight*.1 }
|
|
else:
|
|
weight = { 0:weight, 1:weight, 2:weight, 3:weight, 4:weight_composition*0.25, 5:weight_composition, 6:weight*.1, 7:weight*.1, 8:weight*.1, 9:weight, 10:weight, 11:weight, 12:weight, 13:weight, 14:weight, 15:weight }
|
|
|
|
clipvision_size = 224 if not is_kwai_kolors else 336
|
|
|
|
img_comp_cond_embeds = None
|
|
face_cond_embeds = None
|
|
if is_faceid:
|
|
if insightface is None:
|
|
raise Exception("Insightface model is required for FaceID models")
|
|
|
|
from insightface.utils import face_align
|
|
|
|
insightface.det_model.input_size = (640,640) # reset the detection size
|
|
image_iface = tensor_to_image(image)
|
|
face_cond_embeds = []
|
|
image = []
|
|
|
|
for i in range(image_iface.shape[0]):
|
|
for size in [(size, size) for size in range(640, 256, -64)]:
|
|
insightface.det_model.input_size = size # TODO: hacky but seems to be working
|
|
face = insightface.get(image_iface[i])
|
|
if face:
|
|
if not is_portrait_unnorm:
|
|
face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0))
|
|
else:
|
|
face_cond_embeds.append(torch.from_numpy(face[0].embedding).unsqueeze(0))
|
|
image.append(image_to_tensor(face_align.norm_crop(image_iface[i], landmark=face[0].kps, image_size=336 if is_kwai_kolors_faceid else 256 if is_sdxl else 224)))
|
|
|
|
if 640 not in size:
|
|
print(f"\033[33mINFO: InsightFace detection resolution lowered to {size}.\033[0m")
|
|
break
|
|
else:
|
|
raise Exception('InsightFace: No face detected.')
|
|
face_cond_embeds = torch.stack(face_cond_embeds).to(device, dtype=dtype)
|
|
image = torch.stack(image)
|
|
del image_iface, face
|
|
|
|
if image is not None:
|
|
img_cond_embeds = encode_image_masked(clipvision, image, batch_size=encode_batch_size, tiles=enhance_tiles, ratio=enhance_ratio, clipvision_size=clipvision_size)
|
|
if image_composition is not None:
|
|
img_comp_cond_embeds = encode_image_masked(clipvision, image_composition, batch_size=encode_batch_size, tiles=enhance_tiles, ratio=enhance_ratio, clipvision_size=clipvision_size)
|
|
|
|
if is_plus:
|
|
img_cond_embeds = img_cond_embeds.penultimate_hidden_states
|
|
image_negative = image_negative if image_negative is not None else torch.zeros([1, clipvision_size, clipvision_size, 3])
|
|
img_uncond_embeds = encode_image_masked(clipvision, image_negative, batch_size=encode_batch_size, clipvision_size=clipvision_size).penultimate_hidden_states
|
|
if image_composition is not None:
|
|
img_comp_cond_embeds = img_comp_cond_embeds.penultimate_hidden_states
|
|
else:
|
|
img_cond_embeds = img_cond_embeds.image_embeds if not is_faceid else face_cond_embeds
|
|
if image_negative is not None and not is_faceid:
|
|
img_uncond_embeds = encode_image_masked(clipvision, image_negative, batch_size=encode_batch_size, clipvision_size=clipvision_size).image_embeds
|
|
else:
|
|
img_uncond_embeds = torch.zeros_like(img_cond_embeds)
|
|
if image_composition is not None:
|
|
img_comp_cond_embeds = img_comp_cond_embeds.image_embeds
|
|
del image_negative, image_composition
|
|
|
|
image = None if not is_faceid else image # if it's face_id we need the cropped face for later
|
|
elif pos_embed is not None:
|
|
img_cond_embeds = pos_embed
|
|
|
|
if neg_embed is not None:
|
|
img_uncond_embeds = neg_embed
|
|
else:
|
|
if is_plus:
|
|
img_uncond_embeds = encode_image_masked(clipvision, torch.zeros([1, clipvision_size, clipvision_size, 3]), clipvision_size=clipvision_size).penultimate_hidden_states
|
|
else:
|
|
img_uncond_embeds = torch.zeros_like(img_cond_embeds)
|
|
del pos_embed, neg_embed
|
|
else:
|
|
raise Exception("Images or Embeds are required")
|
|
|
|
# ensure that cond and uncond have the same batch size
|
|
img_uncond_embeds = tensor_to_size(img_uncond_embeds, img_cond_embeds.shape[0])
|
|
|
|
img_cond_embeds = img_cond_embeds.to(device, dtype=dtype)
|
|
img_uncond_embeds = img_uncond_embeds.to(device, dtype=dtype)
|
|
if img_comp_cond_embeds is not None:
|
|
img_comp_cond_embeds = img_comp_cond_embeds.to(device, dtype=dtype)
|
|
|
|
# combine the embeddings if needed
|
|
if combine_embeds != "concat" and img_cond_embeds.shape[0] > 1 and not unfold_batch:
|
|
if combine_embeds == "add":
|
|
img_cond_embeds = torch.sum(img_cond_embeds, dim=0).unsqueeze(0)
|
|
if face_cond_embeds is not None:
|
|
face_cond_embeds = torch.sum(face_cond_embeds, dim=0).unsqueeze(0)
|
|
if img_comp_cond_embeds is not None:
|
|
img_comp_cond_embeds = torch.sum(img_comp_cond_embeds, dim=0).unsqueeze(0)
|
|
elif combine_embeds == "subtract":
|
|
img_cond_embeds = img_cond_embeds[0] - torch.mean(img_cond_embeds[1:], dim=0)
|
|
img_cond_embeds = img_cond_embeds.unsqueeze(0)
|
|
if face_cond_embeds is not None:
|
|
face_cond_embeds = face_cond_embeds[0] - torch.mean(face_cond_embeds[1:], dim=0)
|
|
face_cond_embeds = face_cond_embeds.unsqueeze(0)
|
|
if img_comp_cond_embeds is not None:
|
|
img_comp_cond_embeds = img_comp_cond_embeds[0] - torch.mean(img_comp_cond_embeds[1:], dim=0)
|
|
img_comp_cond_embeds = img_comp_cond_embeds.unsqueeze(0)
|
|
elif combine_embeds == "average":
|
|
img_cond_embeds = torch.mean(img_cond_embeds, dim=0).unsqueeze(0)
|
|
if face_cond_embeds is not None:
|
|
face_cond_embeds = torch.mean(face_cond_embeds, dim=0).unsqueeze(0)
|
|
if img_comp_cond_embeds is not None:
|
|
img_comp_cond_embeds = torch.mean(img_comp_cond_embeds, dim=0).unsqueeze(0)
|
|
elif combine_embeds == "norm average":
|
|
img_cond_embeds = torch.mean(img_cond_embeds / torch.norm(img_cond_embeds, dim=0, keepdim=True), dim=0).unsqueeze(0)
|
|
if face_cond_embeds is not None:
|
|
face_cond_embeds = torch.mean(face_cond_embeds / torch.norm(face_cond_embeds, dim=0, keepdim=True), dim=0).unsqueeze(0)
|
|
if img_comp_cond_embeds is not None:
|
|
img_comp_cond_embeds = torch.mean(img_comp_cond_embeds / torch.norm(img_comp_cond_embeds, dim=0, keepdim=True), dim=0).unsqueeze(0)
|
|
img_uncond_embeds = img_uncond_embeds[0].unsqueeze(0) # TODO: better strategy for uncond could be to average them
|
|
|
|
if attn_mask is not None:
|
|
attn_mask = attn_mask.to(device, dtype=dtype)
|
|
|
|
encoder_hid_proj = None
|
|
|
|
if is_kwai_kolors_faceid and hasattr(model.model, "diffusion_model") and hasattr(model.model.diffusion_model, "encoder_hid_proj"):
|
|
encoder_hid_proj = model.model.diffusion_model.encoder_hid_proj.state_dict()
|
|
|
|
ipa = IPAdapter(
|
|
ipadapter,
|
|
cross_attention_dim=cross_attention_dim,
|
|
output_cross_attention_dim=output_cross_attention_dim,
|
|
clip_embeddings_dim=img_cond_embeds.shape[-1],
|
|
clip_extra_context_tokens=clip_extra_context_tokens,
|
|
is_sdxl=is_sdxl,
|
|
is_plus=is_plus,
|
|
is_full=is_full,
|
|
is_faceid=is_faceid,
|
|
is_portrait_unnorm=is_portrait_unnorm,
|
|
is_kwai_kolors=is_kwai_kolors,
|
|
encoder_hid_proj=encoder_hid_proj,
|
|
weight_kolors=weight_kolors
|
|
).to(device, dtype=dtype)
|
|
|
|
if is_faceid and is_plus:
|
|
cond = ipa.get_image_embeds_faceid_plus(face_cond_embeds, img_cond_embeds, weight_faceidv2, is_faceidv2, encode_batch_size)
|
|
# TODO: check if noise helps with the uncond face embeds
|
|
uncond = ipa.get_image_embeds_faceid_plus(torch.zeros_like(face_cond_embeds), img_uncond_embeds, weight_faceidv2, is_faceidv2, encode_batch_size)
|
|
else:
|
|
cond, uncond = ipa.get_image_embeds(img_cond_embeds, img_uncond_embeds, encode_batch_size)
|
|
if img_comp_cond_embeds is not None:
|
|
cond_comp = ipa.get_image_embeds(img_comp_cond_embeds, img_uncond_embeds, encode_batch_size)[0]
|
|
|
|
cond = cond.to(device, dtype=dtype)
|
|
uncond = uncond.to(device, dtype=dtype)
|
|
|
|
cond_alt = None
|
|
if img_comp_cond_embeds is not None:
|
|
cond_alt = { 3: cond_comp.to(device, dtype=dtype) }
|
|
|
|
del img_cond_embeds, img_uncond_embeds, img_comp_cond_embeds, face_cond_embeds
|
|
|
|
sigma_start = model.get_model_object("model_sampling").percent_to_sigma(start_at)
|
|
sigma_end = model.get_model_object("model_sampling").percent_to_sigma(end_at)
|
|
|
|
patch_kwargs = {
|
|
"ipadapter": ipa,
|
|
"weight": weight,
|
|
"cond": cond,
|
|
"cond_alt": cond_alt,
|
|
"uncond": uncond,
|
|
"weight_type": weight_type,
|
|
"mask": attn_mask,
|
|
"sigma_start": sigma_start,
|
|
"sigma_end": sigma_end,
|
|
"unfold_batch": unfold_batch,
|
|
"embeds_scaling": embeds_scaling,
|
|
}
|
|
|
|
number = 0
|
|
if not is_sdxl:
|
|
for id in [1,2,4,5,7,8]: # id of input_blocks that have cross attention
|
|
patch_kwargs["module_key"] = str(number*2+1)
|
|
set_model_patch_replace(model, patch_kwargs, ("input", id))
|
|
number += 1
|
|
for id in [3,4,5,6,7,8,9,10,11]: # id of output_blocks that have cross attention
|
|
patch_kwargs["module_key"] = str(number*2+1)
|
|
set_model_patch_replace(model, patch_kwargs, ("output", id))
|
|
number += 1
|
|
patch_kwargs["module_key"] = str(number*2+1)
|
|
set_model_patch_replace(model, patch_kwargs, ("middle", 1))
|
|
else:
|
|
for id in [4,5,7,8]: # id of input_blocks that have cross attention
|
|
block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth
|
|
for index in block_indices:
|
|
patch_kwargs["module_key"] = str(number*2+1)
|
|
set_model_patch_replace(model, patch_kwargs, ("input", id, index))
|
|
number += 1
|
|
for id in range(6): # id of output_blocks that have cross attention
|
|
block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth
|
|
for index in block_indices:
|
|
patch_kwargs["module_key"] = str(number*2+1)
|
|
set_model_patch_replace(model, patch_kwargs, ("output", id, index))
|
|
number += 1
|
|
for index in range(10):
|
|
patch_kwargs["module_key"] = str(number*2+1)
|
|
set_model_patch_replace(model, patch_kwargs, ("middle", 1, index))
|
|
number += 1
|
|
|
|
return (model, image)
|
|
|
|
"""
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
Loaders
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
"""
|
|
class IPAdapterUnifiedLoader:
|
|
def __init__(self):
|
|
self.lora = None
|
|
self.clipvision = { "file": None, "model": None }
|
|
self.ipadapter = { "file": None, "model": None }
|
|
self.insightface = { "provider": None, "model": None }
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL", ),
|
|
"preset": (['None', 'LIGHT - SD1.5 only (low strength)', 'STANDARD (medium strength)', 'VIT-G (medium strength)', 'PLUS (high strength)', 'PLUS FACE (portraits)', 'FULL FACE - SD1.5 only (portraits stronger)'], ),
|
|
},
|
|
"optional": {
|
|
"ipadapter": ("IPADAPTER", ),
|
|
}}
|
|
|
|
RETURN_TYPES = ("MODEL", "IPADAPTER", )
|
|
RETURN_NAMES = ("model", "ipadapter", )
|
|
FUNCTION = "load_models"
|
|
CATEGORY = "ipadapter"
|
|
|
|
def load_models(self, model, preset, lora_strength=0.0, provider="CPU", ipadapter=None):
|
|
pipeline = { "clipvision": { 'file': None, 'model': None }, "ipadapter": { 'file': None, 'model': None }, "insightface": { 'provider': None, 'model': None } }
|
|
|
|
if preset == "None":
|
|
return (model, pipeline,)
|
|
else:
|
|
if ipadapter is not None:
|
|
pipeline = ipadapter
|
|
|
|
# 1. Load the clipvision model
|
|
clipvision_file = get_clipvision_file(preset)
|
|
if clipvision_file is None:
|
|
raise Exception("ClipVision model not found.")
|
|
|
|
if clipvision_file != self.clipvision['file']:
|
|
if clipvision_file != pipeline['clipvision']['file']:
|
|
self.clipvision['file'] = clipvision_file
|
|
self.clipvision['model'] = load_clip_vision(clipvision_file)
|
|
print(f"\033[33mINFO: Clip Vision model loaded from {clipvision_file}\033[0m")
|
|
else:
|
|
self.clipvision = pipeline['clipvision']
|
|
|
|
# 2. Load the ipadapter model
|
|
is_sdxl = isinstance(model.model, (comfy.model_base.SDXL, comfy.model_base.SDXLRefiner, comfy.model_base.SDXL_instructpix2pix))
|
|
ipadapter_file, is_insightface, lora_pattern = get_ipadapter_file(preset, is_sdxl)
|
|
if ipadapter_file is None:
|
|
raise Exception("IPAdapter model not found.")
|
|
|
|
if ipadapter_file != self.ipadapter['file']:
|
|
if pipeline['ipadapter']['file'] != ipadapter_file:
|
|
self.ipadapter['file'] = ipadapter_file
|
|
self.ipadapter['model'] = ipadapter_model_loader(ipadapter_file)
|
|
print(f"\033[33mINFO: IPAdapter model loaded from {ipadapter_file}\033[0m")
|
|
else:
|
|
self.ipadapter = pipeline['ipadapter']
|
|
|
|
# 3. Load the lora model if needed
|
|
if lora_pattern is not None:
|
|
lora_file = get_lora_file(lora_pattern)
|
|
lora_model = None
|
|
if lora_file is None:
|
|
raise Exception("LoRA model not found.")
|
|
|
|
if self.lora is not None:
|
|
if lora_file == self.lora['file']:
|
|
lora_model = self.lora['model']
|
|
else:
|
|
self.lora = None
|
|
torch.cuda.empty_cache()
|
|
|
|
if lora_model is None:
|
|
lora_model = comfy.utils.load_torch_file(lora_file, safe_load=True)
|
|
self.lora = { 'file': lora_file, 'model': lora_model }
|
|
print(f"\033[33mINFO: LoRA model loaded from {lora_file}\033[0m")
|
|
|
|
if lora_strength > 0:
|
|
model, _ = load_lora_for_models(model, None, lora_model, lora_strength, 0)
|
|
|
|
# 4. Load the insightface model if needed
|
|
if is_insightface:
|
|
if provider != self.insightface['provider']:
|
|
if pipeline['insightface']['provider'] != provider:
|
|
self.insightface['provider'] = provider
|
|
self.insightface['model'] = insightface_loader(provider)
|
|
print(f"\033[33mINFO: InsightFace model loaded with {provider} provider\033[0m")
|
|
else:
|
|
self.insightface = pipeline['insightface']
|
|
|
|
return (model, { 'clipvision': self.clipvision, 'ipadapter': self.ipadapter, 'insightface': self.insightface }, )
|
|
|
|
class IPAdapterUnifiedLoaderFaceID(IPAdapterUnifiedLoader):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL", ),
|
|
"preset": (['None', 'FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)', 'FACEID PORTRAIT UNNORM - SDXL only (strong)'], ),
|
|
"lora_strength": ("FLOAT", { "default": 0.6, "min": 0, "max": 1, "step": 0.01 }),
|
|
"provider": (["CPU", "CUDA", "ROCM", "DirectML", "OpenVINO", "CoreML"], ),
|
|
},
|
|
"optional": {
|
|
"ipadapter": ("IPADAPTER", ),
|
|
}}
|
|
|
|
RETURN_NAMES = ("MODEL", "ipadapter", )
|
|
CATEGORY = "ipadapter/faceid"
|
|
|
|
class IPAdapterUnifiedLoaderCommunity(IPAdapterUnifiedLoader):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"model": ("MODEL", ),
|
|
"preset": (['None', 'Composition', 'Kolors'], ),
|
|
},
|
|
"optional": {
|
|
"ipadapter": ("IPADAPTER", ),
|
|
}}
|
|
|
|
CATEGORY = "ipadapter/loaders"
|
|
|
|
class IPAdapterModelLoader:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": { "ipadapter_file": (folder_paths.get_filename_list("ipadapter"), )}}
|
|
|
|
RETURN_TYPES = ("IPADAPTER",)
|
|
FUNCTION = "load_ipadapter_model"
|
|
CATEGORY = "ipadapter/loaders"
|
|
|
|
def load_ipadapter_model(self, ipadapter_file):
|
|
ipadapter_file = folder_paths.get_full_path("ipadapter", ipadapter_file)
|
|
return (ipadapter_model_loader(ipadapter_file),)
|
|
|
|
class IPAdapterInsightFaceLoader:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"provider": (["CPU", "CUDA", "ROCM"], ),
|
|
"model_name": (['buffalo_l', 'antelopev2'], )
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("INSIGHTFACE",)
|
|
FUNCTION = "load_insightface"
|
|
CATEGORY = "ipadapter/loaders"
|
|
|
|
def load_insightface(self, provider, model_name):
|
|
return (insightface_loader(provider, model_name=model_name),)
|
|
|
|
"""
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
Main Apply Nodes
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
"""
|
|
class IPAdapterSimple:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"weight_type": (['standard', 'prompt is more important', 'style transfer'], ),
|
|
},
|
|
"optional": {
|
|
"attn_mask": ("MASK",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "apply_ipadapter"
|
|
CATEGORY = "ipadapter"
|
|
|
|
def apply_ipadapter(self, model, ipadapter, image, weight, start_at, end_at, weight_type, attn_mask=None):
|
|
if weight_type.startswith("style"):
|
|
weight_type = "style transfer"
|
|
elif weight_type == "prompt is more important":
|
|
weight_type = "ease out"
|
|
else:
|
|
weight_type = "linear"
|
|
|
|
ipa_args = {
|
|
"image": image,
|
|
"weight": weight,
|
|
"start_at": start_at,
|
|
"end_at": end_at,
|
|
"attn_mask": attn_mask,
|
|
"weight_type": weight_type,
|
|
"insightface": ipadapter['insightface']['model'] if 'insightface' in ipadapter else None,
|
|
}
|
|
|
|
if 'ipadapter' not in ipadapter:
|
|
raise Exception("IPAdapter model not present in the pipeline. Please load the models with the IPAdapterUnifiedLoader node.")
|
|
if 'clipvision' not in ipadapter:
|
|
raise Exception("CLIPVision model not present in the pipeline. Please load the models with the IPAdapterUnifiedLoader node.")
|
|
if ipadapter['ipadapter']['file'] == None or ipadapter['clipvision']['file'] == None:
|
|
return (model,)
|
|
else:
|
|
return ipadapter_execute(model.clone(), ipadapter['ipadapter']['model'], ipadapter['clipvision']['model'], **ipa_args)
|
|
|
|
class IPAdapterAdvanced:
|
|
def __init__(self):
|
|
self.unfold_batch = False
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "apply_ipadapter"
|
|
CATEGORY = "ipadapter"
|
|
|
|
def apply_ipadapter(self, model, ipadapter, start_at=0.0, end_at=1.0, weight=1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None, ipadapter_params=None, encode_batch_size=0, style_boost=None, composition_boost=None, enhance_tiles=1, enhance_ratio=1.0, weight_kolors=1.0):
|
|
is_sdxl = isinstance(model.model, (comfy.model_base.SDXL, comfy.model_base.SDXLRefiner, comfy.model_base.SDXL_instructpix2pix))
|
|
|
|
if 'ipadapter' in ipadapter and ipadapter['ipadapter']['file'] == None:
|
|
return (model,)
|
|
else:
|
|
if 'ipadapter' in ipadapter:
|
|
ipadapter_model = ipadapter['ipadapter']['model']
|
|
clip_vision = clip_vision if clip_vision is not None else ipadapter['clipvision']['model']
|
|
else:
|
|
ipadapter_model = ipadapter
|
|
|
|
if clip_vision is None:
|
|
raise Exception("Missing CLIPVision model.")
|
|
|
|
if image_style is not None: # we are doing style + composition transfer
|
|
if not is_sdxl:
|
|
raise Exception("Style + Composition transfer is only available for SDXL models at the moment.") # TODO: check feasibility for SD1.5 models
|
|
|
|
image = image_style
|
|
weight = weight_style
|
|
if image_composition is None:
|
|
image_composition = image_style
|
|
|
|
weight_type = "strong style and composition" if expand_style else "style and composition"
|
|
if ipadapter_params is not None: # we are doing batch processing
|
|
image = ipadapter_params['image']
|
|
attn_mask = ipadapter_params['attn_mask']
|
|
weight = ipadapter_params['weight']
|
|
weight_type = ipadapter_params['weight_type']
|
|
start_at = ipadapter_params['start_at']
|
|
end_at = ipadapter_params['end_at']
|
|
else:
|
|
# at this point weight can be a list from the batch-weight or a single float
|
|
weight = [weight]
|
|
|
|
image = image if isinstance(image, list) else [image]
|
|
|
|
work_model = model.clone()
|
|
|
|
for i in range(len(image)):
|
|
if image[i] is None:
|
|
continue
|
|
|
|
ipa_args = {
|
|
"image": image[i],
|
|
"image_composition": image_composition,
|
|
"image_negative": image_negative,
|
|
"weight": weight[i],
|
|
"weight_composition": weight_composition,
|
|
"weight_faceidv2": weight_faceidv2,
|
|
"weight_type": weight_type if not isinstance(weight_type, list) else weight_type[i],
|
|
"combine_embeds": combine_embeds,
|
|
"start_at": start_at if not isinstance(start_at, list) else start_at[i],
|
|
"end_at": end_at if not isinstance(end_at, list) else end_at[i],
|
|
"attn_mask": attn_mask if not isinstance(attn_mask, list) else attn_mask[i],
|
|
"unfold_batch": self.unfold_batch,
|
|
"embeds_scaling": embeds_scaling,
|
|
"insightface": insightface if insightface is not None else ipadapter['insightface']['model'] if 'insightface' in ipadapter else None,
|
|
"layer_weights": layer_weights,
|
|
"encode_batch_size": encode_batch_size,
|
|
"style_boost": style_boost,
|
|
"composition_boost": composition_boost,
|
|
"enhance_tiles": enhance_tiles,
|
|
"enhance_ratio": enhance_ratio,
|
|
"weight_kolors": weight_kolors,
|
|
}
|
|
|
|
work_model, face_image = ipadapter_execute(work_model, ipadapter_model, clip_vision, **ipa_args)
|
|
|
|
del ipadapter
|
|
return (work_model, face_image, )
|
|
|
|
class IPAdapterBatch(IPAdapterAdvanced):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
"encode_batch_size": ("INT", { "default": 0, "min": 0, "max": 4096 }),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
class IPAdapterStyleComposition(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image_style": ("IMAGE",),
|
|
"image_composition": ("IMAGE",),
|
|
"weight_style": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"weight_composition": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"expand_style": ("BOOLEAN", { "default": False }),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"], {"default": "average"}),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
CATEGORY = "ipadapter/style_composition"
|
|
|
|
class IPAdapterStyleCompositionBatch(IPAdapterStyleComposition):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image_style": ("IMAGE",),
|
|
"image_composition": ("IMAGE",),
|
|
"weight_style": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"weight_composition": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"expand_style": ("BOOLEAN", { "default": False }),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
class IPAdapterFaceID(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
|
|
"weight_faceidv2": ("FLOAT", { "default": 1.0, "min": -1, "max": 5.0, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
"insightface": ("INSIGHTFACE",),
|
|
}
|
|
}
|
|
|
|
CATEGORY = "ipadapter/faceid"
|
|
RETURN_TYPES = ("MODEL","IMAGE",)
|
|
RETURN_NAMES = ("MODEL", "face_image", )
|
|
|
|
class IPAAdapterFaceIDBatch(IPAdapterFaceID):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
class IPAdapterFaceIDKolors(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
|
|
"weight_faceidv2": ("FLOAT", { "default": 1.0, "min": -1, "max": 5.0, "step": 0.05 }),
|
|
"weight_kolors": ("FLOAT", { "default": 1.0, "min": -1, "max": 5.0, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
"insightface": ("INSIGHTFACE",),
|
|
}
|
|
}
|
|
|
|
CATEGORY = "ipadapter/faceid"
|
|
RETURN_TYPES = ("MODEL","IMAGE",)
|
|
RETURN_NAMES = ("MODEL", "face_image", )
|
|
|
|
class IPAdapterTiled:
|
|
def __init__(self):
|
|
self.unfold_batch = False
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"sharpening": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("MODEL", "IMAGE", "MASK", )
|
|
RETURN_NAMES = ("MODEL", "tiles", "masks", )
|
|
FUNCTION = "apply_tiled"
|
|
CATEGORY = "ipadapter/tiled"
|
|
|
|
def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only', encode_batch_size=0):
|
|
|
|
if 'ipadapter' in ipadapter and ipadapter['ipadapter']['file'] == None:
|
|
return (model,)
|
|
else:
|
|
# 1. Select the models
|
|
if 'ipadapter' in ipadapter:
|
|
ipadapter_model = ipadapter['ipadapter']['model']
|
|
clip_vision = clip_vision if clip_vision is not None else ipadapter['clipvision']['model']
|
|
else:
|
|
ipadapter_model = ipadapter
|
|
clip_vision = clip_vision
|
|
|
|
if clip_vision is None:
|
|
raise Exception("Missing CLIPVision model.")
|
|
|
|
del ipadapter
|
|
|
|
# 2. Extract the tiles
|
|
tile_size = 256 # I'm using 256 instead of 224 as it is more likely divisible by the latent size, it will be downscaled to 224 by the clip vision encoder
|
|
_, oh, ow, _ = image.shape
|
|
if attn_mask is None:
|
|
attn_mask = torch.ones([1, oh, ow], dtype=image.dtype, device=image.device)
|
|
|
|
image = image.permute([0,3,1,2])
|
|
attn_mask = attn_mask.unsqueeze(1)
|
|
# the mask should have the same proportions as the reference image and the latent
|
|
attn_mask = T.Resize((oh, ow), interpolation=T.InterpolationMode.BICUBIC, antialias=True)(attn_mask)
|
|
|
|
# if the image is almost a square, we crop it to a square
|
|
if oh / ow > 0.75 and oh / ow < 1.33:
|
|
# crop the image to a square
|
|
image = T.CenterCrop(min(oh, ow))(image)
|
|
resize = (tile_size*2, tile_size*2)
|
|
|
|
attn_mask = T.CenterCrop(min(oh, ow))(attn_mask)
|
|
# otherwise resize the smallest side and the other proportionally
|
|
else:
|
|
resize = (int(tile_size * ow / oh), tile_size) if oh < ow else (tile_size, int(tile_size * oh / ow))
|
|
|
|
# using PIL for better results
|
|
imgs = []
|
|
for img in image:
|
|
img = T.ToPILImage()(img)
|
|
img = img.resize(resize, resample=Image.Resampling['LANCZOS'])
|
|
imgs.append(T.ToTensor()(img))
|
|
image = torch.stack(imgs)
|
|
del imgs, img
|
|
|
|
# we don't need a high quality resize for the mask
|
|
attn_mask = T.Resize(resize[::-1], interpolation=T.InterpolationMode.BICUBIC, antialias=True)(attn_mask)
|
|
|
|
# we allow a maximum of 4 tiles
|
|
if oh / ow > 4 or oh / ow < 0.25:
|
|
crop = (tile_size, tile_size*4) if oh < ow else (tile_size*4, tile_size)
|
|
image = T.CenterCrop(crop)(image)
|
|
attn_mask = T.CenterCrop(crop)(attn_mask)
|
|
|
|
attn_mask = attn_mask.squeeze(1)
|
|
|
|
if sharpening > 0:
|
|
image = contrast_adaptive_sharpening(image, sharpening)
|
|
|
|
image = image.permute([0,2,3,1])
|
|
|
|
_, oh, ow, _ = image.shape
|
|
|
|
# find the number of tiles for each side
|
|
tiles_x = math.ceil(ow / tile_size)
|
|
tiles_y = math.ceil(oh / tile_size)
|
|
overlap_x = max(0, (tiles_x * tile_size - ow) / (tiles_x - 1 if tiles_x > 1 else 1))
|
|
overlap_y = max(0, (tiles_y * tile_size - oh) / (tiles_y - 1 if tiles_y > 1 else 1))
|
|
|
|
base_mask = torch.zeros([attn_mask.shape[0], oh, ow], dtype=image.dtype, device=image.device)
|
|
|
|
# extract all the tiles from the image and create the masks
|
|
tiles = []
|
|
masks = []
|
|
for y in range(tiles_y):
|
|
for x in range(tiles_x):
|
|
start_x = int(x * (tile_size - overlap_x))
|
|
start_y = int(y * (tile_size - overlap_y))
|
|
tiles.append(image[:, start_y:start_y+tile_size, start_x:start_x+tile_size, :])
|
|
mask = base_mask.clone()
|
|
mask[:, start_y:start_y+tile_size, start_x:start_x+tile_size] = attn_mask[:, start_y:start_y+tile_size, start_x:start_x+tile_size]
|
|
masks.append(mask)
|
|
del mask
|
|
|
|
# 3. Apply the ipadapter to each group of tiles
|
|
model = model.clone()
|
|
for i in range(len(tiles)):
|
|
ipa_args = {
|
|
"image": tiles[i],
|
|
"image_negative": image_negative,
|
|
"weight": weight,
|
|
"weight_type": weight_type,
|
|
"combine_embeds": combine_embeds,
|
|
"start_at": start_at,
|
|
"end_at": end_at,
|
|
"attn_mask": masks[i],
|
|
"unfold_batch": self.unfold_batch,
|
|
"embeds_scaling": embeds_scaling,
|
|
"encode_batch_size": encode_batch_size,
|
|
}
|
|
# apply the ipadapter to the model without cloning it
|
|
model, _ = ipadapter_execute(model, ipadapter_model, clip_vision, **ipa_args)
|
|
|
|
return (model, torch.cat(tiles), torch.cat(masks), )
|
|
|
|
class IPAdapterTiledBatch(IPAdapterTiled):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"sharpening": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
"encode_batch_size": ("INT", { "default": 0, "min": 0, "max": 4096 }),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
class IPAdapterEmbeds:
|
|
def __init__(self):
|
|
self.unfold_batch = False
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"pos_embed": ("EMBEDS",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"neg_embed": ("EMBEDS",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "apply_ipadapter"
|
|
CATEGORY = "ipadapter/embeds"
|
|
|
|
def apply_ipadapter(self, model, ipadapter, pos_embed, weight, weight_type, start_at, end_at, neg_embed=None, attn_mask=None, clip_vision=None, embeds_scaling='V only'):
|
|
ipa_args = {
|
|
"pos_embed": pos_embed,
|
|
"neg_embed": neg_embed,
|
|
"weight": weight,
|
|
"weight_type": weight_type,
|
|
"start_at": start_at,
|
|
"end_at": end_at,
|
|
"attn_mask": attn_mask,
|
|
"embeds_scaling": embeds_scaling,
|
|
"unfold_batch": self.unfold_batch,
|
|
}
|
|
|
|
if 'ipadapter' in ipadapter and ipadapter['ipadapter']['file'] == None:
|
|
return (model,)
|
|
else:
|
|
if 'ipadapter' in ipadapter:
|
|
ipadapter_model = ipadapter['ipadapter']['model']
|
|
clip_vision = clip_vision if clip_vision is not None else ipadapter['clipvision']['model']
|
|
else:
|
|
ipadapter_model = ipadapter
|
|
clip_vision = clip_vision
|
|
|
|
if clip_vision is None and neg_embed is None:
|
|
raise Exception("Missing CLIPVision model.")
|
|
|
|
del ipadapter
|
|
|
|
return ipadapter_execute(model.clone(), ipadapter_model, clip_vision, **ipa_args)
|
|
|
|
class IPAdapterEmbedsBatch(IPAdapterEmbeds):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
class IPAdapterMS(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"weight_faceidv2": ("FLOAT", { "default": 1.0, "min": -1, "max": 5.0, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
"layer_weights": ("STRING", { "default": "", "multiline": True }),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
"insightface": ("INSIGHTFACE",),
|
|
}
|
|
}
|
|
|
|
CATEGORY = "ipadapter/dev"
|
|
|
|
class IPAdapterClipVisionEnhancer(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
"enhance_tiles": ("INT", { "default": 2, "min": 1, "max": 16 }),
|
|
"enhance_ratio": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05 }),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
CATEGORY = "ipadapter/dev"
|
|
|
|
class IPAdapterClipVisionEnhancerBatch(IPAdapterClipVisionEnhancer):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
"enhance_tiles": ("INT", { "default": 2, "min": 1, "max": 16 }),
|
|
"enhance_ratio": ("FLOAT", { "default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05 }),
|
|
"encode_batch_size": ("INT", { "default": 0, "min": 0, "max": 4096 }),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
class IPAdapterFromParams(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"ipadapter_params": ("IPADAPTER_PARAMS", ),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
CATEGORY = "ipadapter/params"
|
|
|
|
class IPAdapterPreciseStyleTransfer(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"style_boost": ("FLOAT", { "default": 1.0, "min": -5, "max": 5, "step": 0.05 }),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
class IPAdapterPreciseStyleTransferBatch(IPAdapterPreciseStyleTransfer):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
class IPAdapterPreciseComposition(IPAdapterAdvanced):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"model": ("MODEL", ),
|
|
"ipadapter": ("IPADAPTER", ),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
|
|
"composition_boost": ("FLOAT", { "default": 0.0, "min": -5, "max": 5, "step": 0.05 }),
|
|
"combine_embeds": (["concat", "add", "subtract", "average", "norm average"],),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
|
|
},
|
|
"optional": {
|
|
"image_negative": ("IMAGE",),
|
|
"attn_mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
class IPAdapterPreciseCompositionBatch(IPAdapterPreciseComposition):
|
|
def __init__(self):
|
|
self.unfold_batch = True
|
|
|
|
"""
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
Helpers
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
"""
|
|
class IPAdapterEncoder:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"ipadapter": ("IPADAPTER",),
|
|
"image": ("IMAGE",),
|
|
"weight": ("FLOAT", { "default": 1.0, "min": -1.0, "max": 3.0, "step": 0.01 }),
|
|
},
|
|
"optional": {
|
|
"mask": ("MASK",),
|
|
"clip_vision": ("CLIP_VISION",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("EMBEDS", "EMBEDS",)
|
|
RETURN_NAMES = ("pos_embed", "neg_embed",)
|
|
FUNCTION = "encode"
|
|
CATEGORY = "ipadapter/embeds"
|
|
|
|
def encode(self, ipadapter, image, weight, mask=None, clip_vision=None):
|
|
|
|
if 'ipadapter' in ipadapter and ipadapter['ipadapter']['file'] == None:
|
|
return (None, None,)
|
|
else:
|
|
if 'ipadapter' in ipadapter:
|
|
ipadapter_model = ipadapter['ipadapter']['model']
|
|
clip_vision = clip_vision if clip_vision is not None else ipadapter['clipvision']['model']
|
|
else:
|
|
ipadapter_model = ipadapter
|
|
clip_vision = clip_vision
|
|
|
|
if clip_vision is None:
|
|
raise Exception("Missing CLIPVision model.")
|
|
|
|
is_plus = "proj.3.weight" in ipadapter_model["image_proj"] or "latents" in ipadapter_model["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter_model["image_proj"]
|
|
is_kwai_kolors = is_plus and "layers.0.0.to_out.weight" in ipadapter_model["image_proj"] and ipadapter_model["image_proj"]["layers.0.0.to_out.weight"].shape[0] == 2048
|
|
|
|
clipvision_size = 224 if not is_kwai_kolors else 336
|
|
|
|
# resize and crop the mask to 224x224
|
|
if mask is not None and mask.shape[1:3] != torch.Size([clipvision_size, clipvision_size]):
|
|
mask = mask.unsqueeze(1)
|
|
transforms = T.Compose([
|
|
T.CenterCrop(min(mask.shape[2], mask.shape[3])),
|
|
T.Resize((clipvision_size, clipvision_size), interpolation=T.InterpolationMode.BICUBIC, antialias=True),
|
|
])
|
|
mask = transforms(mask).squeeze(1)
|
|
#mask = T.Resize((image.shape[1], image.shape[2]), interpolation=T.InterpolationMode.BICUBIC, antialias=True)(mask.unsqueeze(1)).squeeze(1)
|
|
|
|
img_cond_embeds = encode_image_masked(clip_vision, image, mask, clipvision_size=clipvision_size)
|
|
|
|
if is_plus:
|
|
img_cond_embeds = img_cond_embeds.penultimate_hidden_states
|
|
img_uncond_embeds = encode_image_masked(clip_vision, torch.zeros([1, clipvision_size, clipvision_size, 3]), clipvision_size=clipvision_size).penultimate_hidden_states
|
|
else:
|
|
img_cond_embeds = img_cond_embeds.image_embeds
|
|
img_uncond_embeds = torch.zeros_like(img_cond_embeds)
|
|
|
|
if weight != 1:
|
|
img_cond_embeds = img_cond_embeds * weight
|
|
|
|
return (img_cond_embeds, img_uncond_embeds, )
|
|
|
|
class IPAdapterCombineEmbeds:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"embed1": ("EMBEDS",),
|
|
"method": (["concat", "add", "subtract", "average", "norm average", "max", "min"], ),
|
|
},
|
|
"optional": {
|
|
"embed2": ("EMBEDS",),
|
|
"embed3": ("EMBEDS",),
|
|
"embed4": ("EMBEDS",),
|
|
"embed5": ("EMBEDS",),
|
|
}}
|
|
|
|
RETURN_TYPES = ("EMBEDS",)
|
|
FUNCTION = "batch"
|
|
CATEGORY = "ipadapter/embeds"
|
|
|
|
def batch(self, embed1, method, embed2=None, embed3=None, embed4=None, embed5=None):
|
|
if method=='concat' and embed2 is None and embed3 is None and embed4 is None and embed5 is None:
|
|
return (embed1, )
|
|
|
|
embeds = [embed1, embed2, embed3, embed4, embed5]
|
|
embeds = [embed for embed in embeds if embed is not None]
|
|
embeds = torch.cat(embeds, dim=0)
|
|
|
|
if method == "add":
|
|
embeds = torch.sum(embeds, dim=0).unsqueeze(0)
|
|
elif method == "subtract":
|
|
embeds = embeds[0] - torch.mean(embeds[1:], dim=0)
|
|
embeds = embeds.unsqueeze(0)
|
|
elif method == "average":
|
|
embeds = torch.mean(embeds, dim=0).unsqueeze(0)
|
|
elif method == "norm average":
|
|
embeds = torch.mean(embeds / torch.norm(embeds, dim=0, keepdim=True), dim=0).unsqueeze(0)
|
|
elif method == "max":
|
|
embeds = torch.max(embeds, dim=0).values.unsqueeze(0)
|
|
elif method == "min":
|
|
embeds = torch.min(embeds, dim=0).values.unsqueeze(0)
|
|
|
|
return (embeds, )
|
|
|
|
class IPAdapterNoise:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"type": (["fade", "dissolve", "gaussian", "shuffle"], ),
|
|
"strength": ("FLOAT", { "default": 1.0, "min": 0, "max": 1, "step": 0.05 }),
|
|
"blur": ("INT", { "default": 0, "min": 0, "max": 32, "step": 1 }),
|
|
},
|
|
"optional": {
|
|
"image_optional": ("IMAGE",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
FUNCTION = "make_noise"
|
|
CATEGORY = "ipadapter/utils"
|
|
|
|
def make_noise(self, type, strength, blur, image_optional=None):
|
|
if image_optional is None:
|
|
image = torch.zeros([1, 224, 224, 3])
|
|
else:
|
|
transforms = T.Compose([
|
|
T.CenterCrop(min(image_optional.shape[1], image_optional.shape[2])),
|
|
T.Resize((224, 224), interpolation=T.InterpolationMode.BICUBIC, antialias=True),
|
|
])
|
|
image = transforms(image_optional.permute([0,3,1,2])).permute([0,2,3,1])
|
|
|
|
seed = int(torch.sum(image).item()) % 1000000007 # hash the image to get a seed, grants predictability
|
|
torch.manual_seed(seed)
|
|
|
|
if type == "fade":
|
|
noise = torch.rand_like(image)
|
|
noise = image * (1 - strength) + noise * strength
|
|
elif type == "dissolve":
|
|
mask = (torch.rand_like(image) < strength).float()
|
|
noise = torch.rand_like(image)
|
|
noise = image * (1-mask) + noise * mask
|
|
elif type == "gaussian":
|
|
noise = torch.randn_like(image) * strength
|
|
noise = image + noise
|
|
elif type == "shuffle":
|
|
transforms = T.Compose([
|
|
T.ElasticTransform(alpha=75.0, sigma=(1-strength)*3.5),
|
|
T.RandomVerticalFlip(p=1.0),
|
|
T.RandomHorizontalFlip(p=1.0),
|
|
])
|
|
image = transforms(image.permute([0,3,1,2])).permute([0,2,3,1])
|
|
noise = torch.randn_like(image) * (strength*0.75)
|
|
noise = image * (1-noise) + noise
|
|
|
|
del image
|
|
noise = torch.clamp(noise, 0, 1)
|
|
|
|
if blur > 0:
|
|
if blur % 2 == 0:
|
|
blur += 1
|
|
noise = T.functional.gaussian_blur(noise.permute([0,3,1,2]), blur).permute([0,2,3,1])
|
|
|
|
return (noise, )
|
|
|
|
class PrepImageForClipVision:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"image": ("IMAGE",),
|
|
"interpolation": (["LANCZOS", "BICUBIC", "HAMMING", "BILINEAR", "BOX", "NEAREST"],),
|
|
"crop_position": (["top", "bottom", "left", "right", "center", "pad"],),
|
|
"sharpening": ("FLOAT", {"default": 0.0, "min": 0, "max": 1, "step": 0.05}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
FUNCTION = "prep_image"
|
|
|
|
CATEGORY = "ipadapter/utils"
|
|
|
|
def prep_image(self, image, interpolation="LANCZOS", crop_position="center", sharpening=0.0):
|
|
size = (224, 224)
|
|
_, oh, ow, _ = image.shape
|
|
output = image.permute([0,3,1,2])
|
|
|
|
if crop_position == "pad":
|
|
if oh != ow:
|
|
if oh > ow:
|
|
pad = (oh - ow) // 2
|
|
pad = (pad, 0, pad, 0)
|
|
elif ow > oh:
|
|
pad = (ow - oh) // 2
|
|
pad = (0, pad, 0, pad)
|
|
output = T.functional.pad(output, pad, fill=0)
|
|
else:
|
|
crop_size = min(oh, ow)
|
|
x = (ow-crop_size) // 2
|
|
y = (oh-crop_size) // 2
|
|
if "top" in crop_position:
|
|
y = 0
|
|
elif "bottom" in crop_position:
|
|
y = oh-crop_size
|
|
elif "left" in crop_position:
|
|
x = 0
|
|
elif "right" in crop_position:
|
|
x = ow-crop_size
|
|
|
|
x2 = x+crop_size
|
|
y2 = y+crop_size
|
|
|
|
output = output[:, :, y:y2, x:x2]
|
|
|
|
imgs = []
|
|
for img in output:
|
|
img = T.ToPILImage()(img) # using PIL for better results
|
|
img = img.resize(size, resample=Image.Resampling[interpolation])
|
|
imgs.append(T.ToTensor()(img))
|
|
output = torch.stack(imgs, dim=0)
|
|
del imgs, img
|
|
|
|
if sharpening > 0:
|
|
output = contrast_adaptive_sharpening(output, sharpening)
|
|
|
|
output = output.permute([0,2,3,1])
|
|
|
|
return (output, )
|
|
|
|
class IPAdapterSaveEmbeds:
|
|
def __init__(self):
|
|
self.output_dir = folder_paths.get_output_directory()
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"embeds": ("EMBEDS",),
|
|
"filename_prefix": ("STRING", {"default": "IP_embeds"})
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ()
|
|
FUNCTION = "save"
|
|
OUTPUT_NODE = True
|
|
CATEGORY = "ipadapter/embeds"
|
|
|
|
def save(self, embeds, filename_prefix):
|
|
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir)
|
|
file = f"{filename}_{counter:05}.ipadpt"
|
|
file = os.path.join(full_output_folder, file)
|
|
|
|
torch.save(embeds, file)
|
|
return (None, )
|
|
|
|
class IPAdapterLoadEmbeds:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
input_dir = folder_paths.get_input_directory()
|
|
files = [os.path.relpath(os.path.join(root, file), input_dir) for root, dirs, files in os.walk(input_dir) for file in files if file.endswith('.ipadpt')]
|
|
return {"required": {"embeds": [sorted(files), ]}, }
|
|
|
|
RETURN_TYPES = ("EMBEDS", )
|
|
FUNCTION = "load"
|
|
CATEGORY = "ipadapter/embeds"
|
|
|
|
def load(self, embeds):
|
|
path = folder_paths.get_annotated_filepath(embeds)
|
|
return (torch.load(path).cpu(), )
|
|
|
|
class IPAdapterWeights:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"weights": ("STRING", {"default": '1.0, 0.0', "multiline": True }),
|
|
"timing": (["custom", "linear", "ease_in_out", "ease_in", "ease_out", "random"], { "default": "linear" } ),
|
|
"frames": ("INT", {"default": 0, "min": 0, "max": 9999, "step": 1 }),
|
|
"start_frame": ("INT", {"default": 0, "min": 0, "max": 9999, "step": 1 }),
|
|
"end_frame": ("INT", {"default": 9999, "min": 0, "max": 9999, "step": 1 }),
|
|
"add_starting_frames": ("INT", {"default": 0, "min": 0, "max": 9999, "step": 1 }),
|
|
"add_ending_frames": ("INT", {"default": 0, "min": 0, "max": 9999, "step": 1 }),
|
|
"method": (["full batch", "shift batches", "alternate batches"], { "default": "full batch" }),
|
|
}, "optional": {
|
|
"image": ("IMAGE",),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("FLOAT", "FLOAT", "INT", "IMAGE", "IMAGE", "WEIGHTS_STRATEGY")
|
|
RETURN_NAMES = ("weights", "weights_invert", "total_frames", "image_1", "image_2", "weights_strategy")
|
|
FUNCTION = "weights"
|
|
CATEGORY = "ipadapter/weights"
|
|
|
|
def weights(self, weights='', timing='custom', frames=0, start_frame=0, end_frame=9999, add_starting_frames=0, add_ending_frames=0, method='full batch', weights_strategy=None, image=None):
|
|
import random
|
|
|
|
frame_count = image.shape[0] if image is not None else 0
|
|
if weights_strategy is not None:
|
|
weights = weights_strategy["weights"]
|
|
timing = weights_strategy["timing"]
|
|
frames = weights_strategy["frames"]
|
|
start_frame = weights_strategy["start_frame"]
|
|
end_frame = weights_strategy["end_frame"]
|
|
add_starting_frames = weights_strategy["add_starting_frames"]
|
|
add_ending_frames = weights_strategy["add_ending_frames"]
|
|
method = weights_strategy["method"]
|
|
frame_count = weights_strategy["frame_count"]
|
|
else:
|
|
weights_strategy = {
|
|
"weights": weights,
|
|
"timing": timing,
|
|
"frames": frames,
|
|
"start_frame": start_frame,
|
|
"end_frame": end_frame,
|
|
"add_starting_frames": add_starting_frames,
|
|
"add_ending_frames": add_ending_frames,
|
|
"method": method,
|
|
"frame_count": frame_count,
|
|
}
|
|
|
|
# convert the string to a list of floats separated by commas or newlines
|
|
weights = weights.replace("\n", ",")
|
|
weights = [float(weight) for weight in weights.split(",") if weight.strip() != ""]
|
|
|
|
if timing != "custom":
|
|
frames = max(frames, 2)
|
|
start = 0.0
|
|
end = 1.0
|
|
|
|
if len(weights) > 0:
|
|
start = weights[0]
|
|
end = weights[-1]
|
|
|
|
weights = []
|
|
|
|
end_frame = min(end_frame, frames)
|
|
duration = end_frame - start_frame
|
|
if start_frame > 0:
|
|
weights.extend([start] * start_frame)
|
|
|
|
for i in range(duration):
|
|
n = duration - 1
|
|
if timing == "linear":
|
|
weights.append(start + (end - start) * i / n)
|
|
elif timing == "ease_in_out":
|
|
weights.append(start + (end - start) * (1 - math.cos(i / n * math.pi)) / 2)
|
|
elif timing == "ease_in":
|
|
weights.append(start + (end - start) * math.sin(i / n * math.pi / 2))
|
|
elif timing == "ease_out":
|
|
weights.append(start + (end - start) * (1 - math.cos(i / n * math.pi / 2)))
|
|
elif timing == "random":
|
|
weights.append(random.uniform(start, end))
|
|
|
|
weights[-1] = end if timing != "random" else weights[-1]
|
|
if end_frame < frames:
|
|
weights.extend([end] * (frames - end_frame))
|
|
|
|
if len(weights) == 0:
|
|
weights = [0.0]
|
|
|
|
frames = len(weights)
|
|
|
|
# repeat the images for cross fade
|
|
image_1 = None
|
|
image_2 = None
|
|
|
|
# Calculate the min and max of the weights
|
|
min_weight = min(weights)
|
|
max_weight = max(weights)
|
|
|
|
if image is not None:
|
|
|
|
if "shift" in method:
|
|
image_1 = image[:-1]
|
|
image_2 = image[1:]
|
|
|
|
weights = weights * image_1.shape[0]
|
|
image_1 = image_1.repeat_interleave(frames, 0)
|
|
image_2 = image_2.repeat_interleave(frames, 0)
|
|
elif "alternate" in method:
|
|
image_1 = image[::2].repeat_interleave(2, 0)
|
|
image_1 = image_1[1:]
|
|
image_2 = image[1::2].repeat_interleave(2, 0)
|
|
|
|
# Invert the weights relative to their own range
|
|
mew_weights = weights + [max_weight - (w - min_weight) for w in weights]
|
|
|
|
mew_weights = mew_weights * (image_1.shape[0] // 2)
|
|
if image.shape[0] % 2:
|
|
image_1 = image_1[:-1]
|
|
else:
|
|
image_2 = image_2[:-1]
|
|
mew_weights = mew_weights + weights
|
|
|
|
weights = mew_weights
|
|
image_1 = image_1.repeat_interleave(frames, 0)
|
|
image_2 = image_2.repeat_interleave(frames, 0)
|
|
else:
|
|
weights = weights * image.shape[0]
|
|
image_1 = image.repeat_interleave(frames, 0)
|
|
|
|
# add starting and ending frames
|
|
if add_starting_frames > 0:
|
|
weights = [weights[0]] * add_starting_frames + weights
|
|
image_1 = torch.cat([image[:1].repeat(add_starting_frames, 1, 1, 1), image_1], dim=0)
|
|
if image_2 is not None:
|
|
image_2 = torch.cat([image[:1].repeat(add_starting_frames, 1, 1, 1), image_2], dim=0)
|
|
if add_ending_frames > 0:
|
|
weights = weights + [weights[-1]] * add_ending_frames
|
|
image_1 = torch.cat([image_1, image[-1:].repeat(add_ending_frames, 1, 1, 1)], dim=0)
|
|
if image_2 is not None:
|
|
image_2 = torch.cat([image_2, image[-1:].repeat(add_ending_frames, 1, 1, 1)], dim=0)
|
|
|
|
# reverse the weights array
|
|
weights_invert = weights[::-1]
|
|
|
|
frame_count = len(weights)
|
|
|
|
return (weights, weights_invert, frame_count, image_1, image_2, weights_strategy,)
|
|
|
|
class IPAdapterWeightsFromStrategy(IPAdapterWeights):
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"weights_strategy": ("WEIGHTS_STRATEGY",),
|
|
}, "optional": {
|
|
"image": ("IMAGE",),
|
|
}
|
|
}
|
|
|
|
class IPAdapterPromptScheduleFromWeightsStrategy():
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"weights_strategy": ("WEIGHTS_STRATEGY",),
|
|
"prompt": ("STRING", {"default": "", "multiline": True }),
|
|
}}
|
|
|
|
RETURN_TYPES = ("STRING",)
|
|
RETURN_NAMES = ("prompt_schedule", )
|
|
FUNCTION = "prompt_schedule"
|
|
CATEGORY = "ipadapter/weights"
|
|
|
|
def prompt_schedule(self, weights_strategy, prompt=""):
|
|
frames = weights_strategy["frames"]
|
|
add_starting_frames = weights_strategy["add_starting_frames"]
|
|
add_ending_frames = weights_strategy["add_ending_frames"]
|
|
frame_count = weights_strategy["frame_count"]
|
|
|
|
out = ""
|
|
|
|
prompt = [p for p in prompt.split("\n") if p.strip() != ""]
|
|
|
|
if len(prompt) > 0 and frame_count > 0:
|
|
# prompt_pos must be the same size as the image batch
|
|
if len(prompt) > frame_count:
|
|
prompt = prompt[:frame_count]
|
|
elif len(prompt) < frame_count:
|
|
prompt += [prompt[-1]] * (frame_count - len(prompt))
|
|
|
|
if add_starting_frames > 0:
|
|
out += f"\"0\": \"{prompt[0]}\",\n"
|
|
for i in range(frame_count):
|
|
out += f"\"{i * frames + add_starting_frames}\": \"{prompt[i]}\",\n"
|
|
if add_ending_frames > 0:
|
|
out += f"\"{frame_count * frames + add_starting_frames}\": \"{prompt[-1]}\",\n"
|
|
|
|
return (out, )
|
|
|
|
class IPAdapterCombineWeights:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"weights_1": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05 }),
|
|
"weights_2": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05 }),
|
|
}}
|
|
RETURN_TYPES = ("FLOAT", "INT")
|
|
RETURN_NAMES = ("weights", "count")
|
|
FUNCTION = "combine"
|
|
CATEGORY = "ipadapter/utils"
|
|
|
|
def combine(self, weights_1, weights_2):
|
|
if not isinstance(weights_1, list):
|
|
weights_1 = [weights_1]
|
|
if not isinstance(weights_2, list):
|
|
weights_2 = [weights_2]
|
|
weights = weights_1 + weights_2
|
|
|
|
return (weights, len(weights), )
|
|
|
|
class IPAdapterRegionalConditioning:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
#"set_cond_area": (["default", "mask bounds"],),
|
|
"image": ("IMAGE",),
|
|
"image_weight": ("FLOAT", { "default": 1.0, "min": -1.0, "max": 3.0, "step": 0.05 }),
|
|
"prompt_weight": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 10.0, "step": 0.05 }),
|
|
"weight_type": (WEIGHT_TYPES, ),
|
|
"start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
"end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
|
|
}, "optional": {
|
|
"mask": ("MASK",),
|
|
"positive": ("CONDITIONING",),
|
|
"negative": ("CONDITIONING",),
|
|
}}
|
|
|
|
RETURN_TYPES = ("IPADAPTER_PARAMS", "CONDITIONING", "CONDITIONING", )
|
|
RETURN_NAMES = ("IPADAPTER_PARAMS", "POSITIVE", "NEGATIVE")
|
|
FUNCTION = "conditioning"
|
|
|
|
CATEGORY = "ipadapter/params"
|
|
|
|
def conditioning(self, image, image_weight, prompt_weight, weight_type, start_at, end_at, mask=None, positive=None, negative=None):
|
|
set_area_to_bounds = False #if set_cond_area == "default" else True
|
|
|
|
if mask is not None:
|
|
if positive is not None:
|
|
positive = conditioning_set_values(positive, {"mask": mask, "set_area_to_bounds": set_area_to_bounds, "mask_strength": prompt_weight})
|
|
if negative is not None:
|
|
negative = conditioning_set_values(negative, {"mask": mask, "set_area_to_bounds": set_area_to_bounds, "mask_strength": prompt_weight})
|
|
|
|
ipadapter_params = {
|
|
"image": [image],
|
|
"attn_mask": [mask],
|
|
"weight": [image_weight],
|
|
"weight_type": [weight_type],
|
|
"start_at": [start_at],
|
|
"end_at": [end_at],
|
|
}
|
|
|
|
return (ipadapter_params, positive, negative, )
|
|
|
|
class IPAdapterCombineParams:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"params_1": ("IPADAPTER_PARAMS",),
|
|
"params_2": ("IPADAPTER_PARAMS",),
|
|
}, "optional": {
|
|
"params_3": ("IPADAPTER_PARAMS",),
|
|
"params_4": ("IPADAPTER_PARAMS",),
|
|
"params_5": ("IPADAPTER_PARAMS",),
|
|
}}
|
|
|
|
RETURN_TYPES = ("IPADAPTER_PARAMS",)
|
|
FUNCTION = "combine"
|
|
CATEGORY = "ipadapter/params"
|
|
|
|
def combine(self, params_1, params_2, params_3=None, params_4=None, params_5=None):
|
|
ipadapter_params = {
|
|
"image": params_1["image"] + params_2["image"],
|
|
"attn_mask": params_1["attn_mask"] + params_2["attn_mask"],
|
|
"weight": params_1["weight"] + params_2["weight"],
|
|
"weight_type": params_1["weight_type"] + params_2["weight_type"],
|
|
"start_at": params_1["start_at"] + params_2["start_at"],
|
|
"end_at": params_1["end_at"] + params_2["end_at"],
|
|
}
|
|
|
|
if params_3 is not None:
|
|
ipadapter_params["image"] += params_3["image"]
|
|
ipadapter_params["attn_mask"] += params_3["attn_mask"]
|
|
ipadapter_params["weight"] += params_3["weight"]
|
|
ipadapter_params["weight_type"] += params_3["weight_type"]
|
|
ipadapter_params["start_at"] += params_3["start_at"]
|
|
ipadapter_params["end_at"] += params_3["end_at"]
|
|
if params_4 is not None:
|
|
ipadapter_params["image"] += params_4["image"]
|
|
ipadapter_params["attn_mask"] += params_4["attn_mask"]
|
|
ipadapter_params["weight"] += params_4["weight"]
|
|
ipadapter_params["weight_type"] += params_4["weight_type"]
|
|
ipadapter_params["start_at"] += params_4["start_at"]
|
|
ipadapter_params["end_at"] += params_4["end_at"]
|
|
if params_5 is not None:
|
|
ipadapter_params["image"] += params_5["image"]
|
|
ipadapter_params["attn_mask"] += params_5["attn_mask"]
|
|
ipadapter_params["weight"] += params_5["weight"]
|
|
ipadapter_params["weight_type"] += params_5["weight_type"]
|
|
ipadapter_params["start_at"] += params_5["start_at"]
|
|
ipadapter_params["end_at"] += params_5["end_at"]
|
|
|
|
return (ipadapter_params, )
|
|
|
|
"""
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
Register
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
"""
|
|
NODE_CLASS_MAPPINGS = {
|
|
# Main Apply Nodes
|
|
"IPAdapter": IPAdapterSimple,
|
|
"IPAdapterAdvanced": IPAdapterAdvanced,
|
|
"IPAdapterBatch": IPAdapterBatch,
|
|
"IPAdapterFaceID": IPAdapterFaceID,
|
|
"IPAdapterFaceIDKolors": IPAdapterFaceIDKolors,
|
|
"IPAAdapterFaceIDBatch": IPAAdapterFaceIDBatch,
|
|
"IPAdapterTiled": IPAdapterTiled,
|
|
"IPAdapterTiledBatch": IPAdapterTiledBatch,
|
|
"IPAdapterEmbeds": IPAdapterEmbeds,
|
|
"IPAdapterEmbedsBatch": IPAdapterEmbedsBatch,
|
|
"IPAdapterStyleComposition": IPAdapterStyleComposition,
|
|
"IPAdapterStyleCompositionBatch": IPAdapterStyleCompositionBatch,
|
|
"IPAdapterMS": IPAdapterMS,
|
|
"IPAdapterClipVisionEnhancer": IPAdapterClipVisionEnhancer,
|
|
"IPAdapterClipVisionEnhancerBatch": IPAdapterClipVisionEnhancerBatch,
|
|
"IPAdapterFromParams": IPAdapterFromParams,
|
|
"IPAdapterPreciseStyleTransfer": IPAdapterPreciseStyleTransfer,
|
|
"IPAdapterPreciseStyleTransferBatch": IPAdapterPreciseStyleTransferBatch,
|
|
"IPAdapterPreciseComposition": IPAdapterPreciseComposition,
|
|
"IPAdapterPreciseCompositionBatch": IPAdapterPreciseCompositionBatch,
|
|
|
|
# Loaders
|
|
"IPAdapterUnifiedLoader": IPAdapterUnifiedLoader,
|
|
"IPAdapterUnifiedLoaderFaceID": IPAdapterUnifiedLoaderFaceID,
|
|
"IPAdapterModelLoader": IPAdapterModelLoader,
|
|
"IPAdapterInsightFaceLoader": IPAdapterInsightFaceLoader,
|
|
"IPAdapterUnifiedLoaderCommunity": IPAdapterUnifiedLoaderCommunity,
|
|
|
|
# Helpers
|
|
"IPAdapterEncoder": IPAdapterEncoder,
|
|
"IPAdapterCombineEmbeds": IPAdapterCombineEmbeds,
|
|
"IPAdapterNoise": IPAdapterNoise,
|
|
"PrepImageForClipVision": PrepImageForClipVision,
|
|
"IPAdapterSaveEmbeds": IPAdapterSaveEmbeds,
|
|
"IPAdapterLoadEmbeds": IPAdapterLoadEmbeds,
|
|
"IPAdapterWeights": IPAdapterWeights,
|
|
"IPAdapterCombineWeights": IPAdapterCombineWeights,
|
|
"IPAdapterWeightsFromStrategy": IPAdapterWeightsFromStrategy,
|
|
"IPAdapterPromptScheduleFromWeightsStrategy": IPAdapterPromptScheduleFromWeightsStrategy,
|
|
"IPAdapterRegionalConditioning": IPAdapterRegionalConditioning,
|
|
"IPAdapterCombineParams": IPAdapterCombineParams,
|
|
}
|
|
|
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
|
# Main Apply Nodes
|
|
"IPAdapter": "IPAdapter",
|
|
"IPAdapterAdvanced": "IPAdapter Advanced",
|
|
"IPAdapterBatch": "IPAdapter Batch (Adv.)",
|
|
"IPAdapterFaceID": "IPAdapter FaceID",
|
|
"IPAdapterFaceIDKolors": "IPAdapter FaceID Kolors",
|
|
"IPAAdapterFaceIDBatch": "IPAdapter FaceID Batch",
|
|
"IPAdapterTiled": "IPAdapter Tiled",
|
|
"IPAdapterTiledBatch": "IPAdapter Tiled Batch",
|
|
"IPAdapterEmbeds": "IPAdapter Embeds",
|
|
"IPAdapterEmbedsBatch": "IPAdapter Embeds Batch",
|
|
"IPAdapterStyleComposition": "IPAdapter Style & Composition SDXL",
|
|
"IPAdapterStyleCompositionBatch": "IPAdapter Style & Composition Batch SDXL",
|
|
"IPAdapterMS": "IPAdapter Mad Scientist",
|
|
"IPAdapterClipVisionEnhancer": "IPAdapter ClipVision Enhancer",
|
|
"IPAdapterClipVisionEnhancerBatch": "IPAdapter ClipVision Enhancer Batch",
|
|
"IPAdapterFromParams": "IPAdapter from Params",
|
|
"IPAdapterPreciseStyleTransfer": "IPAdapter Precise Style Transfer",
|
|
"IPAdapterPreciseStyleTransferBatch": "IPAdapter Precise Style Transfer Batch",
|
|
"IPAdapterPreciseComposition": "IPAdapter Precise Composition",
|
|
"IPAdapterPreciseCompositionBatch": "IPAdapter Precise Composition Batch",
|
|
|
|
# Loaders
|
|
"IPAdapterUnifiedLoader": "IPAdapter Unified Loader",
|
|
"IPAdapterUnifiedLoaderFaceID": "IPAdapter Unified Loader FaceID",
|
|
"IPAdapterModelLoader": "IPAdapter Model Loader",
|
|
"IPAdapterInsightFaceLoader": "IPAdapter InsightFace Loader",
|
|
"IPAdapterUnifiedLoaderCommunity": "IPAdapter Unified Loader Community",
|
|
|
|
# Helpers
|
|
"IPAdapterEncoder": "IPAdapter Encoder",
|
|
"IPAdapterCombineEmbeds": "IPAdapter Combine Embeds",
|
|
"IPAdapterNoise": "IPAdapter Noise",
|
|
"PrepImageForClipVision": "Prep Image For ClipVision",
|
|
"IPAdapterSaveEmbeds": "IPAdapter Save Embeds",
|
|
"IPAdapterLoadEmbeds": "IPAdapter Load Embeds",
|
|
"IPAdapterWeights": "IPAdapter Weights",
|
|
"IPAdapterWeightsFromStrategy": "IPAdapter Weights From Strategy",
|
|
"IPAdapterPromptScheduleFromWeightsStrategy": "Prompt Schedule From Weights Strategy",
|
|
"IPAdapterCombineWeights": "IPAdapter Combine Weights",
|
|
"IPAdapterRegionalConditioning": "IPAdapter Regional Conditioning",
|
|
"IPAdapterCombineParams": "IPAdapter Combine Params",
|
|
} |