nodes/banodoco-steerable-motion
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated IPAdapter

Browse Source
This commit is contained in:
IDGallagher
2024-04-19 21:19:59 +01:00
parent 05ee88b077
commit f4000bf8d7
3 changed files with 107 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
imports/ComfyUI_IPAdapter_plus/CrossAttentionPatchImport.py
+21 -12
View File
@@ -6,7 +6,7 @@ from .utils import tensor_to_size
class CrossAttentionPatchImport:
# forward for patching
def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'):
def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'):
self.weights = [weight]
self.ipadapters = [ipadapter]
self.conds = [cond]
@@ -17,6 +17,7 @@ class CrossAttentionPatchImport:
self.sigma_starts = [sigma_start]
self.sigma_ends = [sigma_end]
self.unfold_batch = [unfold_batch]
self.image_schedule = [image_schedule]
self.embeds_scaling = [embeds_scaling]
self.number = number
self.layers = 11 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 16 # TODO: check if this is a valid condition to detect all models
@@ -24,7 +25,7 @@ class CrossAttentionPatchImport:
self.k_key = str(self.number*2+1) + "_to_k_ip"
self.v_key = str(self.number*2+1) + "_to_v_ip"
def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, embeds_scaling='V only'):
def set_new_condition(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=None, uncond=None, weight_type="linear", mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False, image_schedule=None, embeds_scaling='V only'):
self.weights.append(weight)
self.ipadapters.append(ipadapter)
self.conds.append(cond)
@@ -35,6 +36,7 @@ class CrossAttentionPatchImport:
self.sigma_starts.append(sigma_start)
self.sigma_ends.append(sigma_end)
self.unfold_batch.append(unfold_batch)
self.image_schedule.append(image_schedule)
self.embeds_scaling.append(embeds_scaling)
def __call__(self, q, k, v, extra_options):
@@ -54,7 +56,7 @@ class CrossAttentionPatchImport:
out = optimized_attention(q, k, v, extra_options["n_heads"])
_, _, oh, ow = extra_options["original_shape"]
for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.embeds_scaling):
for weight, cond, cond_alt, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch, image_schedule, embeds_scaling in zip(self.weights, self.conds, self.conds_alt, self.unconds, self.ipadapters, self.masks, self.weight_types, self.sigma_starts, self.sigma_ends, self.unfold_batch, self.image_schedule, self.embeds_scaling):
if sigma <= sigma_start and sigma >= sigma_end:
if weight_type == 'ease in':
weight = weight * (0.05 + 0.95 * (1 - t_idx / self.layers))
@@ -94,16 +96,23 @@ class CrossAttentionPatchImport:
elif weight == 0:
continue
# if image length matches or exceeds full_length get sub_idx images
if cond.shape[0] >= ad_params["full_length"]:
cond = torch.Tensor(cond[ad_params["sub_idxs"]])
uncond = torch.Tensor(uncond[ad_params["sub_idxs"]])
# otherwise get sub_idxs images
if image_schedule is not None:
# Use the image_schedule as a lookup table to get the embedded image corresponding to each sub_idx
# If image_schedule isn't long enough then use the last image
cond_idxs = [image_schedule[i if i < len(image_schedule) else -1] for i in ad_params["sub_idxs"]]
cond = torch.Tensor(cond[cond_idxs])
uncond = torch.Tensor(uncond[cond_idxs])
else:
cond = tensor_to_size(cond, ad_params["full_length"])
uncond = tensor_to_size(uncond, ad_params["full_length"])
cond = cond[ad_params["sub_idxs"]]
uncond = uncond[ad_params["sub_idxs"]]
# if image length matches or exceeds full_length get sub_idx images
if cond.shape[0] >= ad_params["full_length"]:
cond = torch.Tensor(cond[ad_params["sub_idxs"]])
uncond = torch.Tensor(uncond[ad_params["sub_idxs"]])
# otherwise get sub_idxs images
else:
cond = tensor_to_size(cond, ad_params["full_length"])
uncond = tensor_to_size(uncond, ad_params["full_length"])
cond = cond[ad_params["sub_idxs"]]
uncond = uncond[ad_params["sub_idxs"]]
else:
if isinstance(weight, torch.Tensor):
weight = tensor_to_size(weight, batch_prompt)
imports/ComfyUI_IPAdapter_plus/IPAdapterPlus.py
+54 -11
View File
@@ -45,7 +45,7 @@ WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out'
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""
class IPAdapterImport(nn.Module):
def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False):
def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, is_sdxl=False, is_plus=False, is_full=False, is_faceid=False, is_portrait_unnorm=False):
super().__init__()
self.clip_embeddings_dim = clip_embeddings_dim
@@ -55,12 +55,13 @@ class IPAdapterImport(nn.Module):
self.is_sdxl = is_sdxl
self.is_full = is_full
self.is_plus = is_plus
self.is_portrait_unnorm = is_portrait_unnorm
if is_faceid:
if is_faceid and not is_portrait_unnorm:
self.image_proj_model = self.init_proj_faceid()
elif is_full:
self.image_proj_model = self.init_proj_full()
elif is_plus:
elif is_plus or is_portrait_unnorm:
self.image_proj_model = self.init_proj_plus()
else:
self.image_proj_model = self.init_proj()
@@ -161,6 +162,7 @@ def ipadapter_execute(model,
pos_embed=None,
neg_embed=None,
unfold_batch=False,
image_schedule=None,
embeds_scaling='V only',
layer_weights=None):
device = model_management.get_torch_device()
@@ -170,8 +172,9 @@ def ipadapter_execute(model,
is_full = "proj.3.weight" in ipadapter["image_proj"]
is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"]
is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"]
is_plus = is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]
is_portrait_unnorm = "portraitunnorm" in ipadapter
is_faceid = is_portrait or "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] or is_portrait_unnorm
is_plus = (is_full or "latents" in ipadapter["image_proj"] or "perceiver_resampler.proj_in.weight" in ipadapter["image_proj"]) and not is_portrait_unnorm
is_faceidv2 = "faceidplusv2" in ipadapter
output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1]
is_sdxl = output_cross_attention_dim == 2048
@@ -182,8 +185,8 @@ def ipadapter_execute(model,
if is_faceidv2:
weight_faceidv2 = weight_faceidv2 if weight_faceidv2 is not None else weight*2
cross_attention_dim = 1280 if is_plus and is_sdxl and not is_faceid else output_cross_attention_dim
clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait else 4
cross_attention_dim = 1280 if (is_plus and is_sdxl and not is_faceid) or is_portrait_unnorm else output_cross_attention_dim
clip_extra_context_tokens = 16 if (is_plus and not is_faceid) or is_portrait or is_portrait_unnorm else 4
if image is not None and image.shape[1] != image.shape[2]:
print("\033[33mINFO: the IPAdapter reference image is not a square, CLIPImageProcessor will resize and crop it at the center. If the main focus of the picture is not in the middle the result might not be what you are expecting.\033[0m")
@@ -233,7 +236,10 @@ def ipadapter_execute(model,
insightface.det_model.input_size = size # TODO: hacky but seems to be working
face = insightface.get(image_iface[i])
if face:
face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0))
if not is_portrait_unnorm:
face_cond_embeds.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0))
else:
face_cond_embeds.append(torch.from_numpy(face[0].embedding).unsqueeze(0))
image.append(image_to_tensor(face_align.norm_crop(image_iface[i], landmark=face[0].kps, image_size=256)))
if 640 not in size:
@@ -330,7 +336,8 @@ def ipadapter_execute(model,
is_sdxl=is_sdxl,
is_plus=is_plus,
is_full=is_full,
is_faceid=is_faceid
is_faceid=is_faceid,
is_portrait_unnorm=is_portrait_unnorm,
).to(device, dtype=dtype)
if is_faceid and is_plus:
@@ -365,6 +372,7 @@ def ipadapter_execute(model,
"sigma_start": sigma_start,
"sigma_end": sigma_end,
"unfold_batch": unfold_batch,
"image_schedule": image_schedule,
"embeds_scaling": embeds_scaling,
}
@@ -492,7 +500,7 @@ class IPAdapterUnifiedLoaderFaceID(IPAdapterUnifiedLoader):
def INPUT_TYPES(s):
return {"required": {
"model": ("MODEL", ),
"preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)'], ),
"preset": (['FACEID', 'FACEID PLUS - SD1.5 only', 'FACEID PLUS V2', 'FACEID PORTRAIT (style transfer)', 'FACEID PORTRAIT UNNORM - SDXL only (strong)'], ),
"lora_strength": ("FLOAT", { "default": 0.6, "min": 0, "max": 1, "step": 0.01 }),
"provider": (["CPU", "CUDA", "ROCM", "DirectML", "OpenVINO", "CoreML"], ),
},
@@ -626,7 +634,7 @@ class IPAdapterAdvancedImport:
FUNCTION = "apply_ipadapter"
CATEGORY = "ipadapter"
def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None):
def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, image_schedule=None, attn_mask=None, insightface=None, embeds_scaling='V only', layer_weights=None):
is_sdxl = isinstance(model.model, (comfy.model_base.SDXL, comfy.model_base.SDXLRefiner, comfy.model_base.SDXL_instructpix2pix))
if image_style is not None: # we are doing style + composition transfer
@@ -653,6 +661,7 @@ class IPAdapterAdvancedImport:
"end_at": end_at,
"attn_mask": attn_mask,
"unfold_batch": self.unfold_batch,
"image_schedule": image_schedule,
"embeds_scaling": embeds_scaling,
"insightface": insightface if insightface is not None else ipadapter['insightface']['model'] if 'insightface' in ipadapter else None,
"layer_weights": layer_weights,
@@ -693,6 +702,7 @@ class IPAdapterBatchImport(IPAdapterAdvancedImport):
"image_negative": ("IMAGE",),
"attn_mask": ("MASK",),
"clip_vision": ("CLIP_VISION",),
"image_schedule": ("INT", {"default": None, "forceInput": True} ),
}
}
@@ -745,6 +755,7 @@ class IPAdapterStyleCompositionBatch(IPAdapterStyleComposition):
"image_negative": ("IMAGE",),
"attn_mask": ("MASK",),
"clip_vision": ("CLIP_VISION",),
"image_schedule": ("INT", {"default": None, "forceInput": True} ),
}
}
@@ -935,6 +946,7 @@ class IPAdapterTiledBatchImport(IPAdapterTiledImport):
"image_negative": ("IMAGE",),
"attn_mask": ("MASK",),
"clip_vision": ("CLIP_VISION",),
"image_schedule": ("INT", {"default": None, "forceInput": True} ),
}
}
@@ -1283,6 +1295,37 @@ class IPAdapterLoadEmbeds:
path = folder_paths.get_annotated_filepath(embeds)
return (torch.load(path).cpu(), )
defaultValue="""0:0,
40:1,
80:2,
"""
class IPAdapterImageSchedule:
@classmethod
def INPUT_TYPES(s):
return {"required": {"text": ("STRING", {"multiline": True, "default": defaultValue}),
"max_frames": ("INT", {"default": 120.0, "min": 1.0, "max": 999999.0, "step": 1.0}),
"print_output": ("BOOLEAN", {"default": False})}}
RETURN_TYPES = ("INT",)
FUNCTION = "schedule"
CATEGORY = "ipadapter/utils"
def schedule(self, text, max_frames, print_output):
frames = [0] * max_frames
for item in text.split(","):
item = item.strip()
if ":" in item:
parts = item.split(":")
if len(parts) == 2:
start_frame = int(parts[0])
value = int(parts[1])
for i in range(start_frame, max_frames):
frames[i] = value
if print_output is True:
print("ValueSchedule: ", frames)
return (frames, )
class IPAdapterWeights:
@classmethod
def INPUT_TYPES(s):
imports/ComfyUI_IPAdapter_plus/utils.py
+32 -10
View File
@@ -61,7 +61,7 @@ def get_ipadapter_file(preset, is_sdxl):
if is_sdxl:
raise Exception("full face model is not supported for SDXL")
pattern = 'full.face.sd15\.(safetensors|bin)$'
elif preset.startswith("faceid portrait"):
elif preset.startswith("faceid portrait ("):
if is_sdxl:
pattern = 'portrait.sdxl\.(safetensors|bin)$'
else:
@@ -70,6 +70,12 @@ def get_ipadapter_file(preset, is_sdxl):
if not [e for e in ipadapter_list if re.search(pattern, e, re.IGNORECASE)]:
pattern = 'portrait.sd15\.(safetensors|bin)$'
is_insightface = True
elif preset.startswith("faceid portrait unnorm"):
if is_sdxl:
pattern = 'portrait.sdxl.unnorm\.(safetensors|bin)$'
else:
raise Exception("portrait unnorm model is not supported for SD1.5")
is_insightface = True
elif preset == "faceid":
if is_sdxl:
pattern = 'faceid.sdxl\.(safetensors|bin)$'
@@ -131,6 +137,9 @@ def ipadapter_model_loader(file):
if 'plusv2' in file.lower():
model["faceidplusv2"] = True
if 'unnorm' in file.lower():
model["portraitunnorm"] = True
return model
@@ -145,21 +154,34 @@ def insightface_loader(provider):
model.prepare(ctx_id=0, det_size=(640, 640))
return model
def encode_image_masked(clip_vision, image, mask=None):
def encode_image_masked(clip_vision, images, mask=None):
model_management.load_model_gpu(clip_vision.patcher)
image = image.to(clip_vision.load_device)
pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float()
# Initialize lists to collect outputs
last_hidden_states = []
image_embeds = []
penultimate_hidden_states = []
if mask is not None:
pixel_values = pixel_values * mask.to(clip_vision.load_device)
# Loop over each image in the batch
for image in images:
pixel_values = clip_preprocess(image.to(clip_vision.load_device).unsqueeze(0)).float()
out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2)
if mask is not None:
pixel_values *= mask.to(clip_vision.load_device)
out = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2)
# Collect the outputs for each image
last_hidden_states.append(out[0].to(model_management.intermediate_device()))
image_embeds.append(out[2].to(model_management.intermediate_device()))
penultimate_hidden_states.append(out[1].to(model_management.intermediate_device()))
# Concatenate all collected outputs across the batch
outputs = Output()
outputs["last_hidden_state"] = out[0].to(model_management.intermediate_device())
outputs["image_embeds"] = out[2].to(model_management.intermediate_device())
outputs["penultimate_hidden_states"] = out[1].to(model_management.intermediate_device())
outputs["last_hidden_state"] = torch.cat(last_hidden_states, dim=0)
outputs["image_embeds"] = torch.cat(image_embeds, dim=0)
outputs["penultimate_hidden_states"] = torch.cat(penultimate_hidden_states, dim=0)
return outputs
def tensor_to_size(source, dest_size):