bugfix
This commit is contained in:
matt3o
+196
-7
@@ -376,7 +376,7 @@ class FaceKeypointsPreprocessor:
|
||||
CATEGORY = "InstantID"
|
||||
|
||||
def preprocess_image(self, faceanalysis, image):
|
||||
face_kps = extractFeatures(faceanalysis, image, extract_kps=True)
|
||||
face_kps = extractFeatures(faceanalysis, image[0].unsqueeze(0), extract_kps=True)
|
||||
|
||||
if face_kps is None:
|
||||
face_kps = torch.zeros_like(image)
|
||||
@@ -408,7 +408,7 @@ class ApplyInstantID:
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL", "CONDITIONING", "CONDITIONING",)
|
||||
RETURN_NAMES = ("MODEL", "POSITIVE", "NEGATIVE", )
|
||||
RETURN_NAMES = ("MODEL", "positive", "negative", )
|
||||
FUNCTION = "apply_instantid"
|
||||
CATEGORY = "InstantID"
|
||||
|
||||
@@ -427,7 +427,7 @@ class ApplyInstantID:
|
||||
face_embed = extractFeatures(insightface, image)
|
||||
if face_embed is None:
|
||||
raise Exception('Reference Image: No face detected.')
|
||||
|
||||
|
||||
face_kps = extractFeatures(insightface, image_kps[0].unsqueeze(0) if image_kps is not None else image[0].unsqueeze(0), extract_kps=True)
|
||||
|
||||
if face_kps is None:
|
||||
@@ -439,7 +439,7 @@ class ApplyInstantID:
|
||||
if clip_embed.shape[0] > 1:
|
||||
clip_embed = torch.mean(clip_embed, dim=0).unsqueeze(0)
|
||||
|
||||
clip_embed_zeroed = torch.zeros_like(clip_embed)
|
||||
clip_embed_zeroed = torch.zeros_like(clip_embed) # torch.rand_like(clip_embed)
|
||||
|
||||
clip_embeddings_dim = face_embed.shape[-1]
|
||||
|
||||
@@ -507,11 +507,11 @@ class ApplyInstantID:
|
||||
mask = mask.unsqueeze(0)
|
||||
|
||||
cnets = {}
|
||||
|
||||
cond_uncond = []
|
||||
|
||||
is_cond = True
|
||||
for conditioning in [positive, negative]:
|
||||
c = []
|
||||
is_cond = True
|
||||
for t in conditioning:
|
||||
d = t[1].copy()
|
||||
|
||||
@@ -533,8 +533,8 @@ class ApplyInstantID:
|
||||
|
||||
n = [t[0], d]
|
||||
c.append(n)
|
||||
is_cond = True
|
||||
cond_uncond.append(c)
|
||||
is_cond = False
|
||||
|
||||
return(work_model, cond_uncond[0], cond_uncond[1], )
|
||||
|
||||
@@ -561,12 +561,198 @@ class ApplyInstantIDAdvanced(ApplyInstantID):
|
||||
}
|
||||
}
|
||||
|
||||
class InstantIDAttentionPatch:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"instantid": ("INSTANTID", ),
|
||||
"insightface": ("FACEANALYSIS", ),
|
||||
"image": ("IMAGE", ),
|
||||
"model": ("MODEL", ),
|
||||
"weight": ("FLOAT", {"default": 1.0, "min": -1.0, "max": 3.0, "step": 0.01, }),
|
||||
"start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, }),
|
||||
"end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, }),
|
||||
},
|
||||
"optional": {
|
||||
"mask": ("MASK",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL", "FACE_EMBEDS")
|
||||
FUNCTION = "patch_attention"
|
||||
CATEGORY = "InstantID"
|
||||
|
||||
def patch_attention(self, instantid, insightface, image, model, weight, start_at, end_at, mask=None):
|
||||
self.dtype = torch.float16 if comfy.model_management.should_use_fp16() else torch.float32
|
||||
self.device = comfy.model_management.get_torch_device()
|
||||
|
||||
output_cross_attention_dim = instantid["ip_adapter"]["1.to_k_ip.weight"].shape[1]
|
||||
is_sdxl = output_cross_attention_dim == 2048
|
||||
cross_attention_dim = 1280
|
||||
clip_extra_context_tokens = 16
|
||||
|
||||
face_embed = extractFeatures(insightface, image)
|
||||
if face_embed is None:
|
||||
raise Exception('Reference Image: No face detected.')
|
||||
|
||||
clip_embed = face_embed
|
||||
# InstantID works better with averaged embeds (TODO: needs testing)
|
||||
if clip_embed.shape[0] > 1:
|
||||
clip_embed = torch.mean(clip_embed, dim=0).unsqueeze(0)
|
||||
|
||||
clip_embed_zeroed = torch.zeros_like(clip_embed) # torch.rand_like(clip_embed)
|
||||
|
||||
clip_embeddings_dim = face_embed.shape[-1]
|
||||
|
||||
# 1: patch the attention
|
||||
self.instantid = InstantID(
|
||||
instantid,
|
||||
cross_attention_dim=cross_attention_dim,
|
||||
output_cross_attention_dim=output_cross_attention_dim,
|
||||
clip_embeddings_dim=clip_embeddings_dim,
|
||||
clip_extra_context_tokens=clip_extra_context_tokens,
|
||||
)
|
||||
|
||||
self.instantid.to(self.device, dtype=self.dtype)
|
||||
|
||||
image_prompt_embeds, uncond_image_prompt_embeds = self.instantid.get_image_embeds(clip_embed.to(self.device, dtype=self.dtype), clip_embed_zeroed.to(self.device, dtype=self.dtype))
|
||||
|
||||
image_prompt_embeds = image_prompt_embeds.to(self.device, dtype=self.dtype)
|
||||
uncond_image_prompt_embeds = uncond_image_prompt_embeds.to(self.device, dtype=self.dtype)
|
||||
|
||||
if weight == 0:
|
||||
return (model, { "cond": image_prompt_embeds, "uncond": uncond_image_prompt_embeds } )
|
||||
|
||||
work_model = model.clone()
|
||||
|
||||
sigma_start = work_model.model.model_sampling.percent_to_sigma(start_at)
|
||||
sigma_end = work_model.model.model_sampling.percent_to_sigma(end_at)
|
||||
|
||||
if mask is not None:
|
||||
mask = mask.to(self.device)
|
||||
|
||||
patch_kwargs = {
|
||||
"number": 0,
|
||||
"weight": weight,
|
||||
"ipadapter": self.instantid,
|
||||
"cond": image_prompt_embeds,
|
||||
"uncond": uncond_image_prompt_embeds,
|
||||
"mask": mask,
|
||||
"sigma_start": sigma_start,
|
||||
"sigma_end": sigma_end,
|
||||
"weight_type": "original",
|
||||
}
|
||||
|
||||
if not is_sdxl:
|
||||
for id in [1,2,4,5,7,8]: # id of input_blocks that have cross attention
|
||||
_set_model_patch_replace(work_model, patch_kwargs, ("input", id))
|
||||
patch_kwargs["number"] += 1
|
||||
for id in [3,4,5,6,7,8,9,10,11]: # id of output_blocks that have cross attention
|
||||
_set_model_patch_replace(work_model, patch_kwargs, ("output", id))
|
||||
patch_kwargs["number"] += 1
|
||||
_set_model_patch_replace(work_model, patch_kwargs, ("middle", 0))
|
||||
else:
|
||||
for id in [4,5,7,8]: # id of input_blocks that have cross attention
|
||||
block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth
|
||||
for index in block_indices:
|
||||
_set_model_patch_replace(work_model, patch_kwargs, ("input", id, index))
|
||||
patch_kwargs["number"] += 1
|
||||
for id in range(6): # id of output_blocks that have cross attention
|
||||
block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth
|
||||
for index in block_indices:
|
||||
_set_model_patch_replace(work_model, patch_kwargs, ("output", id, index))
|
||||
patch_kwargs["number"] += 1
|
||||
for index in range(10):
|
||||
_set_model_patch_replace(work_model, patch_kwargs, ("middle", 0, index))
|
||||
patch_kwargs["number"] += 1
|
||||
|
||||
return(work_model, { "cond": image_prompt_embeds, "uncond": uncond_image_prompt_embeds }, )
|
||||
|
||||
class ApplyInstantIDControlNet:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"face_embeds": ("FACE_EMBEDS", ),
|
||||
"control_net": ("CONTROL_NET", ),
|
||||
"image_kps": ("IMAGE", ),
|
||||
"positive": ("CONDITIONING", ),
|
||||
"negative": ("CONDITIONING", ),
|
||||
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, }),
|
||||
"start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, }),
|
||||
"end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, }),
|
||||
},
|
||||
"optional": {
|
||||
"mask": ("MASK",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("CONDITIONING", "CONDITIONING",)
|
||||
RETURN_NAMES = ("positive", "negative", )
|
||||
FUNCTION = "apply_controlnet"
|
||||
CATEGORY = "InstantID"
|
||||
|
||||
def apply_controlnet(self, face_embeds, control_net, image_kps, positive, negative, strength, start_at, end_at, mask=None):
|
||||
self.device = comfy.model_management.get_torch_device()
|
||||
|
||||
if strength == 0:
|
||||
return (positive, negative)
|
||||
|
||||
if mask is not None:
|
||||
mask = mask.to(self.device)
|
||||
|
||||
if mask is not None and len(mask.shape) < 3:
|
||||
mask = mask.unsqueeze(0)
|
||||
|
||||
image_prompt_embeds = face_embeds['cond']
|
||||
uncond_image_prompt_embeds = face_embeds['uncond']
|
||||
|
||||
cnets = {}
|
||||
cond_uncond = []
|
||||
control_hint = image_kps.movedim(-1,1)
|
||||
|
||||
is_cond = True
|
||||
for conditioning in [positive, negative]:
|
||||
c = []
|
||||
for t in conditioning:
|
||||
d = t[1].copy()
|
||||
|
||||
prev_cnet = d.get('control', None)
|
||||
if prev_cnet in cnets:
|
||||
c_net = cnets[prev_cnet]
|
||||
else:
|
||||
c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_at, end_at))
|
||||
c_net.set_previous_controlnet(prev_cnet)
|
||||
cnets[prev_cnet] = c_net
|
||||
|
||||
d['control'] = c_net
|
||||
d['control_apply_to_uncond'] = False
|
||||
d['cross_attn_controlnet'] = image_prompt_embeds.to(comfy.model_management.intermediate_device()) if is_cond else uncond_image_prompt_embeds.to(comfy.model_management.intermediate_device())
|
||||
|
||||
if mask is not None and is_cond:
|
||||
d['mask'] = mask
|
||||
d['set_area_to_bounds'] = False
|
||||
|
||||
n = [t[0], d]
|
||||
c.append(n)
|
||||
cond_uncond.append(c)
|
||||
is_cond = False
|
||||
|
||||
print(cond_uncond[0])
|
||||
|
||||
return(cond_uncond[0], cond_uncond[1])
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"InstantIDModelLoader": InstantIDModelLoader,
|
||||
"InstantIDFaceAnalysis": InstantIDFaceAnalysis,
|
||||
"ApplyInstantID": ApplyInstantID,
|
||||
"ApplyInstantIDAdvanced": ApplyInstantIDAdvanced,
|
||||
"FaceKeypointsPreprocessor": FaceKeypointsPreprocessor,
|
||||
|
||||
"InstantIDAttentionPatch": InstantIDAttentionPatch,
|
||||
"ApplyInstantIDControlNet": ApplyInstantIDControlNet,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
@@ -575,4 +761,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"ApplyInstantID": "Apply InstantID",
|
||||
"ApplyInstantIDAdvanced": "Apply InstantID Advanced",
|
||||
"FaceKeypointsPreprocessor": "Face Keypoints Preprocessor",
|
||||
|
||||
"InstantIDAttentionPatch": "InstantID Patch Attention",
|
||||
"ApplyInstantIDControlNet": "InstantID Apply ControlNet",
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ This extension differs from the many already available as it doesn't use *diffus
|
||||
|
||||
## Important updates
|
||||
|
||||
- **2024/02/26:** Fixed a small but nasty bug. Results will be different and you may need to lower the CFG.
|
||||
|
||||
- **2024/02/20:** I refactored the nodes so they are hopefully easier to use. **This is a breaking update**, the previous workflows won't work anymore.
|
||||
|
||||
## Basic Workflow
|
||||
@@ -20,7 +22,7 @@ In the `examples` directory you'll find some basic workflows.
|
||||
<img src="https://img.youtube.com/vi/wMLiGhogOPE/hqdefault.jpg" alt="Watch the video" />
|
||||
</a>
|
||||
|
||||
**:movie_camera: [Introduction to InstantID features](https://youtu.be/wMLiGhogOPE)**
|
||||
** :movie_camera: [Introduction to InstantID features](https://youtu.be/wMLiGhogOPE)**
|
||||
|
||||
## Installation
|
||||
|
||||
|
||||
Reference in New Issue
Block a user