From 6938fc7ee8fb3dd3900993596ba91b95c311c267 Mon Sep 17 00:00:00 2001 From: matt3o Date: Mon, 26 Feb 2024 21:20:11 +0100 Subject: [PATCH] bugfix --- InstantID.py | 203 +++++++++++++++++++++++++++++++++++++++++++++++++-- README.md | 4 +- 2 files changed, 199 insertions(+), 8 deletions(-) diff --git a/InstantID.py b/InstantID.py index 832d2fc..a35a63d 100644 --- a/InstantID.py +++ b/InstantID.py @@ -376,7 +376,7 @@ class FaceKeypointsPreprocessor: CATEGORY = "InstantID" def preprocess_image(self, faceanalysis, image): - face_kps = extractFeatures(faceanalysis, image, extract_kps=True) + face_kps = extractFeatures(faceanalysis, image[0].unsqueeze(0), extract_kps=True) if face_kps is None: face_kps = torch.zeros_like(image) @@ -408,7 +408,7 @@ class ApplyInstantID: } RETURN_TYPES = ("MODEL", "CONDITIONING", "CONDITIONING",) - RETURN_NAMES = ("MODEL", "POSITIVE", "NEGATIVE", ) + RETURN_NAMES = ("MODEL", "positive", "negative", ) FUNCTION = "apply_instantid" CATEGORY = "InstantID" @@ -427,7 +427,7 @@ class ApplyInstantID: face_embed = extractFeatures(insightface, image) if face_embed is None: raise Exception('Reference Image: No face detected.') - + face_kps = extractFeatures(insightface, image_kps[0].unsqueeze(0) if image_kps is not None else image[0].unsqueeze(0), extract_kps=True) if face_kps is None: @@ -439,7 +439,7 @@ class ApplyInstantID: if clip_embed.shape[0] > 1: clip_embed = torch.mean(clip_embed, dim=0).unsqueeze(0) - clip_embed_zeroed = torch.zeros_like(clip_embed) + clip_embed_zeroed = torch.zeros_like(clip_embed) # torch.rand_like(clip_embed) clip_embeddings_dim = face_embed.shape[-1] @@ -507,11 +507,11 @@ class ApplyInstantID: mask = mask.unsqueeze(0) cnets = {} - cond_uncond = [] + + is_cond = True for conditioning in [positive, negative]: c = [] - is_cond = True for t in conditioning: d = t[1].copy() @@ -533,8 +533,8 @@ class ApplyInstantID: n = [t[0], d] c.append(n) - is_cond = True cond_uncond.append(c) + is_cond = False return(work_model, cond_uncond[0], cond_uncond[1], ) @@ -561,12 +561,198 @@ class ApplyInstantIDAdvanced(ApplyInstantID): } } +class InstantIDAttentionPatch: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "instantid": ("INSTANTID", ), + "insightface": ("FACEANALYSIS", ), + "image": ("IMAGE", ), + "model": ("MODEL", ), + "weight": ("FLOAT", {"default": 1.0, "min": -1.0, "max": 3.0, "step": 0.01, }), + "start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, }), + "end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, }), + }, + "optional": { + "mask": ("MASK",), + } + } + + RETURN_TYPES = ("MODEL", "FACE_EMBEDS") + FUNCTION = "patch_attention" + CATEGORY = "InstantID" + + def patch_attention(self, instantid, insightface, image, model, weight, start_at, end_at, mask=None): + self.dtype = torch.float16 if comfy.model_management.should_use_fp16() else torch.float32 + self.device = comfy.model_management.get_torch_device() + + output_cross_attention_dim = instantid["ip_adapter"]["1.to_k_ip.weight"].shape[1] + is_sdxl = output_cross_attention_dim == 2048 + cross_attention_dim = 1280 + clip_extra_context_tokens = 16 + + face_embed = extractFeatures(insightface, image) + if face_embed is None: + raise Exception('Reference Image: No face detected.') + + clip_embed = face_embed + # InstantID works better with averaged embeds (TODO: needs testing) + if clip_embed.shape[0] > 1: + clip_embed = torch.mean(clip_embed, dim=0).unsqueeze(0) + + clip_embed_zeroed = torch.zeros_like(clip_embed) # torch.rand_like(clip_embed) + + clip_embeddings_dim = face_embed.shape[-1] + + # 1: patch the attention + self.instantid = InstantID( + instantid, + cross_attention_dim=cross_attention_dim, + output_cross_attention_dim=output_cross_attention_dim, + clip_embeddings_dim=clip_embeddings_dim, + clip_extra_context_tokens=clip_extra_context_tokens, + ) + + self.instantid.to(self.device, dtype=self.dtype) + + image_prompt_embeds, uncond_image_prompt_embeds = self.instantid.get_image_embeds(clip_embed.to(self.device, dtype=self.dtype), clip_embed_zeroed.to(self.device, dtype=self.dtype)) + + image_prompt_embeds = image_prompt_embeds.to(self.device, dtype=self.dtype) + uncond_image_prompt_embeds = uncond_image_prompt_embeds.to(self.device, dtype=self.dtype) + + if weight == 0: + return (model, { "cond": image_prompt_embeds, "uncond": uncond_image_prompt_embeds } ) + + work_model = model.clone() + + sigma_start = work_model.model.model_sampling.percent_to_sigma(start_at) + sigma_end = work_model.model.model_sampling.percent_to_sigma(end_at) + + if mask is not None: + mask = mask.to(self.device) + + patch_kwargs = { + "number": 0, + "weight": weight, + "ipadapter": self.instantid, + "cond": image_prompt_embeds, + "uncond": uncond_image_prompt_embeds, + "mask": mask, + "sigma_start": sigma_start, + "sigma_end": sigma_end, + "weight_type": "original", + } + + if not is_sdxl: + for id in [1,2,4,5,7,8]: # id of input_blocks that have cross attention + _set_model_patch_replace(work_model, patch_kwargs, ("input", id)) + patch_kwargs["number"] += 1 + for id in [3,4,5,6,7,8,9,10,11]: # id of output_blocks that have cross attention + _set_model_patch_replace(work_model, patch_kwargs, ("output", id)) + patch_kwargs["number"] += 1 + _set_model_patch_replace(work_model, patch_kwargs, ("middle", 0)) + else: + for id in [4,5,7,8]: # id of input_blocks that have cross attention + block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth + for index in block_indices: + _set_model_patch_replace(work_model, patch_kwargs, ("input", id, index)) + patch_kwargs["number"] += 1 + for id in range(6): # id of output_blocks that have cross attention + block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth + for index in block_indices: + _set_model_patch_replace(work_model, patch_kwargs, ("output", id, index)) + patch_kwargs["number"] += 1 + for index in range(10): + _set_model_patch_replace(work_model, patch_kwargs, ("middle", 0, index)) + patch_kwargs["number"] += 1 + + return(work_model, { "cond": image_prompt_embeds, "uncond": uncond_image_prompt_embeds }, ) + +class ApplyInstantIDControlNet: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "face_embeds": ("FACE_EMBEDS", ), + "control_net": ("CONTROL_NET", ), + "image_kps": ("IMAGE", ), + "positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, }), + "start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, }), + "end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, }), + }, + "optional": { + "mask": ("MASK",), + } + } + + RETURN_TYPES = ("CONDITIONING", "CONDITIONING",) + RETURN_NAMES = ("positive", "negative", ) + FUNCTION = "apply_controlnet" + CATEGORY = "InstantID" + + def apply_controlnet(self, face_embeds, control_net, image_kps, positive, negative, strength, start_at, end_at, mask=None): + self.device = comfy.model_management.get_torch_device() + + if strength == 0: + return (positive, negative) + + if mask is not None: + mask = mask.to(self.device) + + if mask is not None and len(mask.shape) < 3: + mask = mask.unsqueeze(0) + + image_prompt_embeds = face_embeds['cond'] + uncond_image_prompt_embeds = face_embeds['uncond'] + + cnets = {} + cond_uncond = [] + control_hint = image_kps.movedim(-1,1) + + is_cond = True + for conditioning in [positive, negative]: + c = [] + for t in conditioning: + d = t[1].copy() + + prev_cnet = d.get('control', None) + if prev_cnet in cnets: + c_net = cnets[prev_cnet] + else: + c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_at, end_at)) + c_net.set_previous_controlnet(prev_cnet) + cnets[prev_cnet] = c_net + + d['control'] = c_net + d['control_apply_to_uncond'] = False + d['cross_attn_controlnet'] = image_prompt_embeds.to(comfy.model_management.intermediate_device()) if is_cond else uncond_image_prompt_embeds.to(comfy.model_management.intermediate_device()) + + if mask is not None and is_cond: + d['mask'] = mask + d['set_area_to_bounds'] = False + + n = [t[0], d] + c.append(n) + cond_uncond.append(c) + is_cond = False + + print(cond_uncond[0]) + + return(cond_uncond[0], cond_uncond[1]) + + NODE_CLASS_MAPPINGS = { "InstantIDModelLoader": InstantIDModelLoader, "InstantIDFaceAnalysis": InstantIDFaceAnalysis, "ApplyInstantID": ApplyInstantID, "ApplyInstantIDAdvanced": ApplyInstantIDAdvanced, "FaceKeypointsPreprocessor": FaceKeypointsPreprocessor, + + "InstantIDAttentionPatch": InstantIDAttentionPatch, + "ApplyInstantIDControlNet": ApplyInstantIDControlNet, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -575,4 +761,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ApplyInstantID": "Apply InstantID", "ApplyInstantIDAdvanced": "Apply InstantID Advanced", "FaceKeypointsPreprocessor": "Face Keypoints Preprocessor", + + "InstantIDAttentionPatch": "InstantID Patch Attention", + "ApplyInstantIDControlNet": "InstantID Apply ControlNet", } diff --git a/README.md b/README.md index a4b62f6..4697c06 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ This extension differs from the many already available as it doesn't use *diffus ## Important updates +- **2024/02/26:** Fixed a small but nasty bug. Results will be different and you may need to lower the CFG. + - **2024/02/20:** I refactored the nodes so they are hopefully easier to use. **This is a breaking update**, the previous workflows won't work anymore. ## Basic Workflow @@ -20,7 +22,7 @@ In the `examples` directory you'll find some basic workflows. Watch the video -**:movie_camera: [Introduction to InstantID features](https://youtu.be/wMLiGhogOPE)** +** :movie_camera: [Introduction to InstantID features](https://youtu.be/wMLiGhogOPE)** ## Installation