From f97ff961541c20856f74d29b692b01c4b0eaecda Mon Sep 17 00:00:00 2001 From: Tung Nguyen Date: Fri, 12 Apr 2024 16:51:29 +0700 Subject: [PATCH] upgrade to support new ip_adapter --- __init__.py | 3 + modules/controlnet/advanced.py | 11 +- modules/fooocus/__init__.py | 12 +- modules/ip_adapter_nodes.py | 234 +++++++++++++++------------------ 4 files changed, 117 insertions(+), 143 deletions(-) diff --git a/__init__.py b/__init__.py index 2ddbeb1..7938a3f 100644 --- a/__init__.py +++ b/__init__.py @@ -1,6 +1,9 @@ import sys import folder_paths +if not 'saved_prompts' in folder_paths.folder_names_and_paths: + folder_paths.folder_names_and_paths['saved_prompts'] = ([], set(['.txt'])) + custom_nodes = folder_paths.get_folder_paths("custom_nodes") for dir in custom_nodes: if dir not in sys.path: diff --git a/modules/controlnet/advanced.py b/modules/controlnet/advanced.py index 87b4430..fa164dd 100644 --- a/modules/controlnet/advanced.py +++ b/modules/controlnet/advanced.py @@ -9,16 +9,16 @@ from ..utils import load_module custom_nodes = folder_paths.get_folder_paths("custom_nodes") advanced_cnet_dir_names = ["AdvancedControlNet", "ComfyUI-Advanced-ControlNet"] + def comfy_load_controlnet(module_path: str, **_): return comfy.controlnet.load_controlnet(module_path) + try: module_path = None for custom_node in custom_nodes: - custom_node = ( - custom_node if not os.path.islink(custom_node) else os.readlink(custom_node) - ) + custom_node = custom_node if not os.path.islink(custom_node) else os.readlink(custom_node) for module_dir in advanced_cnet_dir_names: if module_dir in os.listdir(custom_node): module_path = os.path.abspath(os.path.join(custom_node, module_dir)) @@ -27,12 +27,11 @@ try: if module_path is None: raise Exception("Could not find AdvancedControlNet nodes") - module_path = os.path.join(module_path, "control/control.py") - module = load_module(module_path) + module_path = os.path.join(module_path, "adv_control/control.py") + module = load_module(module_path, "adv_control/control") print("Loaded AdvancedControlNet nodes from", module_path) comfy_load_controlnet = getattr(module, "load_controlnet") except Exception as e: print(e) - diff --git a/modules/fooocus/__init__.py b/modules/fooocus/__init__.py index e500a77..e5f2b9f 100644 --- a/modules/fooocus/__init__.py +++ b/modules/fooocus/__init__.py @@ -23,10 +23,10 @@ class KSamplerWithSharpness(KSampler): CATEGORY = "Art Venture/Sampling" def sample(self, *args, sharpness=2.0, **kwargs): - patch.sharpness = sharpness - patch_all() + # patch.sharpness = sharpness + # patch_all() results = super().sample(*args, **kwargs) - unpatch_all() + # unpatch_all() return results @@ -46,10 +46,10 @@ class KSamplerAdvancedWithSharpness(KSamplerAdvanced): CATEGORY = "Art Venture/Sampling" def sample(self, *args, sharpness=2.0, **kwargs): - patch.sharpness = sharpness - patch_all() + # patch.sharpness = sharpness + # patch_all() results = super().sample(*args, **kwargs) - unpatch_all() + # unpatch_all() return results diff --git a/modules/ip_adapter_nodes.py b/modules/ip_adapter_nodes.py index b67a43c..9e5b715 100644 --- a/modules/ip_adapter_nodes.py +++ b/modules/ip_adapter_nodes.py @@ -1,7 +1,6 @@ import os import json import torch -import torchvision.transforms as TT from typing import Dict, Tuple, List from pydantic import BaseModel @@ -37,36 +36,66 @@ try: print("Loaded IPAdapter nodes from", module_path) nodes: Dict = getattr(module, "NODE_CLASS_MAPPINGS") - IPAdapterApply = nodes.get("IPAdapterApply") - IPAdapterApplyEncoded = nodes.get("IPAdapterApplyEncoded") + IPAdapterUnifiedLoader = nodes.get("IPAdapterUnifiedLoader") IPAdapterModelLoader = nodes.get("IPAdapterModelLoader") + IPAdapterApply = nodes.get("IPAdapter") + IPAdapterEncoder = nodes.get("IPAdapterEncoder") + IPAdapterEmbeds = nodes.get("IPAdapterEmbeds") + IPAdapterCombineEmbeds = nodes.get("IPAdapterCombineEmbeds") - # from IPAdapter_Plus - def image_add_noise(image: torch.Tensor, noise: float): - image = image.permute([0, 3, 1, 2]) - torch.manual_seed(0) # use a fixed random for reproducible results - transforms = TT.Compose( - [ - TT.CenterCrop(min(image.shape[2], image.shape[3])), - TT.Resize((224, 224), interpolation=TT.InterpolationMode.BICUBIC, antialias=True), - TT.ElasticTransform(alpha=75.0, sigma=noise * 3.5), # shuffle the image - TT.RandomVerticalFlip(p=1.0), # flip the image to change the geometry even more - TT.RandomHorizontalFlip(p=1.0), - ] - ) - image = transforms(image.cpu()) - image = image.permute([0, 2, 3, 1]) - image = image + ((0.25 * (1 - noise) + 0.05) * torch.randn_like(image)) # add further random noise - return image + loader = IPAdapterModelLoader() + unifyLoader = IPAdapterUnifiedLoader() + apply = IPAdapterApply() + encoder = IPAdapterEncoder() + combiner = IPAdapterCombineEmbeds() + embedder = IPAdapterEmbeds() - def zeroed_hidden_states(clip_vision, batch_size): - image = torch.zeros([batch_size, 224, 224, 3]) - comfy.model_management.load_model_gpu(clip_vision.patcher) - pixel_values = comfy.clip_vision.clip_preprocess(image.to(clip_vision.load_device)).float() - outputs = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2) - # we only need the penultimate hidden states - outputs = outputs[1].to(comfy.model_management.intermediate_device()) - return outputs + WEIGHT_TYPES = [ + "linear", + "ease in", + "ease out", + "ease in-out", + "reverse in-out", + "weak input", + "weak output", + "weak middle", + "strong middle", + "style transfer (SDXL)", + "composition (SDXL)", + ] + + PRESETS = [ + "LIGHT - SD1.5 only (low strength)", + "STANDARD (medium strength)", + "VIT-G (medium strength)", + "PLUS (high strength)", + "PLUS FACE (portraits)", + "FULL FACE - SD1.5 only (portraits stronger)", + ] + + class AV_IPAdapterPipeline: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "ip_adapter_name": (folder_paths.get_filename_list("ipadapter"),), + "clip_name": (folder_paths.get_filename_list("clip_vision"),), + } + } + + RETURN_TYPES = ("IPADAPTER",) + RETURN_NAMES = "pipeline" + CATEGORY = "Art Venture/IP Adapter" + FUNCTION = "load_ip_adapter" + + def load_ip_adapter(ip_adapter_name, clip_name): + ip_adapter = loader.load_ipadapter_model(ip_adapter_name)[0] + + clip_path = folder_paths.get_full_path("clip_vision", clip_name) + clip_vision = comfy.clip_vision.load(clip_path) + + pipeline = {"ipadapter": {"model": ip_adapter}, "clipvision": {"model": clip_vision}} + return pipeline class AV_IPAdapter(IPAdapterModelLoader, IPAdapterApply): @classmethod @@ -83,13 +112,19 @@ try: "optional": { "ip_adapter_opt": ("IPADAPTER",), "clip_vision_opt": ("CLIP_VISION",), + "attn_mask": ("MASK",), "start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), "end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "weight_type": ( + ["standard", "prompt is more important", "style transfer (SDXL only)"], + {"default": "standard"}, + ), "enabled": ("BOOLEAN", {"default": True}), }, } RETURN_TYPES = ("MODEL", "IPADAPTER", "CLIP_VISION") + RETURN_NAMES = ("model", "pipeline", "clip_vision") CATEGORY = "Art Venture/IP Adapter" FUNCTION = "apply_ip_adapter" @@ -110,22 +145,27 @@ try: return (model, None, None) if ip_adapter_opt: - ip_adapter = ip_adapter_opt + if "ipadapter" in ip_adapter_opt: + ip_adapter = ip_adapter_opt["ipadapter"]["model"] + else: + ip_adapter = ip_adapter_opt else: assert ip_adapter_name != "None", "IP Adapter name must be specified" - ip_adapter = super().load_ipadapter_model(ip_adapter_name)[0] + ip_adapter = loader.load_ipadapter_model(ip_adapter_name)[0] if clip_vision_opt: clip_vision = clip_vision_opt + elif "clipvision" in ip_adapter_opt: + clip_vision = ip_adapter_opt["clipvision"]["model"] else: assert clip_name != "None", "Clip vision name must be specified" clip_path = folder_paths.get_full_path("clip_vision", clip_name) clip_vision = comfy.clip_vision.load(clip_path) - res: Tuple = super().apply_ipadapter( - ip_adapter, model, weight, clip_vision=clip_vision, image=image, noise=noise, **kwargs - ) - res += (ip_adapter, clip_vision) + pipeline = {"ipadapter": {"model": ip_adapter}, "clipvision": {"model": clip_vision}} + + res: Tuple = apply.apply_ipadapter(model, pipeline, image=image, weight=weight, **kwargs) + res += (pipeline, clip_vision) return res @@ -136,14 +176,13 @@ try: class IPAdapterData(BaseModel): images: List[IPAdapterImage] - class AV_IPAdapterEncodeFromJson: + class AV_StyleApply: @classmethod def INPUT_TYPES(cls): return { "required": { - "clip_name": (["None"] + folder_paths.get_filename_list("clip_vision"),), - "ipadapter_plus": ("BOOLEAN", {"default": False}), - "noise": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "model": ("MODEL",), + "preset": (PRESETS,), "data": ( "STRING", { @@ -152,133 +191,66 @@ try: "dynamicPrompts": False, }, ), + "weight": ("FLOAT", {"default": 0.5, "min": -1, "max": 3, "step": 0.05}), + "weight_type": (WEIGHT_TYPES,), + "start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}), }, "optional": { - "clip_vision_opt": ("CLIP_VISION",), + "mask": ("MASK",), + "enabled": ("BOOLEAN", {"default": True}), }, } - RETURN_TYPES = ("EMBEDS", "IMAGE", "CLIP_VISION", "BOOLEAN") - RETURN_NAMES = ("embeds", "image", "clip_vision", "has_data") - CATEGORY = "Art Venture/IP Adapter" - FUNCTION = "encode" + RETURN_TYPES = ("MODEL", "IMAGE") + CATEGORY = "Art Venture/Style" + FUNCTION = "apply_style" - def encode(self, clip_name: str, ipadapter_plus: bool, noise: float, data: str, clip_vision_opt=None): + def apply_style(self, model, preset: str, data: str, mask=None, enabled=True, **kwargs): data = json.loads(data or "[]") data: IPAdapterData = IPAdapterData(images=data) # validate if len(data.images) == 0: images = torch.zeros((1, 64, 64, 3)) - return (None, images, None, False) + return (model, images) + + (model, pipeline) = unifyLoader.load_models(model, preset) urls = [image.url for image in data.images] pils, _ = load_images_from_url(urls) + embeds_avg = None + neg_embeds_avg = None images = [] - weights = [] for i, pil in enumerate(pils): weight = data.images[i].weight - weight *= 0.1 + (weight - 0.1) - weight = 1.19e-05 if weight <= 1.19e-05 else weight - image = pil2tensor(pil) if i > 0 and image.shape[1:] != images[0].shape[1:]: image = comfy.utils.common_upscale( image.movedim(-1, 1), images[0].shape[2], images[0].shape[1], "bilinear", "center" ).movedim(1, -1) - images.append(image) - weights.append(weight) - if clip_vision_opt: - clip_vision = clip_vision_opt - else: - assert clip_name != "None", "Clip vision name must be specified" - clip_path = folder_paths.get_full_path("clip_vision", clip_name) - clip_vision = comfy.clip_vision.load(clip_path) + embeds = encoder.encode(pipeline, image, weight, mask=mask) + if embeds_avg is None: + embeds_avg = embeds[0] + neg_embeds_avg = embeds[1] + else: + embeds_avg = combiner.batch(embeds_avg, method="average", embed2=embeds[0])[0] + neg_embeds_avg = combiner.batch(neg_embeds_avg, method="average", embed2=embeds[1])[0] images = torch.cat(images) - clip_embed = clip_vision.encode_image(images) - neg_image = image_add_noise(images, noise) if noise > 0 else None - if ipadapter_plus: - clip_embed = clip_embed.penultimate_hidden_states - if noise > 0: - clip_embed_zeroed = clip_vision.encode_image(neg_image).penultimate_hidden_states - else: - clip_embed_zeroed = zeroed_hidden_states(clip_vision, images.shape[0]) - else: - clip_embed = clip_embed.image_embeds - if noise > 0: - clip_embed_zeroed = clip_vision.encode_image(neg_image).image_embeds - else: - clip_embed_zeroed = torch.zeros_like(clip_embed) + model = embedder.apply_ipadapter(model, pipeline, embeds_avg, neg_embed=neg_embeds_avg, **kwargs)[0] - if any(e != 1.0 for e in weights): - weights = ( - torch.tensor(weights).unsqueeze(-1) - if not ipadapter_plus - else torch.tensor(weights).unsqueeze(-1).unsqueeze(-1) - ) - clip_embed = clip_embed * weights - - embeds = torch.stack((clip_embed, clip_embed_zeroed)) - - return (embeds, images, clip_vision, True) - - class AV_IPAdapterApplyEncoded(IPAdapterModelLoader, IPAdapterApplyEncoded): - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "ip_adapter_name": (["None"] + folder_paths.get_filename_list("ipadapter"),), - "embeds": ("EMBEDS",), - "model": ("MODEL",), - "weight": ("FLOAT", {"default": 1.0, "min": -1, "max": 3, "step": 0.05}), - "weight_type": (["original", "linear", "channel penalty"],), - }, - "optional": { - "ip_adapter_opt": ("IPADAPTER",), - "attn_mask": ("MASK",), - "start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), - "end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}), - "enabled": ("BOOLEAN", {"default": True}), - }, - } - - RETURN_TYPES = ("MODEL", "IPADAPTER") - CATEGORY = "Art Venture/IP Adapter" - FUNCTION = "apply_ip_adapter" - - def apply_ip_adapter(self, ip_adapter_name, model, ip_adapter_opt=None, enabled=True, **kwargs): - if not enabled: - return (model, None) - - if ip_adapter_opt: - ip_adapter = ip_adapter_opt - else: - assert ip_adapter_name != "None", "IP Adapter name must be specified" - ip_adapter = super().load_ipadapter_model(ip_adapter_name)[0] - - res: Tuple = super().apply_ipadapter(ip_adapter, model, **kwargs) - res += (ip_adapter,) - - return res + return (model, images) NODE_CLASS_MAPPINGS.update( - { - "AV_IPAdapter": AV_IPAdapter, - "AV_IPAdapterEncodeFromJson": AV_IPAdapterEncodeFromJson, - "AV_IPAdapterApplyEncoded": AV_IPAdapterApplyEncoded, - } + {"AV_IPAdapter": AV_IPAdapter, "AV_IPAdapterPipeline": AV_IPAdapterPipeline, "AV_StyleApply": AV_StyleApply} ) NODE_DISPLAY_NAME_MAPPINGS.update( - { - "AV_IPAdapter": "IP Adapter Apply", - "AV_IPAdapterEncodeFromJson": "IP Adapter Encoder", - "AV_IPAdapterApplyEncoded": "IP Adapter Apply Encoded", - } + {"AV_IPAdapter": "IP Adapter Apply", "AV_IPAdapter": "IP Adapter Pipeline", "AV_StyleApply": "AV Style Apply"} ) except Exception as e: