nodes/sipherxyz-comfyui-art-venture
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

upgrade to support new ip_adapter

Browse Source
This commit is contained in:
Tung Nguyen
2024-04-12 16:51:29 +07:00
parent ccc6203062
commit f97ff96154
4 changed files with 117 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__init__.py
+3
View File
@@ -1,6 +1,9 @@
import sys
import folder_paths
if not 'saved_prompts' in folder_paths.folder_names_and_paths:
folder_paths.folder_names_and_paths['saved_prompts'] = ([], set(['.txt']))
custom_nodes = folder_paths.get_folder_paths("custom_nodes")
for dir in custom_nodes:
if dir not in sys.path:
modules/controlnet/advanced.py
+5 -6
View File
@@ -9,16 +9,16 @@ from ..utils import load_module
custom_nodes = folder_paths.get_folder_paths("custom_nodes")
advanced_cnet_dir_names = ["AdvancedControlNet", "ComfyUI-Advanced-ControlNet"]
def comfy_load_controlnet(module_path: str, **_):
return comfy.controlnet.load_controlnet(module_path)
try:
module_path = None
for custom_node in custom_nodes:
custom_node = (
custom_node if not os.path.islink(custom_node) else os.readlink(custom_node)
)
custom_node = custom_node if not os.path.islink(custom_node) else os.readlink(custom_node)
for module_dir in advanced_cnet_dir_names:
if module_dir in os.listdir(custom_node):
module_path = os.path.abspath(os.path.join(custom_node, module_dir))
@@ -27,12 +27,11 @@ try:
if module_path is None:
raise Exception("Could not find AdvancedControlNet nodes")
module_path = os.path.join(module_path, "control/control.py")
module = load_module(module_path)
module_path = os.path.join(module_path, "adv_control/control.py")
module = load_module(module_path, "adv_control/control")
print("Loaded AdvancedControlNet nodes from", module_path)
comfy_load_controlnet = getattr(module, "load_controlnet")
except Exception as e:
print(e)
modules/fooocus/__init__.py
+6 -6
View File
@@ -23,10 +23,10 @@ class KSamplerWithSharpness(KSampler):
CATEGORY = "Art Venture/Sampling"
def sample(self, *args, sharpness=2.0, **kwargs):
patch.sharpness = sharpness
patch_all()
# patch.sharpness = sharpness
# patch_all()
results = super().sample(*args, **kwargs)
unpatch_all()
# unpatch_all()
return results
@@ -46,10 +46,10 @@ class KSamplerAdvancedWithSharpness(KSamplerAdvanced):
CATEGORY = "Art Venture/Sampling"
def sample(self, *args, sharpness=2.0, **kwargs):
patch.sharpness = sharpness
patch_all()
# patch.sharpness = sharpness
# patch_all()
results = super().sample(*args, **kwargs)
unpatch_all()
# unpatch_all()
return results
modules/ip_adapter_nodes.py
+103 -131
View File
@@ -1,7 +1,6 @@
import os
import json
import torch
import torchvision.transforms as TT
from typing import Dict, Tuple, List
from pydantic import BaseModel
@@ -37,36 +36,66 @@ try:
print("Loaded IPAdapter nodes from", module_path)
nodes: Dict = getattr(module, "NODE_CLASS_MAPPINGS")
IPAdapterApply = nodes.get("IPAdapterApply")
IPAdapterApplyEncoded = nodes.get("IPAdapterApplyEncoded")
IPAdapterUnifiedLoader = nodes.get("IPAdapterUnifiedLoader")
IPAdapterModelLoader = nodes.get("IPAdapterModelLoader")
IPAdapterApply = nodes.get("IPAdapter")
IPAdapterEncoder = nodes.get("IPAdapterEncoder")
IPAdapterEmbeds = nodes.get("IPAdapterEmbeds")
IPAdapterCombineEmbeds = nodes.get("IPAdapterCombineEmbeds")
# from IPAdapter_Plus
def image_add_noise(image: torch.Tensor, noise: float):
image = image.permute([0, 3, 1, 2])
torch.manual_seed(0) # use a fixed random for reproducible results
transforms = TT.Compose(
[
TT.CenterCrop(min(image.shape[2], image.shape[3])),
TT.Resize((224, 224), interpolation=TT.InterpolationMode.BICUBIC, antialias=True),
TT.ElasticTransform(alpha=75.0, sigma=noise * 3.5), # shuffle the image
TT.RandomVerticalFlip(p=1.0), # flip the image to change the geometry even more
TT.RandomHorizontalFlip(p=1.0),
]
)
image = transforms(image.cpu())
image = image.permute([0, 2, 3, 1])
image = image + ((0.25 * (1 - noise) + 0.05) * torch.randn_like(image)) # add further random noise
return image
loader = IPAdapterModelLoader()
unifyLoader = IPAdapterUnifiedLoader()
apply = IPAdapterApply()
encoder = IPAdapterEncoder()
combiner = IPAdapterCombineEmbeds()
embedder = IPAdapterEmbeds()
def zeroed_hidden_states(clip_vision, batch_size):
image = torch.zeros([batch_size, 224, 224, 3])
comfy.model_management.load_model_gpu(clip_vision.patcher)
pixel_values = comfy.clip_vision.clip_preprocess(image.to(clip_vision.load_device)).float()
outputs = clip_vision.model(pixel_values=pixel_values, intermediate_output=-2)
# we only need the penultimate hidden states
outputs = outputs[1].to(comfy.model_management.intermediate_device())
return outputs
WEIGHT_TYPES = [
"linear",
"ease in",
"ease out",
"ease in-out",
"reverse in-out",
"weak input",
"weak output",
"weak middle",
"strong middle",
"style transfer (SDXL)",
"composition (SDXL)",
]
PRESETS = [
"LIGHT - SD1.5 only (low strength)",
"STANDARD (medium strength)",
"VIT-G (medium strength)",
"PLUS (high strength)",
"PLUS FACE (portraits)",
"FULL FACE - SD1.5 only (portraits stronger)",
]
class AV_IPAdapterPipeline:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"ip_adapter_name": (folder_paths.get_filename_list("ipadapter"),),
"clip_name": (folder_paths.get_filename_list("clip_vision"),),
}
}
RETURN_TYPES = ("IPADAPTER",)
RETURN_NAMES = "pipeline"
CATEGORY = "Art Venture/IP Adapter"
FUNCTION = "load_ip_adapter"
def load_ip_adapter(ip_adapter_name, clip_name):
ip_adapter = loader.load_ipadapter_model(ip_adapter_name)[0]
clip_path = folder_paths.get_full_path("clip_vision", clip_name)
clip_vision = comfy.clip_vision.load(clip_path)
pipeline = {"ipadapter": {"model": ip_adapter}, "clipvision": {"model": clip_vision}}
return pipeline
class AV_IPAdapter(IPAdapterModelLoader, IPAdapterApply):
@classmethod
@@ -83,13 +112,19 @@ try:
"optional": {
"ip_adapter_opt": ("IPADAPTER",),
"clip_vision_opt": ("CLIP_VISION",),
"attn_mask": ("MASK",),
"start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),
"end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}),
"weight_type": (
["standard", "prompt is more important", "style transfer (SDXL only)"],
{"default": "standard"},
),
"enabled": ("BOOLEAN", {"default": True}),
},
}
RETURN_TYPES = ("MODEL", "IPADAPTER", "CLIP_VISION")
RETURN_NAMES = ("model", "pipeline", "clip_vision")
CATEGORY = "Art Venture/IP Adapter"
FUNCTION = "apply_ip_adapter"
@@ -110,22 +145,27 @@ try:
return (model, None, None)
if ip_adapter_opt:
ip_adapter = ip_adapter_opt
if "ipadapter" in ip_adapter_opt:
ip_adapter = ip_adapter_opt["ipadapter"]["model"]
else:
ip_adapter = ip_adapter_opt
else:
assert ip_adapter_name != "None", "IP Adapter name must be specified"
ip_adapter = super().load_ipadapter_model(ip_adapter_name)[0]
ip_adapter = loader.load_ipadapter_model(ip_adapter_name)[0]
if clip_vision_opt:
clip_vision = clip_vision_opt
elif "clipvision" in ip_adapter_opt:
clip_vision = ip_adapter_opt["clipvision"]["model"]
else:
assert clip_name != "None", "Clip vision name must be specified"
clip_path = folder_paths.get_full_path("clip_vision", clip_name)
clip_vision = comfy.clip_vision.load(clip_path)
res: Tuple = super().apply_ipadapter(
ip_adapter, model, weight, clip_vision=clip_vision, image=image, noise=noise, **kwargs
)
res += (ip_adapter, clip_vision)
pipeline = {"ipadapter": {"model": ip_adapter}, "clipvision": {"model": clip_vision}}
res: Tuple = apply.apply_ipadapter(model, pipeline, image=image, weight=weight, **kwargs)
res += (pipeline, clip_vision)
return res
@@ -136,14 +176,13 @@ try:
class IPAdapterData(BaseModel):
images: List[IPAdapterImage]
class AV_IPAdapterEncodeFromJson:
class AV_StyleApply:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"clip_name": (["None"] + folder_paths.get_filename_list("clip_vision"),),
"ipadapter_plus": ("BOOLEAN", {"default": False}),
"noise": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"model": ("MODEL",),
"preset": (PRESETS,),
"data": (
"STRING",
{
@@ -152,133 +191,66 @@ try:
"dynamicPrompts": False,
},
),
"weight": ("FLOAT", {"default": 0.5, "min": -1, "max": 3, "step": 0.05}),
"weight_type": (WEIGHT_TYPES,),
"start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),
"end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}),
},
"optional": {
"clip_vision_opt": ("CLIP_VISION",),
"mask": ("MASK",),
"enabled": ("BOOLEAN", {"default": True}),
},
}
RETURN_TYPES = ("EMBEDS", "IMAGE", "CLIP_VISION", "BOOLEAN")
RETURN_NAMES = ("embeds", "image", "clip_vision", "has_data")
CATEGORY = "Art Venture/IP Adapter"
FUNCTION = "encode"
RETURN_TYPES = ("MODEL", "IMAGE")
CATEGORY = "Art Venture/Style"
FUNCTION = "apply_style"
def encode(self, clip_name: str, ipadapter_plus: bool, noise: float, data: str, clip_vision_opt=None):
def apply_style(self, model, preset: str, data: str, mask=None, enabled=True, **kwargs):
data = json.loads(data or "[]")
data: IPAdapterData = IPAdapterData(images=data) # validate
if len(data.images) == 0:
images = torch.zeros((1, 64, 64, 3))
return (None, images, None, False)
return (model, images)
(model, pipeline) = unifyLoader.load_models(model, preset)
urls = [image.url for image in data.images]
pils, _ = load_images_from_url(urls)
embeds_avg = None
neg_embeds_avg = None
images = []
weights = []
for i, pil in enumerate(pils):
weight = data.images[i].weight
weight *= 0.1 + (weight - 0.1)
weight = 1.19e-05 if weight <= 1.19e-05 else weight
image = pil2tensor(pil)
if i > 0 and image.shape[1:] != images[0].shape[1:]:
image = comfy.utils.common_upscale(
image.movedim(-1, 1), images[0].shape[2], images[0].shape[1], "bilinear", "center"
).movedim(1, -1)
images.append(image)
weights.append(weight)
if clip_vision_opt:
clip_vision = clip_vision_opt
else:
assert clip_name != "None", "Clip vision name must be specified"
clip_path = folder_paths.get_full_path("clip_vision", clip_name)
clip_vision = comfy.clip_vision.load(clip_path)
embeds = encoder.encode(pipeline, image, weight, mask=mask)
if embeds_avg is None:
embeds_avg = embeds[0]
neg_embeds_avg = embeds[1]
else:
embeds_avg = combiner.batch(embeds_avg, method="average", embed2=embeds[0])[0]
neg_embeds_avg = combiner.batch(neg_embeds_avg, method="average", embed2=embeds[1])[0]
images = torch.cat(images)
clip_embed = clip_vision.encode_image(images)
neg_image = image_add_noise(images, noise) if noise > 0 else None
if ipadapter_plus:
clip_embed = clip_embed.penultimate_hidden_states
if noise > 0:
clip_embed_zeroed = clip_vision.encode_image(neg_image).penultimate_hidden_states
else:
clip_embed_zeroed = zeroed_hidden_states(clip_vision, images.shape[0])
else:
clip_embed = clip_embed.image_embeds
if noise > 0:
clip_embed_zeroed = clip_vision.encode_image(neg_image).image_embeds
else:
clip_embed_zeroed = torch.zeros_like(clip_embed)
model = embedder.apply_ipadapter(model, pipeline, embeds_avg, neg_embed=neg_embeds_avg, **kwargs)[0]
if any(e != 1.0 for e in weights):
weights = (
torch.tensor(weights).unsqueeze(-1)
if not ipadapter_plus
else torch.tensor(weights).unsqueeze(-1).unsqueeze(-1)
)
clip_embed = clip_embed * weights
embeds = torch.stack((clip_embed, clip_embed_zeroed))
return (embeds, images, clip_vision, True)
class AV_IPAdapterApplyEncoded(IPAdapterModelLoader, IPAdapterApplyEncoded):
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"ip_adapter_name": (["None"] + folder_paths.get_filename_list("ipadapter"),),
"embeds": ("EMBEDS",),
"model": ("MODEL",),
"weight": ("FLOAT", {"default": 1.0, "min": -1, "max": 3, "step": 0.05}),
"weight_type": (["original", "linear", "channel penalty"],),
},
"optional": {
"ip_adapter_opt": ("IPADAPTER",),
"attn_mask": ("MASK",),
"start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),
"end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}),
"enabled": ("BOOLEAN", {"default": True}),
},
}
RETURN_TYPES = ("MODEL", "IPADAPTER")
CATEGORY = "Art Venture/IP Adapter"
FUNCTION = "apply_ip_adapter"
def apply_ip_adapter(self, ip_adapter_name, model, ip_adapter_opt=None, enabled=True, **kwargs):
if not enabled:
return (model, None)
if ip_adapter_opt:
ip_adapter = ip_adapter_opt
else:
assert ip_adapter_name != "None", "IP Adapter name must be specified"
ip_adapter = super().load_ipadapter_model(ip_adapter_name)[0]
res: Tuple = super().apply_ipadapter(ip_adapter, model, **kwargs)
res += (ip_adapter,)
return res
return (model, images)
NODE_CLASS_MAPPINGS.update(
{
"AV_IPAdapter": AV_IPAdapter,
"AV_IPAdapterEncodeFromJson": AV_IPAdapterEncodeFromJson,
"AV_IPAdapterApplyEncoded": AV_IPAdapterApplyEncoded,
}
{"AV_IPAdapter": AV_IPAdapter, "AV_IPAdapterPipeline": AV_IPAdapterPipeline, "AV_StyleApply": AV_StyleApply}
)
NODE_DISPLAY_NAME_MAPPINGS.update(
{
"AV_IPAdapter": "IP Adapter Apply",
"AV_IPAdapterEncodeFromJson": "IP Adapter Encoder",
"AV_IPAdapterApplyEncoded": "IP Adapter Apply Encoded",
}
{"AV_IPAdapter": "IP Adapter Apply", "AV_IPAdapter": "IP Adapter Pipeline", "AV_StyleApply": "AV Style Apply"}
)
except Exception as e: