diff --git a/IPAdapterPlus.py b/IPAdapterPlus.py index e73c03a..af5d090 100644 --- a/IPAdapterPlus.py +++ b/IPAdapterPlus.py @@ -8,6 +8,7 @@ from comfy.clip_vision import ClipVisionModel from torch import nn import torch.nn.functional as F +import torchvision.transforms as TT from .resampler import Resampler @@ -17,8 +18,6 @@ MODELS_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "models") SD_V12_CHANNELS = [320] * 4 + [640] * 4 + [1280] * 4 + [1280] * 6 + [640] * 6 + [320] * 6 + [1280] * 2 SD_XL_CHANNELS = [640] * 8 + [1280] * 40 + [1280] * 60 + [640] * 12 + [1280] * 20 -MAX_NOISE = 5.0 - def get_filename_list(path): return [f for f in os.listdir(path) if f.endswith('.bin')] @@ -77,20 +76,38 @@ def attention(q, k, v, extra_options): out = out.transpose(1, 2).reshape(b, -1, extra_options["n_heads"] * extra_options["dim_head"]) return out -def encode_zero_image(clip_vision, noise=0): - image = torch.zeros( [1, 3, 224, 224] ) +# TODO: still have to find the best way to add noise to the uncod image +def image_add_noise(image, noise): + image = image.permute([0,3,1,2]) + transforms = TT.Compose([ + TT.CenterCrop(min(image.shape[2], image.shape[3])), + TT.Resize((28, 28), antialias=True), + #TT.Resize((224, 224), interpolation=TT.InterpolationMode.NEAREST, antialias=True), + #TT.ColorJitter(contrast=(0.75,0.75)), + #TT.ElasticTransform(alpha=75.0, sigma=noise+.25), + TT.RandomVerticalFlip(p=1.0), + TT.RandomHorizontalFlip(p=1.0), + ]) + torch.manual_seed(0) # use a fixed random for reproducible results + image = transforms(image) + image = image.permute([0,2,3,1]) + image = image + ((1.9-1.9*noise) * torch.randn_like(image) + 0.1) + return image + +def encode_neg_image(clip_vision, image=None, noise=0): + if noise > 0 and image is not None: + image = image_add_noise(image, noise) + + if image is None: + image = torch.zeros( [1, 3, 224, 224] ) + img = torch.clip((255. * image), 0, 255).round().int() img = list(map(lambda a: a, img)) inputs = clip_vision.processor(images=img, return_tensors="pt") comfy.model_management.load_model_gpu(clip_vision.patcher) pixel_values = inputs['pixel_values'].to(clip_vision.load_device) - if noise > 0: - min_noise = -noise - max_noise = noise - torch.manual_seed(0) # fixed seed for reproducible results - pixel_values = (max_noise - min_noise) * torch.rand_like(pixel_values) + min_noise - else: + if noise == 0: pixel_values = torch.zeros_like(pixel_values) if clip_vision.dtype != torch.float32: @@ -116,17 +133,16 @@ class IPAdapter(nn.Module): def __init__(self, ipadapter_model, cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4): super().__init__() - self.ipadapter_model = ipadapter_model self.clip_embeddings_dim = clip_embeddings_dim - self.cross_attention_dim = self.ipadapter_model["ip_adapter"]["1.to_k_ip.weight"].shape[1] + self.cross_attention_dim = ipadapter_model["ip_adapter"]["1.to_k_ip.weight"].shape[1] self.clip_extra_context_tokens = clip_extra_context_tokens self.image_proj_model = self.init_proj() - self.image_proj_model.load_state_dict(self.ipadapter_model["image_proj"]) + self.image_proj_model.load_state_dict(ipadapter_model["image_proj"]) self.ip_layers = To_KV(cross_attention_dim) - self.ip_layers.load_state_dict(self.ipadapter_model["ip_adapter"]) - + self.ip_layers.load_state_dict(ipadapter_model["ip_adapter"]) + def init_proj(self): image_proj_model = ImageProjModel( cross_attention_dim=self.cross_attention_dim, @@ -192,34 +208,11 @@ class CrossAttentionPatch: ip_v = ipadapter.ip_layers.to_kvs[self.number*2+1](uncond_cond) ip_out = attention(q, ip_k, ip_v, extra_options) - - if mask is not None: - mask_size = mask.shape[0] * mask.shape[1] - down_sample_rate = int((mask_size // 64 // out.shape[1]) ** (1/2)) - mask_downsample = torch.nn.functional.interpolate(mask.unsqueeze(0).unsqueeze(0), scale_factor= 1/8/down_sample_rate, mode="nearest").squeeze(0) - mask_downsample = mask_downsample.view(1,-1, 1).repeat(out.shape[0], 1, out.shape[2]) - ip_out = ip_out * mask_downsample out = out + ip_out * weight return out.to(dtype=org_dtype) -class IPAdapterCLIPVisionEncode: - @classmethod - def INPUT_TYPES(s): - return {"required": { "clip_vision": ("CLIP_VISION",), - "image": ("IMAGE",) - }} - RETURN_TYPES = ("CLIP_VISION_OUTPUT",) - FUNCTION = "encode" - - CATEGORY = "ipadapter" - - def encode(self, clip_vision, image): - output = clip_vision.encode_image(image) - - return ((output, clip_vision), ) - class IPAdapterModelLoader: @classmethod def INPUT_TYPES(s): @@ -246,63 +239,44 @@ class IPAdapterApply: def INPUT_TYPES(s): return { "required": { - "model": ("MODEL", ), "ipadapter": ("IPADAPTER", ), - "clip_vision_output": ("CLIP_VISION_OUTPUT", ), + "clip_vision": ("CLIP_VISION",), + "image": ("IMAGE",), + "model": ("MODEL", ), "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.1 }), + "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }) }, - "hidden": { - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.05 }), - }, - "optional": { - "mask": ("MASK",), - } } - RETURN_TYPES = ("MODEL",) + RETURN_TYPES = ("MODEL", ) FUNCTION = "apply_ipadapter" CATEGORY = "ipadapter" uncond_hidden_states = None - def apply_ipadapter(self, model, ipadapter, clip_vision_output, weight, noise=0, mask=None): + def apply_ipadapter(self, ipadapter, clip_vision, image, model, weight, noise): self.dtype = model.model.diffusion_model.dtype self.device = comfy.model_management.get_torch_device() self.weight = weight - # check if we are using the 'CLIP Vision Encode IPAdapter' node - self.is_ipadapter_encode = type(clip_vision_output) is tuple - if self.is_ipadapter_encode: - clip_embed, clip_vision_model = clip_vision_output - else: - clip_embed = clip_vision_output.image_embeds + cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1] + self.is_sdxl = cross_attention_dim == 2048 + clip_embed = clip_vision.encode_image(image) self.is_plus = "latents" in ipadapter["image_proj"] if self.is_plus: - # IPAdapter Plus requires the IPAdapter Encoder - if not self.is_ipadapter_encode: - raise Exception("IPAdapter Plus requires 'CLIP Vision Encode (IPAdapter)' node") - - clip_embed = clip_embed.penultimate_hidden_states - clip_embed_zeroed = encode_zero_image(clip_vision_model, noise).penultimate_hidden_states - clip_extra_context_tokens = 16 + clip_embed = clip_embed.penultimate_hidden_states + clip_embed_zeroed = encode_neg_image(clip_vision, image=image, noise=noise).penultimate_hidden_states else: - if self.is_ipadapter_encode: - clip_embed = clip_embed.image_embeds - + clip_extra_context_tokens = 4 + clip_embed = clip_embed.image_embeds if noise > 0: - noise = MAX_NOISE * noise - clip_embed_zeroed = encode_zero_image(clip_vision_model, noise).image_embeds + clip_embed_zeroed = encode_neg_image(clip_vision, image=image, noise=noise).image_embeds else: clip_embed_zeroed = torch.zeros_like(clip_embed) - clip_extra_context_tokens = 4 - clip_embeddings_dim = clip_embed.shape[-1] - cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1] - - self.is_sdxl = cross_attention_dim == 2048 IPA = IPAdapterPlus if self.is_plus else IPAdapter @@ -328,7 +302,6 @@ class IPAdapterApply: "dtype": self.dtype, "cond": image_prompt_embeds, "uncond": uncond_image_prompt_embeds, - "mask": mask if mask is None else mask.to(self.device, dtype=self.dtype) } if not self.is_sdxl: @@ -354,16 +327,14 @@ class IPAdapterApply: set_model_patch_replace(work_model, patch_kwargs, ("midlle", 0, index)) patch_kwargs["number"] += 1 - return (work_model,) + return (work_model, ) NODE_CLASS_MAPPINGS = { "IPAdapterModelLoader": IPAdapterModelLoader, "IPAdapterApply": IPAdapterApply, - "IPAdapterCLIPVisionEncode": IPAdapterCLIPVisionEncode, } NODE_DISPLAY_NAME_MAPPINGS = { "IPAdapterModelLoader": "Load IPAdapter Model", "IPAdapterApply": "Apply IPAdapter", - "IPAdapterCLIPVisionEncode": "CLIP Vision Encode (IPAdapter)" } \ No newline at end of file diff --git a/IPAdapter_workflow-sdxl_vit-h.json b/IPAdapter_workflow-sdxl_vit-h.json new file mode 100644 index 0000000..f448e74 --- /dev/null +++ b/IPAdapter_workflow-sdxl_vit-h.json @@ -0,0 +1,597 @@ +{ + "last_node_id": 12, + "last_link_id": 13, + "nodes": [ + { + "id": 8, + "type": "CLIPTextEncode", + "pos": [ + 650, + 420 + ], + "size": [ + 210, + 120 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 6 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, horror" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 650, + 250 + ], + "size": [ + 210, + 120 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 8 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "beautiful renaissance girl, detailed" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 11, + "type": "VAEDecode", + "pos": [ + 1300, + 170 + ], + "size": [ + 140, + 50 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 12, + "type": "SaveImage", + "pos": [ + 1300, + 270 + ], + "size": [ + 400, + 450 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 13 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 6, + "type": "LoadImage", + "pos": [ + 40, + 60 + ], + "size": [ + 220, + 320 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 3 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "venere.jpg", + "image" + ] + }, + { + "id": 3, + "type": "IPAdapterModelLoader", + "pos": [ + 290, + 60 + ], + "size": [ + 300, + 60 + ], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 1 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter_sdxl_vit-h.bin" + ] + }, + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 290, + 280 + ], + "size": [ + 300, + 100 + ], + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 4 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 5, + 6 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd_xl_base_1.0.safetensors" + ] + }, + { + "id": 2, + "type": "VAELoader", + "pos": [ + 940, + 480 + ], + "size": [ + 300, + 60 + ], + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 12 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "sdxl_vae.safetensors" + ] + }, + { + "id": 10, + "type": "EmptyLatentImage", + "pos": [ + 650, + 590 + ], + "size": [ + 210, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 10 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 4, + "type": "CLIPVisionLoader", + "pos": [ + 290, + 170 + ], + "size": [ + 300, + 60 + ], + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 2 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IP-Adapter_sd15_pytorch_model.bin" + ] + }, + { + "id": 5, + "type": "IPAdapterApply", + "pos": [ + 650, + 60 + ], + "size": [ + 210, + 140 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 1 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 2 + }, + { + "name": "image", + "type": "IMAGE", + "link": 3 + }, + { + "name": "model", + "type": "MODEL", + "link": 4 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterApply" + }, + "widgets_values": [ + 0.8, + 0 + ] + }, + { + "id": 9, + "type": "KSampler", + "pos": [ + 930, + 170 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 7 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 8 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 9 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 10 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 11 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 0, + "fixed", + 25, + 5, + "ddim", + "ddim_uniform", + 1 + ] + } + ], + "links": [ + [ + 1, + 3, + 0, + 5, + 0, + "IPADAPTER" + ], + [ + 2, + 4, + 0, + 5, + 1, + "CLIP_VISION" + ], + [ + 3, + 6, + 0, + 5, + 2, + "IMAGE" + ], + [ + 4, + 1, + 0, + 5, + 3, + "MODEL" + ], + [ + 5, + 1, + 1, + 7, + 0, + "CLIP" + ], + [ + 6, + 1, + 1, + 8, + 0, + "CLIP" + ], + [ + 7, + 5, + 0, + 9, + 0, + "MODEL" + ], + [ + 8, + 7, + 0, + 9, + 1, + "CONDITIONING" + ], + [ + 9, + 8, + 0, + 9, + 2, + "CONDITIONING" + ], + [ + 10, + 10, + 0, + 9, + 3, + "LATENT" + ], + [ + 11, + 9, + 0, + 11, + 0, + "LATENT" + ], + [ + 12, + 2, + 0, + 11, + 1, + "VAE" + ], + [ + 13, + 11, + 0, + 12, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/IPAdapter_workflow.json b/IPAdapter_workflow.json index eeaafae..9ab3fd6 100644 --- a/IPAdapter_workflow.json +++ b/IPAdapter_workflow.json @@ -1,20 +1,166 @@ { - "last_node_id": 33, - "last_link_id": 70, + "last_node_id": 12, + "last_link_id": 13, "nodes": [ + { + "id": 4, + "type": "CLIPVisionLoader", + "pos": [ + 290, + 170 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 2 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IP-Adapter_sd15_pytorch_model.bin" + ] + }, + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 290, + 280 + ], + "size": { + "0": 300, + "1": 100 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 4 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 5, + 6 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "absolutereality_v181.safetensors" + ] + }, + { + "id": 10, + "type": "EmptyLatentImage", + "pos": [ + 650, + 590 + ], + "size": { + "0": 210, + "1": 110 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 10 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 512, + 512, + 1 + ] + }, + { + "id": 3, + "type": "IPAdapterModelLoader", + "pos": [ + 290, + 60 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 1 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter_sd15.bin" + ] + }, { "id": 8, "type": "CLIPTextEncode", "pos": [ - 790, - 310 + 650, + 420 ], "size": { "0": 210, - "1": 76 + "1": 120 }, "flags": {}, - "order": 6, + "order": 7, "mode": 0, "inputs": [ { @@ -28,7 +174,7 @@ "name": "CONDITIONING", "type": "CONDITIONING", "links": [ - 20 + 9 ], "shape": 3, "slot_index": 0 @@ -38,30 +184,30 @@ "Node name for S&R": "CLIPTextEncode" }, "widgets_values": [ - "a girl with fox ears" + "blurry, horror" ], - "color": "#232", - "bgcolor": "#353" + "color": "#322", + "bgcolor": "#533" }, { - "id": 9, + "id": 7, "type": "CLIPTextEncode", "pos": [ - 790, - 430 + 650, + 250 ], "size": { "0": 210, - "1": 76 + "1": 120 }, "flags": {}, - "order": 7, + "order": 6, "mode": 0, "inputs": [ { "name": "clip", "type": "CLIP", - "link": 7 + "link": 5 } ], "outputs": [ @@ -69,7 +215,7 @@ "name": "CONDITIONING", "type": "CONDITIONING", "links": [ - 21 + 8 ], "shape": 3, "slot_index": 0 @@ -79,17 +225,17 @@ "Node name for S&R": "CLIPTextEncode" }, "widgets_values": [ - "horror" + "beautiful renaissance girl, detailed" ], - "color": "#322", - "bgcolor": "#533" + "color": "#232", + "bgcolor": "#353" }, { - "id": 12, + "id": 11, "type": "VAEDecode", "pos": [ - 1430, - 320 + 1300, + 170 ], "size": { "0": 140, @@ -102,12 +248,12 @@ { "name": "samples", "type": "LATENT", - "link": 12 + "link": 11 }, { "name": "vae", "type": "VAE", - "link": 13 + "link": 12 } ], "outputs": [ @@ -115,7 +261,7 @@ "name": "IMAGE", "type": "IMAGE", "links": [ - 31 + 13 ], "shape": 3, "slot_index": 0 @@ -125,26 +271,139 @@ "Node name for S&R": "VAEDecode" } }, + { + "id": 2, + "type": "VAELoader", + "pos": [ + 940, + 480 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 12 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "vae-ft-mse-840000-ema-pruned.safetensors" + ] + }, + { + "id": 12, + "type": "SaveImage", + "pos": [ + 1300, + 270 + ], + "size": { + "0": 400, + "1": 450 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 13 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 5, + "type": "IPAdapterApply", + "pos": [ + 650, + 60 + ], + "size": { + "0": 210, + "1": 142 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 1 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 2 + }, + { + "name": "image", + "type": "IMAGE", + "link": 3 + }, + { + "name": "model", + "type": "MODEL", + "link": 4 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterApply" + }, + "widgets_values": [ + 1, + 0 + ] + }, { "id": 6, "type": "LoadImage", "pos": [ - 120, - 200 + 40, + 60 ], "size": { "0": 220, - "1": 314 + "1": 320 }, "flags": {}, - "order": 0, + "order": 5, "mode": 0, "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ - 70 + 3 ], "shape": 3, "slot_index": 0 @@ -160,16 +419,16 @@ "Node name for S&R": "LoadImage" }, "widgets_values": [ - "example.png", + "venere.jpg", "image" ] }, { - "id": 10, + "id": 9, "type": "KSampler", "pos": [ - 1070, - 320 + 930, + 170 ], "size": { "0": 315, @@ -182,23 +441,22 @@ { "name": "model", "type": "MODEL", - "link": 44 + "link": 7 }, { "name": "positive", "type": "CONDITIONING", - "link": 20 + "link": 8 }, { "name": "negative", "type": "CONDITIONING", - "link": 21 + "link": 9 }, { "name": "latent_image", "type": "LATENT", - "link": 11, - "slot_index": 3 + "link": 10 } ], "outputs": [ @@ -206,7 +464,7 @@ "name": "LATENT", "type": "LATENT", "links": [ - 12 + 11 ], "shape": 3, "slot_index": 0 @@ -216,289 +474,60 @@ "Node name for S&R": "KSampler" }, "widgets_values": [ - 12, + 0, "fixed", - 20, - 7, - "dpmpp_2m", - "karras", + 25, + 6, + "ddim", + "ddim_uniform", 1 ] - }, - { - "id": 31, - "type": "IPAdapterApply", - "pos": [ - 757, - 120 - ], - "size": { - "0": 315, - "1": 118 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 43 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": 45 - }, - { - "name": "clip_vision_output", - "type": "CLIP_VISION_OUTPUT", - "link": 68 - }, - { - "name": "mask", - "type": "MASK", - "link": null - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 44 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterApply" - }, - "widgets_values": [ - 1 - ] - }, - { - "id": 20, - "type": "SaveImage", - "pos": [ - 1430, - 420 - ], - "size": { - "0": 526.6748046875, - "1": 579.490478515625 - }, - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 31 - } - ], - "properties": {}, - "widgets_values": [ - "ComfyUI" - ] - }, - { - "id": 1, - "type": "CLIPVisionLoader", - "pos": [ - 30, - 90 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "CLIP_VISION", - "type": "CLIP_VISION", - "links": [ - 69 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPVisionLoader" - }, - "widgets_values": [ - "IP-Adapter_sd15_pytorch_model.bin" - ] - }, - { - "id": 33, - "type": "IPAdapterCLIPVisionEncode", - "pos": [ - 443, - 178 - ], - "size": { - "0": 253.60000610351562, - "1": 50 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 69 - }, - { - "name": "image", - "type": "IMAGE", - "link": 70 - } - ], - "outputs": [ - { - "name": "CLIP_VISION_OUTPUT", - "type": "CLIP_VISION_OUTPUT", - "links": [ - 68 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterCLIPVisionEncode" - } - }, - { - "id": 11, - "type": "EmptyLatentImage", - "pos": [ - 790, - 560 - ], - "size": { - "0": 210, - "1": 110 - }, - "flags": {}, - "order": 2, - "mode": 0, - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 11 - ], - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 512, - 512, - 1 - ] - }, - { - "id": 7, - "type": "CheckpointLoaderSimple", - "pos": [ - 387, - 295 - ], - "size": { - "0": 315, - "1": 98 - }, - "flags": {}, - "order": 3, - "mode": 0, - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 43 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 6, - 7 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 13 - ], - "shape": 3, - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "v1-5-pruned-emaonly.safetensors" - ] - }, - { - "id": 3, - "type": "IPAdapterModelLoader", - "pos": [ - 430, - 60 - ], - "size": { - "0": 270, - "1": 60 - }, - "flags": {}, - "order": 4, - "mode": 0, - "outputs": [ - { - "name": "IPADAPTER", - "type": "IPADAPTER", - "links": [ - 45 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterModelLoader" - }, - "widgets_values": [ - "ip-adapter_sd15.bin" - ] } ], "links": [ [ + 1, + 3, + 0, + 5, + 0, + "IPADAPTER" + ], + [ + 2, + 4, + 0, + 5, + 1, + "CLIP_VISION" + ], + [ + 3, 6, + 0, + 5, + 2, + "IMAGE" + ], + [ + 4, + 1, + 0, + 5, + 3, + "MODEL" + ], + [ + 5, + 1, + 1, 7, + 0, + "CLIP" + ], + [ + 6, + 1, 1, 8, 0, @@ -506,106 +535,58 @@ ], [ 7, - 7, - 1, + 5, + 0, 9, 0, - "CLIP" + "MODEL" ], [ - 11, - 11, + 8, + 7, 0, + 9, + 1, + "CONDITIONING" + ], + [ + 9, + 8, + 0, + 9, + 2, + "CONDITIONING" + ], + [ 10, + 10, + 0, + 9, 3, "LATENT" ], [ - 12, - 10, + 11, + 9, 0, - 12, + 11, 0, "LATENT" ], [ - 13, - 7, - 2, 12, + 2, + 0, + 11, 1, "VAE" ], [ - 20, - 8, + 13, + 11, 0, - 10, - 1, - "CONDITIONING" - ], - [ - 21, - 9, - 0, - 10, - 2, - "CONDITIONING" - ], - [ - 31, 12, 0, - 20, - 0, - "IMAGE" - ], - [ - 43, - 7, - 0, - 31, - 0, - "MODEL" - ], - [ - 44, - 31, - 0, - 10, - 0, - "MODEL" - ], - [ - 45, - 3, - 0, - 31, - 1, - "IPADAPTER" - ], - [ - 68, - 33, - 0, - 31, - 2, - "CLIP_VISION_OUTPUT" - ], - [ - 69, - 1, - 0, - 33, - 0, - "CLIP_VISION" - ], - [ - 70, - 6, - 0, - 33, - 1, "IMAGE" ] ], diff --git a/README.md b/README.md index b33144c..d5d8ba4 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # ComfyUI_IPAdapter_Plus -ComfyUI reference implementation for [IPAdapter](https://github.com/tencent-ailab/IP-Adapter/tree/6fb9d3554a5c774f41e187e8fdbc7b9a1db8c2e3) models. +ComfyUI reference implementation for [IPAdapter](https://github.com/tencent-ailab/IP-Adapter/) models. -The code is mostly taken from the original IPAdapter repository and [laksjdjf's](https://github.com/laksjdjf/IPAdapter-ComfyUI/tree/main) implementation. I just made the extension closer to how ComfyUI makes things. +The difference of this extension and other IPAdapter implementations is that I try to follow as close as possible the ComfyUI way of doing things. This should result in faster, more reliable code. ## Updates +**2023/9/15**: Huge code cleanup! I streamlined the node structure for a tidier workflow. **IMPORTANT** this is a breaking update, we don't need the dedicated clip vision encoder anymore. Please check the new included workflows. Also introduced the new `noise` option, see below for details. + **2023/9/11**: The code has been rewritten to take advantage of the new ComfyUI updates regarding clip vision. Expect lower memory usage. ## Example workflow @@ -25,40 +27,66 @@ For SD1.5 you need: For SDXL you need: - [ip-adapter_sdxl.bin](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/ip-adapter_sdxl.bin) +- [ip-adapter_sdxl_vit-h.bin](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/ip-adapter_sdxl_vit-h.bin) **This model requires the use of the SD1.5 encoder despite being for SDXL** -Additionally you need the clip vision models: +Additionally you need the image encoders to be placed in the `ComfyUI/models/clip_vision/` directory: -- SD 1.5: [pytorch_model.bin](https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/pytorch_model.bin) -- SDXL: [pytorch_model.bin](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/image_encoder/pytorch_model.bin) +- [SD 1.5 model](https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors) (use this also for the SDXL **ip-adapter_sdxl_vit-h.bin** model) +- [SDXL model](https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/image_encoder/model.safetensors) -You can rename them to something easier to remember (eg: `ip-adapter_sd15-image-encoder.bin`) and place them under `ComfyUI/models/clip_vision/`. +You can rename them to something easier to remember or put them into a sub-directory. + +## What it is + +The IPAdapter are very powerful moderls for image to image conditioning. Given a reference image you can do image variations augmented by text prompt, control net and masks. ## How to use -There's a basic workflow included in this repo and a few examples in the [example](./examples/) directory. +There's a basic workflow included in this repo and a few examples in the [example](./examples/) directory. Usually it's a good idea to lower the `weight` to at least `0.8`. -**IMPORTANT:** Remember to use the `CLIP Vision Encode (IPAdapter)` node (and not the standard one included with ComfyUI). Also be sure to **select the model of the same kind (1.5 vs XL)**, there are 3 of them: clip vision, IPApdapter and the main checkpoint. +The `noise` paramenter is an exclusive experimental exploitation of the IPAdapter models. You can set it to even just `0.01` for an arguably better result. **Please report your experience with the noise option**! -### Samplers +
+Nerd-explain of the noise option -Weirdly enough all **dpmpp_2m** (including _sde) samplers seem to perfom a little worse with IPAdapter and they generally need 5-10 more steps than usual. **ddmin** and **euler** seem to be pretty good. +I made quite a few tests and I was not sure if it was worth it to include this feature, but the results are interesting enough to warrant at least a test period. Let me know what you think. -### IPAdapter + Canny ControlNet +canny controlnet + +Basically the IPAdapter sends two pictures for the conditioning, one is the reference the other --that you don't see-- is an empty image that could be considered like a negative conditioning. + +What I'm doing is to send a very noisy image instead of an empty one. The `noise` parameter determines the amount of noise. A value of `0.01` adds a lot of noise and creates an image close to the original created by the IPAdapter (a very noisy image being intelligible could be considered like an empty image); a value of `1.0` removes most of noise so the image gets conditioned more. + +
+ +### KSampler configuration suggestions + +Weirdly enough all `dpmpp_2m` (including `_sde`) samplers seem to perfom poorly with IPAdapter and they generally need 5-10 more steps than usual. `ddmin` and `euler` seem to be pretty good. + +The model tends to burn the images a little. If needed lower the CFG scale. + +It generally requires a few more steps than usual, if the result is underwhelming try to add 10+ steps. + +### IPAdapter + ControlNet The model is very effective when paired with a ControlNet. In the example below I experimented with Canny. [The workflow](./examples/IPAdapter_Canny.json) is in the examples directory. -canny controlnet +canny controlnet ### IPAdapter Face -IPAdapter offers an interesting model for a kind of "face swap" effect. [The workflow is provided](./examples/face.json). +IPAdapter offers an interesting model for a kind of "face swap" effect. [The workflow is provided](./examples/face.json). Set a close up face as reference image and then input your text prompt as always. The generated character should have the face of the reference. It also works with img2img given a high denoise. -face swap +face swap ### Masking -Masking in img2img generally works but I find inpainting to be far more effective. The [inpainting workflow](./examples/inpainting.json) uses the face model together with an inpainting checkpoint. +The most effect way to apply the IPAdapter to a region is by [inpainting workflow](./examples/inpainting.json). Remeber to use a specific checkpoint for inpainting otherwise it won't work. Even if you are inpainting a face I fince that the IPAdapter-Plus (not the face one), works best. inpainting -**Important:** when masking the *IPAdapter Apply* node be sure that the mask is of the same size of the latent. \ No newline at end of file +## Credits + +- [IPAdapter](ttps://github.com/tencent-ailab/IP-Adapter/) +- [laksjdjf](https://github.com/laksjdjf/IPAdapter-ComfyUI/) +- [ComfyUI](https://github.com/comfyanonymous/ComfyUI) \ No newline at end of file diff --git a/examples/IPAdapter_Canny.json b/examples/IPAdapter_Canny.json index 75d5fa9..c3b9901 100644 --- a/examples/IPAdapter_Canny.json +++ b/examples/IPAdapter_Canny.json @@ -1,486 +1,59 @@ { - "last_node_id": 52, - "last_link_id": 128, + "last_node_id": 17, + "last_link_id": 19, "nodes": [ { - "id": 33, - "type": "IPAdapterCLIPVisionEncode", + "id": 4, + "type": "CLIPVisionLoader", "pos": [ - 590, - 370 + 290, + 170 ], "size": { - "0": 253.60000610351562, - "1": 50 - }, - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 69 - }, - { - "name": "image", - "type": "IMAGE", - "link": 128 - } - ], - "outputs": [ - { - "name": "CLIP_VISION_OUTPUT", - "type": "CLIP_VISION_OUTPUT", - "links": [ - 68 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterCLIPVisionEncode" - } - }, - { - "id": 12, - "type": "VAEDecode", - "pos": [ - 1830, - 150 - ], - "size": { - "0": 140, - "1": 50 - }, - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 12 - }, - { - "name": "vae", - "type": "VAE", - "link": 13 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 31 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - } - }, - { - "id": 20, - "type": "SaveImage", - "pos": [ - 1830, - 250 - ], - "size": { - "0": 350, - "1": 390 - }, - "flags": {}, - "order": 17, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 31 - } - ], - "properties": {}, - "widgets_values": [ - "ComfyUI" - ] - }, - { - "id": 7, - "type": "CheckpointLoaderSimple", - "pos": [ - 560, - 90 - ], - "size": { - "0": 315, - "1": 98 - }, - "flags": {}, - "order": 4, - "mode": 0, - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 97 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 6, - 7 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 13 - ], - "shape": 3, - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 3, - "type": "IPAdapterModelLoader", - "pos": [ - 600, - 240 - ], - "size": { - "0": 270, + "0": 300, "1": 60 }, "flags": {}, - "order": 2, - "mode": 0, - "outputs": [ - { - "name": "IPADAPTER", - "type": "IPADAPTER", - "links": [ - 45 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterModelLoader" - }, - "widgets_values": [ - "ip-adapter_sd15.bin" - ] - }, - { - "id": 9, - "type": "CLIPTextEncode", - "pos": [ - 940, - 400 - ], - "size": { - "0": 210, - "1": 76 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 7 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 125 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "horror, dark, blurry" - ], - "color": "#322", - "bgcolor": "#533" - }, - { - "id": 8, - "type": "CLIPTextEncode", - "pos": [ - 940, - 270 - ], - "size": { - "0": 210, - "1": 76 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 6 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 126 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "professional photography, colorful" - ], - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 42, - "type": "Canny", - "pos": [ - 940, - 650 - ], - "size": [ - 210, - 80 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 118 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 119, - 120 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "Canny" - }, - "widgets_values": [ - 0.15, - 0.4 - ] - }, - { - "id": 49, - "type": "PreviewImage", - "pos": [ - 940, - 770 - ], - "size": [ - 210, - 246 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 119 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, - { - "id": 36, - "type": "LoadImage", - "pos": [ - 700, - 650 - ], - "size": { - "0": 220, - "1": 314 - }, - "flags": {}, - "order": 6, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 118 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "cwf_portrait.jpg", - "image" - ] - }, - { - "id": 45, - "type": "ControlNetLoader", - "pos": [ - 810, - 550 - ], - "size": [ - 338.6476156616212, - 58 - ], - "flags": {}, - "order": 3, - "mode": 0, - "outputs": [ - { - "name": "CONTROL_NET", - "type": "CONTROL_NET", - "links": [ - 113 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ControlNetLoader" - }, - "widgets_values": [ - "control_v11p_sd15_canny_fp16.safetensors" - ] - }, - { - "id": 47, - "type": "ControlNetApply", - "pos": [ - 1200, - 480 - ], - "size": [ - 210, - 100 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "conditioning", - "type": "CONDITIONING", - "link": 126 - }, - { - "name": "control_net", - "type": "CONTROL_NET", - "link": 113 - }, - { - "name": "image", - "type": "IMAGE", - "link": 120 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 111 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ControlNetApply" - }, - "widgets_values": [ - 1 - ] - }, - { - "id": 11, - "type": "EmptyLatentImage", - "pos": [ - 1200, - 310 - ], - "size": { - "0": 210, - "1": 110 - }, - "flags": {}, "order": 0, "mode": 0, + "outputs": [ + { + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 2 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IP-Adapter_sd15_pytorch_model.bin" + ] + }, + { + "id": 10, + "type": "EmptyLatentImage", + "pos": [ + 650, + 590 + ], + "size": { + "0": 210, + "1": 110 + }, + "flags": {}, + "order": 1, + "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ - 11 + 10 ], "shape": 3, "slot_index": 0 @@ -496,46 +69,146 @@ ] }, { - "id": 10, - "type": "KSampler", + "id": 8, + "type": "CLIPTextEncode", "pos": [ - 1470, - 150 + 650, + 420 ], "size": { - "0": 315, - "1": 262 + "0": 210, + "1": 120 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 6 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, horror" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 650, + 250 + ], + "size": { + "0": 210, + "1": 120 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "beautiful renaissance girl with a necklace, detailed" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 11, + "type": "VAEDecode", + "pos": [ + 1300, + 170 + ], + "size": { + "0": 140, + "1": 50 }, "flags": {}, "order": 15, "mode": 0, "inputs": [ { - "name": "model", - "type": "MODEL", - "link": 109 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 111 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 125 - }, - { - "name": "latent_image", + "name": "samples", "type": "LATENT", - "link": 11, - "slot_index": 3 + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 } ], "outputs": [ { - "name": "LATENT", - "type": "LATENT", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 2, + "type": "VAELoader", + "pos": [ + 940, + 480 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", "links": [ 12 ], @@ -544,92 +217,58 @@ } ], "properties": { - "Node name for S&R": "KSampler" + "Node name for S&R": "VAELoader" }, "widgets_values": [ - 1, - "fixed", - 20, - 6, - "dpmpp_2m", - "karras", - 1 + "vae-ft-mse-840000-ema-pruned.safetensors" ] }, { - "id": 31, - "type": "IPAdapterApply", + "id": 12, + "type": "SaveImage", "pos": [ - 940, - 90 + 1300, + 270 ], "size": { - "0": 315, - "1": 118 + "0": 400, + "1": 450 }, "flags": {}, - "order": 14, + "order": 16, "mode": 0, "inputs": [ { - "name": "model", - "type": "MODEL", - "link": 97 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": 45 - }, - { - "name": "clip_vision_output", - "type": "CLIP_VISION_OUTPUT", - "link": 68 - }, - { - "name": "mask", - "type": "MASK", - "link": null + "name": "images", + "type": "IMAGE", + "link": 13 } ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 109 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterApply" - }, + "properties": {}, "widgets_values": [ - 0.8 + "IPAdapter" ] }, { "id": 6, "type": "LoadImage", "pos": [ - 70, - 390 - ], - "size": [ - 220, - 240 + 40, + 60 ], + "size": { + "0": 220, + "1": 320 + }, "flags": {}, - "order": 5, + "order": 3, "mode": 0, "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ - 127 + 3 ], "shape": 3, "slot_index": 0 @@ -645,29 +284,125 @@ "Node name for S&R": "LoadImage" }, "widgets_values": [ - "360_F_536931911_6LTOivzgUYNHMYV5zHvwHoedsY7zlzsi.jpg", + "venere.jpg", "image" ] }, { - "id": 52, - "type": "ImageScale", + "id": 14, + "type": "LoadImage", "pos": [ - 330, - 390 + 50, + 860 + ], + "size": [ + 220, + 320 + ], + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 14 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "portrait.jpg", + "image" + ] + }, + { + "id": 15, + "type": "ControlNetLoader", + "pos": [ + 190, + 750 + ], + "size": [ + 310, + 60 + ], + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ControlNetLoader" + }, + "widgets_values": [ + "control_v11p_sd15_canny_fp16.safetensors" + ] + }, + { + "id": 17, + "type": "PreviewImage", + "pos": [ + 579.412728881836, + 903.3208389282227 ], "size": [ 210, - 130 + 246 ], "flags": {}, - "order": 9, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 19 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 13, + "type": "Canny", + "pos": [ + 290, + 860 + ], + "size": [ + 210, + 80 + ], + "flags": {}, + "order": 8, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", - "link": 127 + "link": 14 } ], "outputs": [ @@ -675,59 +410,310 @@ "name": "IMAGE", "type": "IMAGE", "links": [ - 128 + 18, + 19 ], "shape": 3, "slot_index": 0 } ], "properties": { - "Node name for S&R": "ImageScale" + "Node name for S&R": "Canny" }, "widgets_values": [ - "nearest-exact", - 512, - 512, - "center" + 0.1, + 0.5 + ] + }, + { + "id": 16, + "type": "ControlNetApply", + "pos": [ + 540, + 750 + ], + "size": { + "0": 317.4000244140625, + "1": 98 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 16 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 15 + }, + { + "name": "image", + "type": "IMAGE", + "link": 18 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 17 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ControlNetApply" + }, + "widgets_values": [ + 0.8 + ] + }, + { + "id": 3, + "type": "IPAdapterModelLoader", + "pos": [ + 290, + 60 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 1 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.bin" + ] + }, + { + "id": 5, + "type": "IPAdapterApply", + "pos": [ + 650, + 60 + ], + "size": { + "0": 210, + "1": 142 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 1 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 2 + }, + { + "name": "image", + "type": "IMAGE", + "link": 3 + }, + { + "name": "model", + "type": "MODEL", + "link": 4 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterApply" + }, + "widgets_values": [ + 0.8, + 0 ] }, { "id": 1, - "type": "CLIPVisionLoader", + "type": "CheckpointLoaderSimple", "pos": [ - 230, + 290, 280 ], "size": { - "0": 315, - "1": 58 + "0": 300, + "1": 100 }, "flags": {}, - "order": 1, + "order": 7, "mode": 0, "outputs": [ { - "name": "CLIP_VISION", - "type": "CLIP_VISION", + "name": "MODEL", + "type": "MODEL", "links": [ - 69 + 4 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 5, + 6 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 9, + "type": "KSampler", + "pos": [ + 930, + 170 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 7 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 17 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 9 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 10 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 11 ], "shape": 3, "slot_index": 0 } ], "properties": { - "Node name for S&R": "CLIPVisionLoader" + "Node name for S&R": "KSampler" }, "widgets_values": [ - "IP-Adapter_sd15_pytorch_model.bin" + 27, + "fixed", + 30, + 6, + "ddim", + "ddim_uniform", + 1 ] } ], "links": [ [ + 1, + 3, + 0, + 5, + 0, + "IPADAPTER" + ], + [ + 2, + 4, + 0, + 5, + 1, + "CLIP_VISION" + ], + [ + 3, 6, + 0, + 5, + 2, + "IMAGE" + ], + [ + 4, + 1, + 0, + 5, + 3, + "MODEL" + ], + [ + 5, + 1, + 1, 7, + 0, + "CLIP" + ], + [ + 6, + 1, 1, 8, 0, @@ -735,154 +721,98 @@ ], [ 7, - 7, - 1, + 5, + 0, 9, 0, - "CLIP" + "MODEL" ], [ - 11, - 11, + 9, + 8, 0, + 9, + 2, + "CONDITIONING" + ], + [ 10, + 10, + 0, + 9, 3, "LATENT" ], [ - 12, - 10, + 11, + 9, 0, - 12, + 11, 0, "LATENT" ], [ - 13, - 7, - 2, 12, + 2, + 0, + 11, 1, "VAE" ], [ - 31, - 12, + 13, + 11, 0, - 20, + 12, 0, "IMAGE" ], [ - 45, - 3, + 14, + 14, 0, - 31, - 1, - "IPADAPTER" + 13, + 0, + "IMAGE" ], [ - 68, - 33, + 15, + 15, 0, - 31, - 2, - "CLIP_VISION_OUTPUT" - ], - [ - 69, - 1, - 0, - 33, - 0, - "CLIP_VISION" - ], - [ - 97, - 7, - 0, - 31, - 0, - "MODEL" - ], - [ - 109, - 31, - 0, - 10, - 0, - "MODEL" - ], - [ - 111, - 47, - 0, - 10, - 1, - "CONDITIONING" - ], - [ - 113, - 45, - 0, - 47, + 16, 1, "CONTROL_NET" ], [ - 118, - 36, + 16, + 7, 0, - 42, + 16, 0, - "IMAGE" + "CONDITIONING" ], [ - 119, - 42, + 17, + 16, 0, - 49, - 0, - "IMAGE" - ], - [ - 120, - 42, - 0, - 47, - 2, - "IMAGE" - ], - [ - 125, 9, + 1, + "CONDITIONING" + ], + [ + 18, + 13, 0, - 10, + 16, 2, - "CONDITIONING" - ], - [ - 126, - 8, - 0, - 47, - 0, - "CONDITIONING" - ], - [ - 127, - 6, - 0, - 52, - 0, "IMAGE" ], [ - 128, - 52, + 19, + 13, + 0, + 17, 0, - 33, - 1, "IMAGE" ] ], diff --git a/examples/face.json b/examples/IPAdapter_face.json similarity index 79% rename from examples/face.json rename to examples/IPAdapter_face.json index 677d90b..d126485 100644 --- a/examples/face.json +++ b/examples/IPAdapter_face.json @@ -1,54 +1,86 @@ { - "last_node_id": 52, - "last_link_id": 132, + "last_node_id": 18, + "last_link_id": 28, "nodes": [ { - "id": 33, - "type": "IPAdapterCLIPVisionEncode", + "id": 4, + "type": "CLIPVisionLoader", "pos": [ - 590, - 370 + 290, + 170 ], "size": { - "0": 253.60000610351562, - "1": 50 + "0": 300, + "1": 60 }, "flags": {}, - "order": 7, + "order": 0, "mode": 0, - "inputs": [ - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 69 - }, - { - "name": "image", - "type": "IMAGE", - "link": 132 - } - ], "outputs": [ { - "name": "CLIP_VISION_OUTPUT", - "type": "CLIP_VISION_OUTPUT", + "name": "CLIP_VISION", + "type": "CLIP_VISION", "links": [ - 68 + 2 ], "shape": 3, "slot_index": 0 } ], "properties": { - "Node name for S&R": "IPAdapterCLIPVisionEncode" - } + "Node name for S&R": "CLIPVisionLoader" + }, + "widgets_values": [ + "IP-Adapter_sd15_pytorch_model.bin" + ] }, { - "id": 12, + "id": 8, + "type": "CLIPTextEncode", + "pos": [ + 650, + 420 + ], + "size": { + "0": 210, + "1": 120 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 6 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, horror" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 11, "type": "VAEDecode", "pos": [ - 1830, - 150 + 1300, + 170 ], "size": { "0": 140, @@ -61,12 +93,12 @@ { "name": "samples", "type": "LATENT", - "link": 12 + "link": 11 }, { "name": "vae", "type": "VAE", - "link": 13 + "link": 12 } ], "outputs": [ @@ -74,7 +106,7 @@ "name": "IMAGE", "type": "IMAGE", "links": [ - 31 + 13 ], "shape": 3, "slot_index": 0 @@ -85,15 +117,15 @@ } }, { - "id": 20, + "id": 12, "type": "SaveImage", "pos": [ - 1830, - 250 + 1300, + 270 ], "size": { - "0": 350, - "1": 390 + "0": 400, + "1": 450 }, "flags": {}, "order": 11, @@ -102,24 +134,136 @@ { "name": "images", "type": "IMAGE", - "link": 31 + "link": 13 } ], "properties": {}, "widgets_values": [ - "ComfyUI" + "IPAdapter" ] }, { - "id": 9, - "type": "CLIPTextEncode", + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 290, + 280 + ], + "size": { + "0": 300, + "1": 100 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 4 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 5, + 6 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "realisticVisionV51_v51VAE.safetensors" + ] + }, + { + "id": 3, + "type": "IPAdapterModelLoader", + "pos": [ + 290, + 60 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 1 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus-face_sd15.bin" + ] + }, + { + "id": 2, + "type": "VAELoader", "pos": [ 940, - 400 + 480 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 12 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "vae-ft-mse-840000-ema-pruned.safetensors" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 650, + 250 ], "size": { "0": 210, - "1": 76 + "1": 120 }, "flags": {}, "order": 6, @@ -128,7 +272,7 @@ { "name": "clip", "type": "CLIP", - "link": 7 + "link": 5 } ], "outputs": [ @@ -136,7 +280,7 @@ "name": "CONDITIONING", "type": "CONDITIONING", "links": [ - 125 + 25 ], "shape": 3, "slot_index": 0 @@ -146,31 +290,31 @@ "Node name for S&R": "CLIPTextEncode" }, "widgets_values": [ - "blurry, illustration" + "closeup photo of a guy flexing muscles at the gym" ], - "color": "#322", - "bgcolor": "#533" + "color": "#232", + "bgcolor": "#353" }, { - "id": 11, + "id": 10, "type": "EmptyLatentImage", "pos": [ - 1200, - 310 + 650, + 590 ], "size": { "0": 210, "1": 110 }, "flags": {}, - "order": 0, + "order": 4, "mode": 0, "outputs": [ { "name": "LATENT", "type": "LATENT", "links": [ - 11 + 26 ], "shape": 3, "slot_index": 0 @@ -186,195 +330,39 @@ ] }, { - "id": 1, - "type": "CLIPVisionLoader", - "pos": [ - 230, - 280 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "CLIP_VISION", - "type": "CLIP_VISION", - "links": [ - 69 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPVisionLoader" - }, - "widgets_values": [ - "IP-Adapter_sd15_pytorch_model.bin" - ] - }, - { - "id": 3, - "type": "IPAdapterModelLoader", - "pos": [ - 600, - 240 - ], - "size": { - "0": 270, - "1": 60 - }, - "flags": {}, - "order": 2, - "mode": 0, - "outputs": [ - { - "name": "IPADAPTER", - "type": "IPADAPTER", - "links": [ - 45 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterModelLoader" - }, - "widgets_values": [ - "ip-adapter-plus-face_sd15.bin" - ] - }, - { - "id": 8, - "type": "CLIPTextEncode", - "pos": [ - 941, - 253 - ], - "size": [ - 219.03509521484375, - 114.86827850341797 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 6 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 129 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "closeup photo of a man wearing a spacesuit" - ], - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 7, - "type": "CheckpointLoaderSimple", - "pos": [ - 560, - 90 - ], - "size": { - "0": 315, - "1": 98 - }, - "flags": {}, - "order": 3, - "mode": 0, - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 97 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 6, - 7 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 13 - ], - "shape": 3, - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "realisticVisionV30_v30VAE.safetensors" - ] - }, - { - "id": 31, + "id": 5, "type": "IPAdapterApply", "pos": [ - 940, - 90 + 650, + 60 ], "size": { - "0": 315, - "1": 118 + "0": 210, + "1": 142 }, "flags": {}, "order": 8, "mode": 0, "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 97 - }, { "name": "ipadapter", "type": "IPADAPTER", - "link": 45 + "link": 1 }, { - "name": "clip_vision_output", - "type": "CLIP_VISION_OUTPUT", - "link": 68 + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 2 }, { - "name": "mask", - "type": "MASK", - "link": null + "name": "image", + "type": "IMAGE", + "link": 28 + }, + { + "name": "model", + "type": "MODEL", + "link": 4 } ], "outputs": [ @@ -382,7 +370,7 @@ "name": "MODEL", "type": "MODEL", "links": [ - 109 + 7 ], "shape": 3, "slot_index": 0 @@ -392,15 +380,16 @@ "Node name for S&R": "IPAdapterApply" }, "widgets_values": [ - 0.8 + 0.6000000000000001, + 0 ] }, { - "id": 10, + "id": 9, "type": "KSampler", "pos": [ - 1470, - 150 + 930, + 170 ], "size": { "0": 315, @@ -413,23 +402,22 @@ { "name": "model", "type": "MODEL", - "link": 109 + "link": 7 }, { "name": "positive", "type": "CONDITIONING", - "link": 129 + "link": 25 }, { "name": "negative", "type": "CONDITIONING", - "link": 125 + "link": 9 }, { "name": "latent_image", "type": "LATENT", - "link": 11, - "slot_index": 3 + "link": 26 } ], "outputs": [ @@ -437,7 +425,7 @@ "name": "LATENT", "type": "LATENT", "links": [ - 12 + 11 ], "shape": 3, "slot_index": 0 @@ -447,35 +435,35 @@ "Node name for S&R": "KSampler" }, "widgets_values": [ - 8, + 32, "fixed", - 20, - 7, - "dpmpp_2m", - "karras", + 35, + 6, + "ddim", + "ddim_uniform", 1 ] }, { - "id": 6, + "id": 14, "type": "LoadImage", "pos": [ - 2200, - 215 + 40, + 60 ], "size": [ 220, - 314 + 320 ], "flags": {}, - "order": 4, + "order": 5, "mode": 0, "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ - 132 + 28 ], "shape": 3, "slot_index": 0 @@ -498,8 +486,40 @@ ], "links": [ [ - 6, + 1, + 3, + 0, + 5, + 0, + "IPADAPTER" + ], + [ + 2, + 4, + 0, + 5, + 1, + "CLIP_VISION" + ], + [ + 4, + 1, + 0, + 5, + 3, + "MODEL" + ], + [ + 5, + 1, + 1, 7, + 0, + "CLIP" + ], + [ + 6, + 1, 1, 8, 0, @@ -507,106 +527,66 @@ ], [ 7, - 7, - 1, + 5, + 0, 9, 0, - "CLIP" + "MODEL" ], [ - 11, - 11, + 9, + 8, 0, - 10, - 3, - "LATENT" - ], - [ - 12, - 10, - 0, - 12, - 0, - "LATENT" - ], - [ - 13, - 7, + 9, 2, + "CONDITIONING" + ], + [ + 11, + 9, + 0, + 11, + 0, + "LATENT" + ], + [ 12, + 2, + 0, + 11, 1, "VAE" ], [ - 31, - 12, + 13, + 11, 0, - 20, + 12, 0, "IMAGE" ], [ - 45, - 3, - 0, - 31, - 1, - "IPADAPTER" - ], - [ - 68, - 33, - 0, - 31, - 2, - "CLIP_VISION_OUTPUT" - ], - [ - 69, - 1, - 0, - 33, - 0, - "CLIP_VISION" - ], - [ - 97, + 25, 7, 0, - 31, - 0, - "MODEL" - ], - [ - 109, - 31, - 0, - 10, - 0, - "MODEL" - ], - [ - 125, 9, - 0, + 1, + "CONDITIONING" + ], + [ + 26, 10, + 0, + 9, + 3, + "LATENT" + ], + [ + 28, + 14, + 0, + 5, 2, - "CONDITIONING" - ], - [ - 129, - 8, - 0, - 10, - 1, - "CONDITIONING" - ], - [ - 132, - 6, - 0, - 33, - 1, "IMAGE" ] ], diff --git a/examples/canny_cn.jpg b/examples/canny_cn.jpg deleted file mode 100644 index 524354e..0000000 Binary files a/examples/canny_cn.jpg and /dev/null differ diff --git a/examples/canny_controlnet.jpg b/examples/canny_controlnet.jpg new file mode 100644 index 0000000..8fd1ef5 Binary files /dev/null and b/examples/canny_controlnet.jpg differ diff --git a/examples/face_swap.jpg b/examples/face_swap.jpg index 1b10e09..d638483 100644 Binary files a/examples/face_swap.jpg and b/examples/face_swap.jpg differ diff --git a/examples/inpainting.jpg b/examples/inpainting.jpg index 5cd32f5..d8ab437 100644 Binary files a/examples/inpainting.jpg and b/examples/inpainting.jpg differ diff --git a/examples/inpainting.json b/examples/ipadapter_inpaint.json similarity index 76% rename from examples/inpainting.json rename to examples/ipadapter_inpaint.json index b450927..f6a1edd 100644 --- a/examples/inpainting.json +++ b/examples/ipadapter_inpaint.json @@ -1,58 +1,17 @@ { - "last_node_id": 63, - "last_link_id": 158, + "last_node_id": 23, + "last_link_id": 37, "nodes": [ { - "id": 12, - "type": "VAEDecode", - "pos": [ - 1650, - 90 - ], - "size": { - "0": 140, - "1": 50 - }, - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 12 - }, - { - "name": "vae", - "type": "VAE", - "link": 13 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 31 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - } - }, - { - "id": 1, + "id": 4, "type": "CLIPVisionLoader", "pos": [ - 230, - 280 + 290, + 170 ], "size": { - "0": 315, - "1": 58 + "0": 300, + "1": 60 }, "flags": {}, "order": 0, @@ -62,7 +21,7 @@ "name": "CLIP_VISION", "type": "CLIP_VISION", "links": [ - 69 + 2 ], "shape": 3, "slot_index": 0 @@ -76,25 +35,174 @@ ] }, { - "id": 6, + "id": 8, + "type": "CLIPTextEncode", + "pos": [ + 650, + 420 + ], + "size": { + "0": 210, + "1": 120 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 6 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, horror" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 11, + "type": "VAEDecode", + "pos": [ + 1300, + 170 + ], + "size": { + "0": 140, + "1": 50 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 12, + "type": "SaveImage", + "pos": [ + 1300, + 270 + ], + "size": { + "0": 400, + "1": 450 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 13 + } + ], + "properties": {}, + "widgets_values": [ + "IPAdapter" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 650, + 250 + ], + "size": { + "0": 210, + "1": 120 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "closeup photo of a renaissance astronaut " + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 14, "type": "LoadImage", "pos": [ - 327, - 413 + 40, + 60 ], "size": { "0": 220, - "1": 314 + "1": 320 }, "flags": {}, - "order": 3, + "order": 1, "mode": 0, "outputs": [ { "name": "IMAGE", "type": "IMAGE", "links": [ - 132 + 28 ], "shape": 3, "slot_index": 0 @@ -115,97 +223,69 @@ ] }, { - "id": 59, - "type": "LoadImage", + "id": 5, + "type": "IPAdapterApply", "pos": [ - 595, - 548 - ], - "size": [ - 315, - 314 - ], - "flags": {}, - "order": 4, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 158 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": [ - 157 - ], - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "clipspace/clipspace-mask-4664807.png [input]", - "image" - ] - }, - { - "id": 33, - "type": "IPAdapterCLIPVisionEncode", - "pos": [ - 590, - 370 + 650, + 60 ], "size": { - "0": 253.60000610351562, - "1": 50 + "0": 210, + "1": 142 }, "flags": {}, - "order": 7, + "order": 8, "mode": 0, "inputs": [ + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 1 + }, { "name": "clip_vision", "type": "CLIP_VISION", - "link": 69 + "link": 2 }, { "name": "image", "type": "IMAGE", - "link": 132 + "link": 28 + }, + { + "name": "model", + "type": "MODEL", + "link": 4 } ], "outputs": [ { - "name": "CLIP_VISION_OUTPUT", - "type": "CLIP_VISION_OUTPUT", + "name": "MODEL", + "type": "MODEL", "links": [ - 68 + 7 ], "shape": 3, "slot_index": 0 } ], "properties": { - "Node name for S&R": "IPAdapterCLIPVisionEncode" - } + "Node name for S&R": "IPAdapterApply" + }, + "widgets_values": [ + 0.8, + 0 + ] }, { - "id": 3, - "type": "IPAdapterModelLoader", + "id": 2, + "type": "VAELoader", "pos": [ - 583, - 230 + 290, + 430 ], "size": { - "0": 270, + "0": 300, "1": 60 }, "flags": {}, @@ -213,42 +293,43 @@ "mode": 0, "outputs": [ { - "name": "IPADAPTER", - "type": "IPADAPTER", + "name": "VAE", + "type": "VAE", "links": [ - 45 + 12, + 30 ], "shape": 3, "slot_index": 0 } ], "properties": { - "Node name for S&R": "IPAdapterModelLoader" + "Node name for S&R": "VAELoader" }, "widgets_values": [ - "ip-adapter-plus-face_sd15.bin" + "vae-ft-mse-840000-ema-pruned.safetensors" ] }, { - "id": 7, + "id": 1, "type": "CheckpointLoaderSimple", "pos": [ - 536, - 68 + 290, + 280 ], "size": { - "0": 315, - "1": 98 + "0": 300, + "1": 100 }, "flags": {}, - "order": 1, + "order": 3, "mode": 0, "outputs": [ { "name": "MODEL", "type": "MODEL", "links": [ - 142 + 4 ], "shape": 3, "slot_index": 0 @@ -257,8 +338,8 @@ "name": "CLIP", "type": "CLIP", "links": [ - 6, - 7 + 5, + 6 ], "shape": 3, "slot_index": 1 @@ -266,12 +347,8 @@ { "name": "VAE", "type": "VAE", - "links": [ - 13, - 155 - ], - "shape": 3, - "slot_index": 2 + "links": null, + "shape": 3 } ], "properties": { @@ -281,223 +358,12 @@ "absolutereality_v181INPAINTING.safetensors" ] }, - { - "id": 20, - "type": "SaveImage", - "pos": [ - 1650, - 190 - ], - "size": { - "0": 350, - "1": 390 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 31 - } - ], - "properties": {}, - "widgets_values": [ - "ComfyUI" - ] - }, - { - "id": 63, - "type": "VAEEncodeForInpaint", - "pos": [ - 968, - 550 - ], - "size": [ - 226.8000030517578, - 98 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "pixels", - "type": "IMAGE", - "link": 158 - }, - { - "name": "vae", - "type": "VAE", - "link": 155 - }, - { - "name": "mask", - "type": "MASK", - "link": 157 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 156 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEEncodeForInpaint" - }, - "widgets_values": [ - 6 - ] - }, - { - "id": 31, - "type": "IPAdapterApply", - "pos": [ - 942, - 77 - ], - "size": [ - 250, - 120 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 142 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": 45 - }, - { - "name": "clip_vision_output", - "type": "CLIP_VISION_OUTPUT", - "link": 68 - }, - { - "name": "mask", - "type": "MASK", - "link": null - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 150 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapterApply" - }, - "widgets_values": [ - 0.8 - ] - }, - { - "id": 8, - "type": "CLIPTextEncode", - "pos": [ - 941, - 253 - ], - "size": [ - 250, - 80 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 6 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 129 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "closeup photo of a blond woman in a forest, professional, detailed, 4k" - ], - "color": "#232", - "bgcolor": "#353" - }, { "id": 9, - "type": "CLIPTextEncode", - "pos": [ - 940, - 380 - ], - "size": [ - 250, - 76 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 7 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 125 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, illustration, hat, naked" - ], - "color": "#322", - "bgcolor": "#533" - }, - { - "id": 10, "type": "KSampler", "pos": [ - 1290, - 90 + 930, + 170 ], "size": { "0": 315, @@ -510,23 +376,22 @@ { "name": "model", "type": "MODEL", - "link": 150 + "link": 7 }, { "name": "positive", "type": "CONDITIONING", - "link": 129 + "link": 25 }, { "name": "negative", "type": "CONDITIONING", - "link": 125 + "link": 9 }, { "name": "latent_image", "type": "LATENT", - "link": 156, - "slot_index": 3 + "link": 32 } ], "outputs": [ @@ -534,7 +399,7 @@ "name": "LATENT", "type": "LATENT", "links": [ - 12 + 11 ], "shape": 3, "slot_index": 0 @@ -544,20 +409,175 @@ "Node name for S&R": "KSampler" }, "widgets_values": [ - 45, + 34, "fixed", - 30, - 7, - "dpmpp_2m_sde", - "karras", + 35, + 6, + "ddim", + "ddim_uniform", 1 ] + }, + { + "id": 3, + "type": "IPAdapterModelLoader", + "pos": [ + 290, + 60 + ], + "size": { + "0": 300, + "1": 60 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IPADAPTER", + "type": "IPADAPTER", + "links": [ + 1 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterModelLoader" + }, + "widgets_values": [ + "ip-adapter-plus_sd15.bin" + ] + }, + { + "id": 20, + "type": "VAEEncodeForInpaint", + "pos": [ + 650, + 590 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 29 + }, + { + "name": "vae", + "type": "VAE", + "link": 30 + }, + { + "name": "mask", + "type": "MASK", + "link": 37 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 32 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEEncodeForInpaint" + }, + "widgets_values": [ + 12 + ] + }, + { + "id": 19, + "type": "LoadImage", + "pos": [ + 370, + 600 + ], + "size": { + "0": 220, + "1": 320 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 29 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 37 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "clipspace/clipspace-mask-8127362.png [input]", + "image" + ] } ], "links": [ [ - 6, + 1, + 3, + 0, + 5, + 0, + "IPADAPTER" + ], + [ + 2, + 4, + 0, + 5, + 1, + "CLIP_VISION" + ], + [ + 4, + 1, + 0, + 5, + 3, + "MODEL" + ], + [ + 5, + 1, + 1, 7, + 0, + "CLIP" + ], + [ + 6, + 1, 1, 8, 0, @@ -565,131 +585,91 @@ ], [ 7, - 7, - 1, + 5, + 0, 9, 0, - "CLIP" + "MODEL" ], [ - 12, - 10, + 9, + 8, 0, - 12, + 9, + 2, + "CONDITIONING" + ], + [ + 11, + 9, + 0, + 11, 0, "LATENT" ], [ - 13, - 7, - 2, 12, + 2, + 0, + 11, 1, "VAE" ], [ - 31, + 13, + 11, + 0, 12, 0, + "IMAGE" + ], + [ + 25, + 7, + 0, + 9, + 1, + "CONDITIONING" + ], + [ + 28, + 14, + 0, + 5, + 2, + "IMAGE" + ], + [ + 29, + 19, + 0, 20, 0, "IMAGE" ], [ - 45, - 3, - 0, - 31, - 1, - "IPADAPTER" - ], - [ - 68, - 33, - 0, - 31, + 30, 2, - "CLIP_VISION_OUTPUT" - ], - [ - 69, - 1, 0, - 33, - 0, - "CLIP_VISION" - ], - [ - 125, - 9, - 0, - 10, - 2, - "CONDITIONING" - ], - [ - 129, - 8, - 0, - 10, - 1, - "CONDITIONING" - ], - [ - 132, - 6, - 0, - 33, - 1, - "IMAGE" - ], - [ - 142, - 7, - 0, - 31, - 0, - "MODEL" - ], - [ - 150, - 31, - 0, - 10, - 0, - "MODEL" - ], - [ - 155, - 7, - 2, - 63, + 20, 1, "VAE" ], [ - 156, - 63, + 32, + 20, 0, - 10, + 9, 3, "LATENT" ], [ - 157, - 59, + 37, + 19, 1, - 63, + 20, 2, "MASK" - ], - [ - 158, - 59, - 0, - 63, - 0, - "IMAGE" ] ], "groups": [], diff --git a/examples/noise_example.jpg b/examples/noise_example.jpg new file mode 100644 index 0000000..c73dc93 Binary files /dev/null and b/examples/noise_example.jpg differ diff --git a/ipadapter_workflow.png b/ipadapter_workflow.png index ae53b9e..23a8c92 100644 Binary files a/ipadapter_workflow.png and b/ipadapter_workflow.png differ