Merge pull request #8 from smthemex/PR

add new

Hidiffusion_node.py

@@ -5,7 +5,6 @@ import torch
 import os
 from PIL import Image
 import numpy as np
-import sys
 from diffusers import (StableDiffusionXLPipeline, DiffusionPipeline, DDIMScheduler, ControlNetModel,
                        KDPM2AncestralDiscreteScheduler, LMSDiscreteScheduler,
                        AutoPipelineForInpainting, DPMSolverMultistepScheduler, DPMSolverSinglestepScheduler,
@@ -15,16 +14,17 @@ from diffusers import (StableDiffusionXLPipeline, DiffusionPipeline, DDIMSchedul
                        EulerAncestralDiscreteScheduler, UniPCMultistepScheduler, AutoencoderKL,
                        StableDiffusionXLControlNetPipeline, DDPMScheduler, TCDScheduler, LCMScheduler,
                        StableDiffusionPipeline, StableDiffusionControlNetPipeline, StableDiffusionXLInpaintPipeline)
-from diffusers.models.unets import unet_2d_condition
 from diffusers.loaders.single_file_utils import load_single_file_checkpoint,infer_diffusers_model_type
-from huggingface_hub import hf_hub_download
 
 from .hidiffusion.hidiffusion import apply_hidiffusion,remove_hidiffusion
 import folder_paths
-from safetensors.torch import load_file, load
+from safetensors.torch import load_file
 import yaml
 import diffusers
 import random
+from omegaconf import OmegaConf
+from comfy.model_management import cleanup_models
+from comfy.clip_vision import load as load_clip
 
 dif_version = str(diffusers.__version__)
 dif_version_int = int(dif_version.split(".")[1])
@@ -76,7 +76,6 @@ lcm_unet = ["dmd2_sdxl_4step_unet_fp16.bin", "dmd2_sdxl_1step_unet_fp16.bin", "l
             "Hyper-SDXL-1step-Unet.safetensors"]
 
 def tensor_to_image(tensor):
-    #tensor = tensor.cpu()
     image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
     image = Image.fromarray(image_np, mode='RGB')
     return image
@@ -161,31 +160,6 @@ def get_sheduler(name):
     return scheduler
 
 
-def get_local_path(file_path, model_path):
-    path = os.path.join(file_path, "models", "diffusers", model_path)
-    model_path = os.path.normpath(path)
-    if sys.platform == 'win32':
-        model_path = model_path.replace('\\', "/")
-    return model_path
-
-
-def get_instance_path(path):
-    instance_path = os.path.normpath(path)
-    if sys.platform == 'win32':
-        instance_path = instance_path.replace('\\', "/")
-    return instance_path
-
-
-def instance_path(path, repo):
-    if repo == "":
-        if path == "none":
-            repo = "none"
-        else:
-            model_path = get_local_path(file_path, path)
-            repo = get_instance_path(model_path)
-    return repo
-
-
 class HI_Diffusers_Model_Loader:
     def __init__(self):
         pass
@@ -204,7 +178,9 @@ class HI_Diffusers_Model_Loader:
                 "trigger_words": ("STRING", {"default": "best quality"}),
                 "scheduler": (scheduler_list,),
                 "apply_window_attn":("BOOLEAN", {"default": False},),
-                "ip_adapter": ("BOOLEAN", {"default": False},),
+                "ip_ckpt": (["none"] + folder_paths.get_filename_list("photomaker"),),
+                "clip_vision": (["none"] + folder_paths.get_filename_list("clip_vision"),),
+                
             }
         }
      
@@ -215,7 +191,7 @@ class HI_Diffusers_Model_Loader:
     CATEGORY = "Hidiffusion_Pro"
 
     def loader_models(self,function_choice, ckpt_name,vae_id,unet_model, controlnet_model,
-                      lora,lora_scale,trigger_words,scheduler,apply_window_attn,ip_adapter):
+                      lora,lora_scale,trigger_words,scheduler,apply_window_attn,ip_ckpt,clip_vision):
         
         ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name) if ckpt_name!="none" else None
         sd_type=""
@@ -223,22 +199,15 @@ class HI_Diffusers_Model_Loader:
             sd = load_single_file_checkpoint(ckpt_path)
             sd_type = infer_diffusers_model_type(sd)
             del sd
-            
-        
+    
         vae_id=vae_id if vae_id!="none" else None
         controlnet_path=folder_paths.get_full_path("controlnet", controlnet_model) if controlnet_model!="none" else None
         unet_ckpt = folder_paths.get_full_path("unet", unet_model)  if unet_model!="none" else None
+        ip_ckpt = folder_paths.get_full_path("photomaker", ip_ckpt) if ip_ckpt != "none" else None
+        clip_vision = folder_paths.get_full_path("clip_vision", clip_vision) if clip_vision != "none" else None
         
         scheduler_used = get_sheduler(scheduler)
         
-        ip_path = os.path.join(folder_paths.models_dir, "photomaker")
-        sdxl_models_pre = os.path.join(ip_path, "sdxl_models", "image_encoder")
-        if not os.path.exists(sdxl_models_pre):
-            os.makedirs(sdxl_models_pre)
-        sd_models_pre = os.path.join(ip_path, "models", "image_encoder")
-        if not os.path.exists(sd_models_pre):
-            os.makedirs(sd_models_pre)
-            
         if sd_type == "v1" or sd_type == "v2":
             model_type="stable-diffusion-v1-5"
             model_config=os.path.join(dir_path,"sd15_config")
@@ -281,7 +250,7 @@ class HI_Diffusers_Model_Loader:
             
             if dif_version_int >= 28:
                 model = StableDiffusionXLPipeline.from_single_file(
-                    pretrained_model_link_or_path=ckpt_path, config=model_config,original_config=original_config_file, torch_dtype=torch.float16)
+                    ckpt_path, config=model_config,original_config=original_config_file, torch_dtype=torch.float16)
             else:
                 model = StableDiffusionXLPipeline.from_single_file(
                     ckpt_path,config=model_config, original_config_file=original_config_file, torch_dtype=torch.float16)
@@ -290,6 +259,7 @@ class HI_Diffusers_Model_Loader:
                 controlnet = ControlNetModel.from_unet(model.unet)
                 cn_state_dict = load_file(controlnet_path)
                 controlnet.load_state_dict(cn_state_dict, strict=False)
+                controlnet.to(torch.float16)
                 if function_choice == "img2img":
                     model = StableDiffusionXLControlNetImg2ImgPipeline.from_single_file(ckpt_path,config=model_config,original_config=original_config_file,controlnet=controlnet,
                                                                                        torch_dtype=torch.float16,
@@ -324,7 +294,6 @@ class HI_Diffusers_Model_Loader:
         
         if lora!="none":
             lora_path = folder_paths.get_full_path("loras", lora)
-            lora_path = get_instance_path(lora_path)
             model.load_lora_weights(lora_path, adapter_name=trigger_words)
             model.fuse_lora(lora_scale=lora_scale, adapter_names=[trigger_words,])
        
@@ -332,66 +301,25 @@ class HI_Diffusers_Model_Loader:
         model.enable_vae_tiling()
         apply_hidiffusion(model,apply_window_attn=apply_window_attn,model_type_str=model_type)
         model.enable_model_cpu_offload()  # need below apply_hidiffusion(model)
-     
-        if ip_adapter:
+        ip_adapter = False
+        if ip_ckpt is not None and clip_vision is not None:
             model.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2)
-            ip_model_dir_origin = get_instance_path(ip_path)
             device = "cuda"
-            if model_type == "stable-diffusion-xl-base-1.0":  # #ip-adapter_sdxl.bin
-                ip_model_path_xl = os.path.join(ip_path, "sdxl_models", "ip-adapter_sdxl.bin")
-               
-                if not os.path.exists(ip_model_path_xl):
-                    adapter_path_xl = hf_hub_download(
-                        repo_id="h94/IP-Adapter",
-                        subfolder="sdxl_models",
-                        filename="ip-adapter_sdxl.bin",
-                        local_dir=ip_model_dir_origin,
-                    )
-                else:
-                    adapter_path_xl = get_instance_path(ip_model_path_xl)
-                image_encoder_path_xl = get_instance_path(
-                    os.path.join(ip_path, "sdxl_models", "image_encoder", "model.safetensors"))
-                if not os.path.exists(ip_model_path_xl):
-                    os.makedirs(ip_model_path_xl)
-                ip_model_path_dir = get_instance_path(os.path.join(ip_path, "sdxl_models", "image_encoder"))
-                if not os.path.exists(image_encoder_path_xl):
-                    adapter_encoder_xl = hf_hub_download(
-                        repo_id="h94/IP-Adapter",
-                        subfolder="sdxl_models/image_encoder",
-                        local_dir=ip_model_dir_origin,
-                    )
-                else:
-                    adapter_encoder_xl = get_instance_path(ip_model_path_dir)
-                model = IPAdapterXL(model, adapter_encoder_xl, adapter_path_xl, device,
-                                    target_blocks=["up_blocks.0.attentions.1"])
-
-            elif model_type == "stable-diffusion-v1-5":  # SD1.5 ip-adapter_sd15.bin
-                # sd1.5
-                ip_model_path = os.path.join(ip_path, "models", "ip-adapter_sd15.bin")
-                if not os.path.exists(ip_model_path):
-                    adapter_path_sd = hf_hub_download(
-                        repo_id="h94/IP-Adapter",
-                        subfolder="models",
-                        filename="ip-adapter_sd15.bin",
-                        local_dir=ip_model_dir_origin,
-                    )
-                else:
-                    adapter_path_sd = get_instance_path(ip_model_path)
-
-                image_encoder_path = get_instance_path(
-                    os.path.join(ip_path, "models", "image_encoder", "model.safetensors"))
-                ip_model_encoder_dir = get_instance_path(os.path.join(ip_path, "models", "image_encoder"))
-                if not os.path.exists(image_encoder_path):
-                    adapter_encoder = hf_hub_download(
-                        repo_id="h94/IP-Adapter",
-                        subfolder="models/image_encoder",
-                        filename="model.safetensors",
-                        local_dir=ip_model_dir_origin,
-                    )
-                else:
-                    adapter_encoder = get_instance_path(ip_model_encoder_dir)
-                model = IPAdapter(model, adapter_encoder, adapter_path_sd, device, target_blocks=["block"])
+            image_encoder = load_clip(clip_vision)
+            if sd_type == "xl_base":
+                config_path=os.path.join(dir_path,"weights","sdxl","config.json")
+                image_encoder_config = OmegaConf.load(config_path)
+                model = IPAdapterXL(model, image_encoder, ip_ckpt, device,image_encoder_config,
+                                target_blocks=["up_blocks.0.attentions.1"])
+            elif sd_type == "v1":
+                config_path = os.path.join(dir_path, "weights", "sd15","config.json")
+                image_encoder_config = OmegaConf.load(config_path)
+                model = IPAdapter(model, image_encoder, ip_ckpt, device,image_encoder_config, target_blocks=["block"])
+            else:
+                raise "unsupport model,only support SDXL or SD1.5"
             torch.cuda.empty_cache()
+            ip_adapter=True
+            
         torch.cuda.empty_cache()
         pipe={"model":model,"controlnet_path":controlnet_path,"sd_type":sd_type,"lora":lora,"trigger_words":trigger_words,"ip_adapter":ip_adapter,"function_choice":function_choice}
         torch.cuda.empty_cache()
@@ -444,7 +372,7 @@ class Hi_Sampler:
         
         if ip_adapter:
             ip_image = kwargs.get("ip_image")
-            ip_image = input_size_adaptation_output(ip_image, pre_input, width, height)
+            #ip_image = input_size_adaptation_output(ip_image, pre_input, width, height)
             if lora != "none":
                 prompt = prompt + " " + trigger_words
             if  controlnet_path is None:

README.md

@@ -7,19 +7,22 @@ HiDiffusion  From: [link](https://github.com/megvii-research/HiDiffusion)
 Update 
 ----
 
-**09/07**   
-*fix runway error/del repo /auto choice model type  
-*修复runwaybug / 去掉repo加载模型的方式  /自动选择模型的类别   s
+**09/08**   
+* adapter style now using  single file.
+* adapter style改成单体文件模式
+ 
 
 **Previous updates**
+*修复runwaybug / 去掉repo加载模型的方式  /自动选择模型的类别  
 * 增加adapter_style支持，SDXL需求的显存较大，虽然能跑CPU，但是不推荐，会爆显存，SD1.5测试没问题。  
 * 增加 manne加速Lora  
 * 加入controlnet-tile-sdxl的支持，内置图片预处理，默认512尺寸，新增apply_window_attn 条件控制。  
 * 修复节点连接逻辑，现在文生图模式，无需接入image，无controlnet也无需接入control_image   
 * 支持SDXL-lighting\Hyper\LCM\DMD2\的加速Unet，建议适当提高步数；    
 * 基于官方的更新，加入lora支持，需要填关键词；    
-* 加入skip，去掉意义不大的其他参数；          
-             
+* 加入skip，去掉意义不大的其他参数；     
+     
+ *fix runway error/del repo /auto choice model type             
 * Adding adapter_style support, SDXL requires a large amount of graphics memory. Although it can run on CPU, it is not recommended as it may cause graphics memory " explosion". SD1.5 testing is not a problem.   
 * add manne lighting lora  
 * Added support for control net file sdxl, built-in image preprocessing, default size of 512, and added condition control for apply_window_attn.   
@@ -63,25 +66,21 @@ yaml
 |     ├──any SDXL lighting  Unet 
 ```
 3.4 if using adapter style   
-如果不存放对应，运行时会自动下载。  If the corresponding file is not stored, it will be automatically downloaded at runtime.  
+
 ```
 ├── ComfyUI/models/photomaker
-|     ├──models/
 |         ├── ip-adapter_sd15.bin
-|         ├── image_encoder/
-|             ├── config.json
-|             ├── model.safetensors
-|     ├──sdxl_models/
 |         ├── ip-adapter_sdxl.bin
-|         ├── image_encoder/
-|             ├── config.json
-|             ├── model.safetensors
+├── ComfyUI/models/clip_vision
+|             ├── sdxl_model.safetensors  # rename from   sd15/ncoder/model.safetensors
+|             ├── sdm_model.safetensors  # rename from  sdxl/encoder/model.safetensors
+
 ```
 
 4 example
 -----
 
-new workflow example  最新的示例
+new workflow example   new
  ![](https://github.com/smthemex/ComfyUI_HiDiffusion_Pro/blob/main/example/new.png)
 
  sd1.5 using ip_adapter_style  使用ip_adapter_style  

example/new.json

@@ -1,32 +1,144 @@
 {
-  "last_node_id": 5,
-  "last_link_id": 4,
+  "last_node_id": 9,
+  "last_link_id": 10,
   "nodes": [
     {
-      "id": 5,
+      "id": 3,
+      "type": "SaveImage",
+      "pos": [
+        4311,
+        -264
+      ],
+      "size": {
+        "0": 367.2918701171875,
+        "1": 381.46820068359375
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 9,
+          "label": "images"
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        "ComfyUI"
+      ]
+    },
+    {
+      "id": 6,
+      "type": "LoadImage",
+      "pos": [
+        3473,
+        10
+      ],
+      "size": {
+        "0": 315,
+        "1": 314
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            10
+          ],
+          "shape": 3,
+          "label": "IMAGE",
+          "slot_index": 0
+        },
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": null,
+          "shape": 3,
+          "label": "MASK"
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "4.jpg",
+        "image"
+      ]
+    },
+    {
+      "id": 8,
+      "type": "HI_Diffusers_Model_Loader",
+      "pos": [
+        3472,
+        -365
+      ],
+      "size": {
+        "0": 315,
+        "1": 322
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "pipe",
+          "type": "HIDIF_MODEL",
+          "links": [
+            7
+          ],
+          "slot_index": 0,
+          "shape": 3,
+          "label": "pipe"
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "HI_Diffusers_Model_Loader"
+      },
+      "widgets_values": [
+        "img2img",
+        "0SDXL\\juggernautXL_v8Rundiffusion.safetensors",
+        "none",
+        "sdxl_lightning_4step_unet.safetensors",
+        "none",
+        "none",
+        0.8,
+        "best quality",
+        "Euler",
+        false,
+        "none",
+        "none"
+      ]
+    },
+    {
+      "id": 9,
       "type": "Hi_Sampler",
       "pos": [
-        3915,
-        -197
+        3875,
+        -292
       ],
       "size": {
         "0": 400,
         "1": 426
       },
       "flags": {},
-      "order": 1,
+      "order": 2,
       "mode": 0,
       "inputs": [
         {
           "name": "pipe",
           "type": "HIDIF_MODEL",
-          "link": 3,
+          "link": 7,
           "label": "pipe"
         },
         {
           "name": "image",
           "type": "IMAGE",
-          "link": null,
+          "link": 10,
           "label": "image"
         },
         {
@@ -47,7 +159,7 @@
           "name": "image",
           "type": "IMAGE",
           "links": [
-            4
+            9
           ],
           "slot_index": 0,
           "shape": 3,
@@ -63,112 +175,50 @@
         0.5,
         1,
         512,
-        635048020868939,
+        1108821181920656,
         "randomize",
-        30,
-        7.5,
+        12,
+        3,
         2048,
         2048,
         1
       ]
-    },
-    {
-      "id": 4,
-      "type": "HI_Diffusers_Model_Loader",
-      "pos": [
-        3512,
-        -184
-      ],
-      "size": [
-        366.36151183893753,
-        307.7217713270144
-      ],
-      "flags": {},
-      "order": 0,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "pipe",
-          "type": "HIDIF_MODEL",
-          "links": [
-            3
-          ],
-          "slot_index": 0,
-          "shape": 3,
-          "label": "pipe"
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "HI_Diffusers_Model_Loader"
-      },
-      "widgets_values": [
-        "txt2img",
-        "0SDXL\\animagineXLV3_v30.safetensors",
-        "none",
-        "none",
-        "none",
-        "none",
-        0.8,
-        "best quality",
-        "DDIM",
-        false,
-        false
-      ]
-    },
-    {
-      "id": 3,
-      "type": "SaveImage",
-      "pos": [
-        4336,
-        -177
-      ],
-      "size": [
-        367.29185650870204,
-        381.4682013986403
-      ],
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "images",
-          "type": "IMAGE",
-          "link": 4,
-          "label": "images"
-        }
-      ],
-      "properties": {},
-      "widgets_values": [
-        "ComfyUI"
-      ]
     }
   ],
   "links": [
     [
-      3,
-      4,
+      7,
+      8,
       0,
-      5,
+      9,
       0,
       "HIDIF_MODEL"
     ],
     [
-      4,
-      5,
+      9,
+      9,
       0,
       3,
       0,
       "IMAGE"
+    ],
+    [
+      10,
+      6,
+      0,
+      9,
+      1,
+      "IMAGE"
     ]
   ],
   "groups": [],
   "config": {},
   "extra": {
     "ds": {
-      "scale": 1.283902517749519,
+      "scale": 1.0610764609500176,
       "offset": [
-        -3462.543452344591,
-        367.22629242211144
+        -3283.5070365830798,
+        461.2881609587324
       ]
     }
   },

ip_adapter/ip_adapter.py

@@ -9,7 +9,7 @@ from safetensors import safe_open
 from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
 
 from .utils import is_torch2_available, get_generator
-
+from comfy.model_management import cleanup_models
 if is_torch2_available():
     from .attention_processor import (
         AttnProcessor2_0 as AttnProcessor,
@@ -64,20 +64,22 @@ class MLPProjModel(torch.nn.Module):
 
 
 class IPAdapter:
-    def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, num_tokens=4, target_blocks=["block"]):
+    def __init__(self, sd_pipe, image_encoder, ip_ckpt, device,image_encoder_config, num_tokens=4, target_blocks=["block"]):
         self.device = device
-        self.image_encoder_path = image_encoder_path
+        # self.image_encoder_path = image_encoder_path
         self.ip_ckpt = ip_ckpt
         self.num_tokens = num_tokens
         self.target_blocks = target_blocks
-
+        self.image_encoder_config=image_encoder_config
         self.pipe = sd_pipe.to(self.device)
         self.set_ip_adapter()
 
         # load image encoder
-        self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
-            self.device, dtype=torch.float16
-        )
+        # self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
+        #     self.device, dtype=torch.float16
+        # )
+        self.image_encoder = image_encoder.encode_image
+        
         self.clip_image_processor = CLIPImageProcessor()
         # image proj model
         self.image_proj_model = self.init_proj()
@@ -87,7 +89,7 @@ class IPAdapter:
     def init_proj(self):
         image_proj_model = ImageProjModel(
             cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
-            clip_embeddings_dim=self.image_encoder.config.projection_dim,
+            clip_embeddings_dim=self.image_encoder_config["projection_dim"],
             clip_extra_context_tokens=self.num_tokens,
         ).to(self.device, dtype=torch.float16)
         return image_proj_model
@@ -153,17 +155,19 @@ class IPAdapter:
 
     @torch.inference_mode()
     def get_image_embeds(self, pil_image=None, clip_image_embeds=None, content_prompt_embeds=None):
-        if pil_image is not None:
-            if isinstance(pil_image, Image.Image):
-                pil_image = [pil_image]
-            clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
-            clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
+        
+        if isinstance(pil_image, torch.Tensor):
+
+            clip_image_embeds = self.image_encoder(pil_image)["image_embeds"]
+            clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
+            del self.image_encoder
+            cleanup_models(keep_clone_weights_loaded=False)
         else:
             clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
-        
+            clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
         if content_prompt_embeds is not None:
             clip_image_embeds = clip_image_embeds - content_prompt_embeds
-
+            clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
         image_prompt_embeds = self.image_proj_model(clip_image_embeds)
         uncond_image_prompt_embeds = self.image_proj_model(torch.zeros_like(clip_image_embeds))
         return image_prompt_embeds, uncond_image_prompt_embeds
@@ -188,9 +192,9 @@ class IPAdapter:
         **kwargs,
     ):
         self.set_scale(scale)
-
-        if pil_image is not None:
-            num_prompts = 1 if isinstance(pil_image, Image.Image) else len(pil_image)
+        d1, _, _, _ = pil_image.size()
+        if isinstance(pil_image,torch.Tensor) :
+            num_prompts = d1
         else:
             num_prompts = clip_image_embeds.size(0)
 
@@ -256,8 +260,8 @@ class IPAdapterXL(IPAdapter):
         **kwargs,
     ):
         self.set_scale(scale)
-
-        num_prompts = 1 if isinstance(pil_image, Image.Image) else len(pil_image)
+        d1,_,_,_=pil_image.size()
+        num_prompts = d1
 
         if prompt is None:
             prompt = "best quality, high quality"
@@ -313,6 +317,7 @@ class IPAdapterXL(IPAdapter):
 
         self.generator = get_generator(seed, self.device)
         
+        cleanup_models(keep_clone_weights_loaded=False)
         images = self.pipe(
             prompt_embeds=prompt_embeds,
             negative_prompt_embeds=negative_prompt_embeds,

weights/sd15/config.json

@@ -0,0 +1,23 @@
+{
+  "_name_or_path": "./image_encoder",
+  "architectures": [
+    "CLIPVisionModelWithProjection"
+  ],
+  "attention_dropout": 0.0,
+  "dropout": 0.0,
+  "hidden_act": "gelu",
+  "hidden_size": 1280,
+  "image_size": 224,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_norm_eps": 1e-05,
+  "model_type": "clip_vision_model",
+  "num_attention_heads": 16,
+  "num_channels": 3,
+  "num_hidden_layers": 32,
+  "patch_size": 14,
+  "projection_dim": 1024,
+  "torch_dtype": "float16",
+  "transformers_version": "4.28.0.dev0"
+}

weights/sdxl/config.json

@@ -0,0 +1,81 @@
+{
+  "architectures": [
+    "CLIPVisionModelWithProjection"
+  ],
+  "_name_or_path": "",
+  "add_cross_attention": false,
+  "architectures_": null,
+  "attention_dropout": 0.0,
+  "bad_words_ids": null,
+  "begin_suppress_tokens": null,
+  "bos_token_id": null,
+  "chunk_size_feed_forward": 0,
+  "cross_attention_hidden_size": null,
+  "decoder_start_token_id": null,
+  "diversity_penalty": 0.0,
+  "do_sample": false,
+  "dropout": 0.0,
+  "early_stopping": false,
+  "encoder_no_repeat_ngram_size": 0,
+  "eos_token_id": null,
+  "exponential_decay_length_penalty": null,
+  "finetuning_task": null,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "hidden_act": "gelu",
+  "hidden_size": 1664,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+      },
+  "image_size": 224,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "is_decoder": false,
+  "is_encoder_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+      },
+  "layer_norm_eps": 1e-05,
+  "length_penalty": 1.0,
+  "max_length": 20,
+  "min_length": 0,
+  "model_type": "clip_vision_model",
+  "no_repeat_ngram_size": 0,
+  "num_attention_heads": 16,
+  "num_beam_groups": 1,
+  "num_beams": 1,
+  "num_channels": 3,
+  "num_hidden_layers": 48,
+  "num_return_sequences": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_scores": false,
+  "pad_token_id": null,
+  "patch_size": 14,
+  "prefix": null,
+  "problem_type": null,
+  "pruned_heads": {},
+  "remove_invalid_values": false,
+  "repetition_penalty": 1.0,
+  "return_dict": true,
+  "return_dict_in_generate": false,
+  "sep_token_id": null,
+  "suppress_tokens": null,
+  "task_specific_params": null,
+  "temperature": 1.0,
+  "tf_legacy_loss": false,
+  "tie_encoder_decoder": false,
+  "tie_word_embeddings": true,
+  "tokenizer_class": null,
+  "top_k": 50,
+  "top_p": 1.0,
+  "torch_dtype": null,
+  "torchscript": false,
+  "transformers_version": "4.24.0",
+  "typical_p": 1.0,
+  "use_bfloat16": false,
+  "projection_dim": 1280
+}