diff --git a/Hidiffusion_node.py b/Hidiffusion_node.py index a89cf74..1a1e6d8 100644 --- a/Hidiffusion_node.py +++ b/Hidiffusion_node.py @@ -5,7 +5,6 @@ import torch import os from PIL import Image import numpy as np -import sys from diffusers import (StableDiffusionXLPipeline, DiffusionPipeline, DDIMScheduler, ControlNetModel, KDPM2AncestralDiscreteScheduler, LMSDiscreteScheduler, AutoPipelineForInpainting, DPMSolverMultistepScheduler, DPMSolverSinglestepScheduler, @@ -15,16 +14,17 @@ from diffusers import (StableDiffusionXLPipeline, DiffusionPipeline, DDIMSchedul EulerAncestralDiscreteScheduler, UniPCMultistepScheduler, AutoencoderKL, StableDiffusionXLControlNetPipeline, DDPMScheduler, TCDScheduler, LCMScheduler, StableDiffusionPipeline, StableDiffusionControlNetPipeline, StableDiffusionXLInpaintPipeline) -from diffusers.models.unets import unet_2d_condition from diffusers.loaders.single_file_utils import load_single_file_checkpoint,infer_diffusers_model_type -from huggingface_hub import hf_hub_download from .hidiffusion.hidiffusion import apply_hidiffusion,remove_hidiffusion import folder_paths -from safetensors.torch import load_file, load +from safetensors.torch import load_file import yaml import diffusers import random +from omegaconf import OmegaConf +from comfy.model_management import cleanup_models +from comfy.clip_vision import load as load_clip dif_version = str(diffusers.__version__) dif_version_int = int(dif_version.split(".")[1]) @@ -76,7 +76,6 @@ lcm_unet = ["dmd2_sdxl_4step_unet_fp16.bin", "dmd2_sdxl_1step_unet_fp16.bin", "l "Hyper-SDXL-1step-Unet.safetensors"] def tensor_to_image(tensor): - #tensor = tensor.cpu() image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy() image = Image.fromarray(image_np, mode='RGB') return image @@ -161,31 +160,6 @@ def get_sheduler(name): return scheduler -def get_local_path(file_path, model_path): - path = os.path.join(file_path, "models", "diffusers", model_path) - model_path = os.path.normpath(path) - if sys.platform == 'win32': - model_path = model_path.replace('\\', "/") - return model_path - - -def get_instance_path(path): - instance_path = os.path.normpath(path) - if sys.platform == 'win32': - instance_path = instance_path.replace('\\', "/") - return instance_path - - -def instance_path(path, repo): - if repo == "": - if path == "none": - repo = "none" - else: - model_path = get_local_path(file_path, path) - repo = get_instance_path(model_path) - return repo - - class HI_Diffusers_Model_Loader: def __init__(self): pass @@ -204,7 +178,9 @@ class HI_Diffusers_Model_Loader: "trigger_words": ("STRING", {"default": "best quality"}), "scheduler": (scheduler_list,), "apply_window_attn":("BOOLEAN", {"default": False},), - "ip_adapter": ("BOOLEAN", {"default": False},), + "ip_ckpt": (["none"] + folder_paths.get_filename_list("photomaker"),), + "clip_vision": (["none"] + folder_paths.get_filename_list("clip_vision"),), + } } @@ -215,7 +191,7 @@ class HI_Diffusers_Model_Loader: CATEGORY = "Hidiffusion_Pro" def loader_models(self,function_choice, ckpt_name,vae_id,unet_model, controlnet_model, - lora,lora_scale,trigger_words,scheduler,apply_window_attn,ip_adapter): + lora,lora_scale,trigger_words,scheduler,apply_window_attn,ip_ckpt,clip_vision): ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name) if ckpt_name!="none" else None sd_type="" @@ -223,22 +199,15 @@ class HI_Diffusers_Model_Loader: sd = load_single_file_checkpoint(ckpt_path) sd_type = infer_diffusers_model_type(sd) del sd - - + vae_id=vae_id if vae_id!="none" else None controlnet_path=folder_paths.get_full_path("controlnet", controlnet_model) if controlnet_model!="none" else None unet_ckpt = folder_paths.get_full_path("unet", unet_model) if unet_model!="none" else None + ip_ckpt = folder_paths.get_full_path("photomaker", ip_ckpt) if ip_ckpt != "none" else None + clip_vision = folder_paths.get_full_path("clip_vision", clip_vision) if clip_vision != "none" else None scheduler_used = get_sheduler(scheduler) - ip_path = os.path.join(folder_paths.models_dir, "photomaker") - sdxl_models_pre = os.path.join(ip_path, "sdxl_models", "image_encoder") - if not os.path.exists(sdxl_models_pre): - os.makedirs(sdxl_models_pre) - sd_models_pre = os.path.join(ip_path, "models", "image_encoder") - if not os.path.exists(sd_models_pre): - os.makedirs(sd_models_pre) - if sd_type == "v1" or sd_type == "v2": model_type="stable-diffusion-v1-5" model_config=os.path.join(dir_path,"sd15_config") @@ -281,7 +250,7 @@ class HI_Diffusers_Model_Loader: if dif_version_int >= 28: model = StableDiffusionXLPipeline.from_single_file( - pretrained_model_link_or_path=ckpt_path, config=model_config,original_config=original_config_file, torch_dtype=torch.float16) + ckpt_path, config=model_config,original_config=original_config_file, torch_dtype=torch.float16) else: model = StableDiffusionXLPipeline.from_single_file( ckpt_path,config=model_config, original_config_file=original_config_file, torch_dtype=torch.float16) @@ -290,6 +259,7 @@ class HI_Diffusers_Model_Loader: controlnet = ControlNetModel.from_unet(model.unet) cn_state_dict = load_file(controlnet_path) controlnet.load_state_dict(cn_state_dict, strict=False) + controlnet.to(torch.float16) if function_choice == "img2img": model = StableDiffusionXLControlNetImg2ImgPipeline.from_single_file(ckpt_path,config=model_config,original_config=original_config_file,controlnet=controlnet, torch_dtype=torch.float16, @@ -324,7 +294,6 @@ class HI_Diffusers_Model_Loader: if lora!="none": lora_path = folder_paths.get_full_path("loras", lora) - lora_path = get_instance_path(lora_path) model.load_lora_weights(lora_path, adapter_name=trigger_words) model.fuse_lora(lora_scale=lora_scale, adapter_names=[trigger_words,]) @@ -332,66 +301,25 @@ class HI_Diffusers_Model_Loader: model.enable_vae_tiling() apply_hidiffusion(model,apply_window_attn=apply_window_attn,model_type_str=model_type) model.enable_model_cpu_offload() # need below apply_hidiffusion(model) - - if ip_adapter: + ip_adapter = False + if ip_ckpt is not None and clip_vision is not None: model.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2) - ip_model_dir_origin = get_instance_path(ip_path) device = "cuda" - if model_type == "stable-diffusion-xl-base-1.0": # #ip-adapter_sdxl.bin - ip_model_path_xl = os.path.join(ip_path, "sdxl_models", "ip-adapter_sdxl.bin") - - if not os.path.exists(ip_model_path_xl): - adapter_path_xl = hf_hub_download( - repo_id="h94/IP-Adapter", - subfolder="sdxl_models", - filename="ip-adapter_sdxl.bin", - local_dir=ip_model_dir_origin, - ) - else: - adapter_path_xl = get_instance_path(ip_model_path_xl) - image_encoder_path_xl = get_instance_path( - os.path.join(ip_path, "sdxl_models", "image_encoder", "model.safetensors")) - if not os.path.exists(ip_model_path_xl): - os.makedirs(ip_model_path_xl) - ip_model_path_dir = get_instance_path(os.path.join(ip_path, "sdxl_models", "image_encoder")) - if not os.path.exists(image_encoder_path_xl): - adapter_encoder_xl = hf_hub_download( - repo_id="h94/IP-Adapter", - subfolder="sdxl_models/image_encoder", - local_dir=ip_model_dir_origin, - ) - else: - adapter_encoder_xl = get_instance_path(ip_model_path_dir) - model = IPAdapterXL(model, adapter_encoder_xl, adapter_path_xl, device, - target_blocks=["up_blocks.0.attentions.1"]) - - elif model_type == "stable-diffusion-v1-5": # SD1.5 ip-adapter_sd15.bin - # sd1.5 - ip_model_path = os.path.join(ip_path, "models", "ip-adapter_sd15.bin") - if not os.path.exists(ip_model_path): - adapter_path_sd = hf_hub_download( - repo_id="h94/IP-Adapter", - subfolder="models", - filename="ip-adapter_sd15.bin", - local_dir=ip_model_dir_origin, - ) - else: - adapter_path_sd = get_instance_path(ip_model_path) - - image_encoder_path = get_instance_path( - os.path.join(ip_path, "models", "image_encoder", "model.safetensors")) - ip_model_encoder_dir = get_instance_path(os.path.join(ip_path, "models", "image_encoder")) - if not os.path.exists(image_encoder_path): - adapter_encoder = hf_hub_download( - repo_id="h94/IP-Adapter", - subfolder="models/image_encoder", - filename="model.safetensors", - local_dir=ip_model_dir_origin, - ) - else: - adapter_encoder = get_instance_path(ip_model_encoder_dir) - model = IPAdapter(model, adapter_encoder, adapter_path_sd, device, target_blocks=["block"]) + image_encoder = load_clip(clip_vision) + if sd_type == "xl_base": + config_path=os.path.join(dir_path,"weights","sdxl","config.json") + image_encoder_config = OmegaConf.load(config_path) + model = IPAdapterXL(model, image_encoder, ip_ckpt, device,image_encoder_config, + target_blocks=["up_blocks.0.attentions.1"]) + elif sd_type == "v1": + config_path = os.path.join(dir_path, "weights", "sd15","config.json") + image_encoder_config = OmegaConf.load(config_path) + model = IPAdapter(model, image_encoder, ip_ckpt, device,image_encoder_config, target_blocks=["block"]) + else: + raise "unsupport model,only support SDXL or SD1.5" torch.cuda.empty_cache() + ip_adapter=True + torch.cuda.empty_cache() pipe={"model":model,"controlnet_path":controlnet_path,"sd_type":sd_type,"lora":lora,"trigger_words":trigger_words,"ip_adapter":ip_adapter,"function_choice":function_choice} torch.cuda.empty_cache() @@ -444,7 +372,7 @@ class Hi_Sampler: if ip_adapter: ip_image = kwargs.get("ip_image") - ip_image = input_size_adaptation_output(ip_image, pre_input, width, height) + #ip_image = input_size_adaptation_output(ip_image, pre_input, width, height) if lora != "none": prompt = prompt + " " + trigger_words if controlnet_path is None: diff --git a/README.md b/README.md index ae355f7..cf91908 100644 --- a/README.md +++ b/README.md @@ -7,19 +7,22 @@ HiDiffusion From: [link](https://github.com/megvii-research/HiDiffusion) Update ---- -**09/07** -*fix runway error/del repo /auto choice model type -*修复runwaybug / 去掉repo加载模型的方式 /自动选择模型的类别 s +**09/08** +* adapter style now using single file. +* adapter style改成单体文件模式 + **Previous updates** +*修复runwaybug / 去掉repo加载模型的方式 /自动选择模型的类别 * 增加adapter_style支持，SDXL需求的显存较大，虽然能跑CPU，但是不推荐，会爆显存，SD1.5测试没问题。 * 增加 manne加速Lora * 加入controlnet-tile-sdxl的支持，内置图片预处理，默认512尺寸，新增apply_window_attn 条件控制。 * 修复节点连接逻辑，现在文生图模式，无需接入image，无controlnet也无需接入control_image * 支持SDXL-lighting\Hyper\LCM\DMD2\的加速Unet，建议适当提高步数； * 基于官方的更新，加入lora支持，需要填关键词； -* 加入skip，去掉意义不大的其他参数； - +* 加入skip，去掉意义不大的其他参数； + + *fix runway error/del repo /auto choice model type * Adding adapter_style support, SDXL requires a large amount of graphics memory. Although it can run on CPU, it is not recommended as it may cause graphics memory " explosion". SD1.5 testing is not a problem. * add manne lighting lora * Added support for control net file sdxl, built-in image preprocessing, default size of 512, and added condition control for apply_window_attn. @@ -63,25 +66,21 @@ yaml | ├──any SDXL lighting Unet ``` 3.4 if using adapter style -如果不存放对应，运行时会自动下载。 If the corresponding file is not stored, it will be automatically downloaded at runtime. + ``` ├── ComfyUI/models/photomaker -| ├──models/ | ├── ip-adapter_sd15.bin -| ├── image_encoder/ -| ├── config.json -| ├── model.safetensors -| ├──sdxl_models/ | ├── ip-adapter_sdxl.bin -| ├── image_encoder/ -| ├── config.json -| ├── model.safetensors +├── ComfyUI/models/clip_vision +| ├── sdxl_model.safetensors # rename from sd15/ncoder/model.safetensors +| ├── sdm_model.safetensors # rename from sdxl/encoder/model.safetensors + ``` 4 example ----- -new workflow example 最新的示例 +new workflow example new ![](https://github.com/smthemex/ComfyUI_HiDiffusion_Pro/blob/main/example/new.png) sd1.5 using ip_adapter_style 使用ip_adapter_style diff --git a/example/controlnet_img2img.png b/example/controlnet_img2img.png index 6356818..6a038fc 100644 Binary files a/example/controlnet_img2img.png and b/example/controlnet_img2img.png differ diff --git a/example/img2img_lora.png b/example/img2img_lora.png index 4408a7c..f4962e6 100644 Binary files a/example/img2img_lora.png and b/example/img2img_lora.png differ diff --git a/example/lightingUnet.png b/example/lightingUnet.png index 488ba61..76416fa 100644 Binary files a/example/lightingUnet.png and b/example/lightingUnet.png differ diff --git a/example/new.json b/example/new.json index 180a5f6..a05759b 100644 --- a/example/new.json +++ b/example/new.json @@ -1,32 +1,144 @@ { - "last_node_id": 5, - "last_link_id": 4, + "last_node_id": 9, + "last_link_id": 10, "nodes": [ { - "id": 5, + "id": 3, + "type": "SaveImage", + "pos": [ + 4311, + -264 + ], + "size": { + "0": 367.2918701171875, + "1": 381.46820068359375 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9, + "label": "images" + } + ], + "properties": {}, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 6, + "type": "LoadImage", + "pos": [ + 3473, + 10 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 10 + ], + "shape": 3, + "label": "IMAGE", + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3, + "label": "MASK" + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "4.jpg", + "image" + ] + }, + { + "id": 8, + "type": "HI_Diffusers_Model_Loader", + "pos": [ + 3472, + -365 + ], + "size": { + "0": 315, + "1": 322 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "pipe", + "type": "HIDIF_MODEL", + "links": [ + 7 + ], + "slot_index": 0, + "shape": 3, + "label": "pipe" + } + ], + "properties": { + "Node name for S&R": "HI_Diffusers_Model_Loader" + }, + "widgets_values": [ + "img2img", + "0SDXL\\juggernautXL_v8Rundiffusion.safetensors", + "none", + "sdxl_lightning_4step_unet.safetensors", + "none", + "none", + 0.8, + "best quality", + "Euler", + false, + "none", + "none" + ] + }, + { + "id": 9, "type": "Hi_Sampler", "pos": [ - 3915, - -197 + 3875, + -292 ], "size": { "0": 400, "1": 426 }, "flags": {}, - "order": 1, + "order": 2, "mode": 0, "inputs": [ { "name": "pipe", "type": "HIDIF_MODEL", - "link": 3, + "link": 7, "label": "pipe" }, { "name": "image", "type": "IMAGE", - "link": null, + "link": 10, "label": "image" }, { @@ -47,7 +159,7 @@ "name": "image", "type": "IMAGE", "links": [ - 4 + 9 ], "slot_index": 0, "shape": 3, @@ -63,112 +175,50 @@ 0.5, 1, 512, - 635048020868939, + 1108821181920656, "randomize", - 30, - 7.5, + 12, + 3, 2048, 2048, 1 ] - }, - { - "id": 4, - "type": "HI_Diffusers_Model_Loader", - "pos": [ - 3512, - -184 - ], - "size": [ - 366.36151183893753, - 307.7217713270144 - ], - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "pipe", - "type": "HIDIF_MODEL", - "links": [ - 3 - ], - "slot_index": 0, - "shape": 3, - "label": "pipe" - } - ], - "properties": { - "Node name for S&R": "HI_Diffusers_Model_Loader" - }, - "widgets_values": [ - "txt2img", - "0SDXL\\animagineXLV3_v30.safetensors", - "none", - "none", - "none", - "none", - 0.8, - "best quality", - "DDIM", - false, - false - ] - }, - { - "id": 3, - "type": "SaveImage", - "pos": [ - 4336, - -177 - ], - "size": [ - 367.29185650870204, - 381.4682013986403 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 4, - "label": "images" - } - ], - "properties": {}, - "widgets_values": [ - "ComfyUI" - ] } ], "links": [ [ - 3, - 4, + 7, + 8, 0, - 5, + 9, 0, "HIDIF_MODEL" ], [ - 4, - 5, + 9, + 9, 0, 3, 0, "IMAGE" + ], + [ + 10, + 6, + 0, + 9, + 1, + "IMAGE" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 1.283902517749519, + "scale": 1.0610764609500176, "offset": [ - -3462.543452344591, - 367.22629242211144 + -3283.5070365830798, + 461.2881609587324 ] } }, diff --git a/example/sd15ipstyle.png b/example/sd15ipstyle.png index 2ecfb92..9c3dfd5 100644 Binary files a/example/sd15ipstyle.png and b/example/sd15ipstyle.png differ diff --git a/ip_adapter/ip_adapter.py b/ip_adapter/ip_adapter.py index b9646a4..c742934 100644 --- a/ip_adapter/ip_adapter.py +++ b/ip_adapter/ip_adapter.py @@ -9,7 +9,7 @@ from safetensors import safe_open from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection from .utils import is_torch2_available, get_generator - +from comfy.model_management import cleanup_models if is_torch2_available(): from .attention_processor import ( AttnProcessor2_0 as AttnProcessor, @@ -64,20 +64,22 @@ class MLPProjModel(torch.nn.Module): class IPAdapter: - def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, num_tokens=4, target_blocks=["block"]): + def __init__(self, sd_pipe, image_encoder, ip_ckpt, device,image_encoder_config, num_tokens=4, target_blocks=["block"]): self.device = device - self.image_encoder_path = image_encoder_path + # self.image_encoder_path = image_encoder_path self.ip_ckpt = ip_ckpt self.num_tokens = num_tokens self.target_blocks = target_blocks - + self.image_encoder_config=image_encoder_config self.pipe = sd_pipe.to(self.device) self.set_ip_adapter() # load image encoder - self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to( - self.device, dtype=torch.float16 - ) + # self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to( + # self.device, dtype=torch.float16 + # ) + self.image_encoder = image_encoder.encode_image + self.clip_image_processor = CLIPImageProcessor() # image proj model self.image_proj_model = self.init_proj() @@ -87,7 +89,7 @@ class IPAdapter: def init_proj(self): image_proj_model = ImageProjModel( cross_attention_dim=self.pipe.unet.config.cross_attention_dim, - clip_embeddings_dim=self.image_encoder.config.projection_dim, + clip_embeddings_dim=self.image_encoder_config["projection_dim"], clip_extra_context_tokens=self.num_tokens, ).to(self.device, dtype=torch.float16) return image_proj_model @@ -153,17 +155,19 @@ class IPAdapter: @torch.inference_mode() def get_image_embeds(self, pil_image=None, clip_image_embeds=None, content_prompt_embeds=None): - if pil_image is not None: - if isinstance(pil_image, Image.Image): - pil_image = [pil_image] - clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values - clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds + + if isinstance(pil_image, torch.Tensor): + + clip_image_embeds = self.image_encoder(pil_image)["image_embeds"] + clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16) + del self.image_encoder + cleanup_models(keep_clone_weights_loaded=False) else: clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16) - + clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16) if content_prompt_embeds is not None: clip_image_embeds = clip_image_embeds - content_prompt_embeds - + clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16) image_prompt_embeds = self.image_proj_model(clip_image_embeds) uncond_image_prompt_embeds = self.image_proj_model(torch.zeros_like(clip_image_embeds)) return image_prompt_embeds, uncond_image_prompt_embeds @@ -188,9 +192,9 @@ class IPAdapter: **kwargs, ): self.set_scale(scale) - - if pil_image is not None: - num_prompts = 1 if isinstance(pil_image, Image.Image) else len(pil_image) + d1, _, _, _ = pil_image.size() + if isinstance(pil_image,torch.Tensor) : + num_prompts = d1 else: num_prompts = clip_image_embeds.size(0) @@ -256,8 +260,8 @@ class IPAdapterXL(IPAdapter): **kwargs, ): self.set_scale(scale) - - num_prompts = 1 if isinstance(pil_image, Image.Image) else len(pil_image) + d1,_,_,_=pil_image.size() + num_prompts = d1 if prompt is None: prompt = "best quality, high quality" @@ -313,6 +317,7 @@ class IPAdapterXL(IPAdapter): self.generator = get_generator(seed, self.device) + cleanup_models(keep_clone_weights_loaded=False) images = self.pipe( prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, diff --git a/weights/sd15/config.json b/weights/sd15/config.json new file mode 100644 index 0000000..9c841dd --- /dev/null +++ b/weights/sd15/config.json @@ -0,0 +1,23 @@ +{ + "_name_or_path": "./image_encoder", + "architectures": [ + "CLIPVisionModelWithProjection" + ], + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "gelu", + "hidden_size": 1280, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 5120, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 32, + "patch_size": 14, + "projection_dim": 1024, + "torch_dtype": "float16", + "transformers_version": "4.28.0.dev0" +} diff --git a/weights/sdxl/config.json b/weights/sdxl/config.json new file mode 100644 index 0000000..8c8ae3e --- /dev/null +++ b/weights/sdxl/config.json @@ -0,0 +1,81 @@ +{ + "architectures": [ + "CLIPVisionModelWithProjection" + ], + "_name_or_path": "", + "add_cross_attention": false, + "architectures_": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu", + "hidden_size": 1664, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 8192, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-05, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "clip_vision_model", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_channels": 3, + "num_hidden_layers": 48, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 14, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.24.0", + "typical_p": 1.0, + "use_bfloat16": false, + "projection_dim": 1280 +}