nodes/smthemex-ComfyUI_CSGO_Wrapper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3 from smthemex/PR

Browse Source 
add new
This commit is contained in:
smthemex
2024-09-07 11:57:10 +08:00
committed by GitHub
parent e21d705fd1 c17dfa7ee5
commit 9fe8229570
35 changed files with 197655 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CSGO/ip_adapter/__pycache__/__init__.cpython-311.pyc
BIN
View File
Binary file not shown.
CSGO/ip_adapter/__pycache__/attention_processor.cpython-311.pyc
BIN
View File
Binary file not shown.
CSGO/ip_adapter/__pycache__/ip_adapter.cpython-311.pyc
BIN
View File
Binary file not shown.
CSGO/ip_adapter/__pycache__/resampler.cpython-311.pyc
BIN
View File
Binary file not shown.
CSGO/ip_adapter/__pycache__/utils.cpython-311.pyc
BIN
View File
Binary file not shown.
CSGO/ip_adapter/ip_adapter.py
+181 -77
View File
@@ -13,6 +13,12 @@ from .utils import is_torch2_available, get_generator
# import torchvision.transforms.functional as Func
# from .clip_style_models import CSD_CLIP, convert_state_dict
def tensor2pil(tensor):
image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
image = Image.fromarray(image_np, mode='RGB')
return image
if is_torch2_available():
from .attention_processor import (
@@ -73,7 +79,7 @@ class MLPProjModel(torch.nn.Module):
class IPAdapter:
def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, num_tokens=4, target_blocks=["block"]):
def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device,encoder_config, num_tokens=4, target_blocks=["block"]):
self.device = device
self.image_encoder_path = image_encoder_path
self.ip_ckpt = ip_ckpt
@@ -84,19 +90,21 @@ class IPAdapter:
self.set_ip_adapter()
# load image encoder
self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
self.device, dtype=torch.float16
)
# self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
# self.device, dtype=torch.float16
# )
self.clip_image_processor = CLIPImageProcessor()
# image proj model
self.image_proj_model = self.init_proj()
self.load_ip_adapter()
self.image_encoder = image_encoder_path.encode_image # comfy
self.encoder_config = encoder_config
def init_proj(self):
image_proj_model = ImageProjModel(
cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
clip_embeddings_dim=self.image_encoder.config.projection_dim,
clip_embeddings_dim=self.encoder_config["projection_dim"],
clip_extra_context_tokens=self.num_tokens,
).to(self.device, dtype=torch.float16)
return image_proj_model
@@ -248,10 +256,10 @@ class IPAdapter:
class IPAdapter_CS:
def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, num_content_tokens=4,
def __init__(self, sd_pipe, image_encoder_path, ip_ckpt, device, encoder_config,num_content_tokens=4,
num_style_tokens=4,
target_content_blocks=["block"], target_style_blocks=["block"], content_image_encoder_path=None,
controlnet_adapter=False,
controlnet_adapter=False,
controlnet_target_content_blocks=None,
controlnet_target_style_blocks=None,
content_model_resampler=False,
@@ -264,10 +272,9 @@ class IPAdapter_CS:
self.num_style_tokens = num_style_tokens
self.content_target_blocks = target_content_blocks
self.style_target_blocks = target_style_blocks
self.content_model_resampler = content_model_resampler
self.style_model_resampler = style_model_resampler
self.controlnet_adapter = controlnet_adapter
self.controlnet_target_content_blocks = controlnet_target_content_blocks
self.controlnet_target_style_blocks = controlnet_target_style_blocks
@@ -278,20 +285,31 @@ class IPAdapter_CS:
# load image encoder
if content_image_encoder_path is not None:
self.content_image_encoder = AutoModel.from_pretrained(content_image_encoder_path).to(self.device,
dtype=torch.float16)
self.content_image_processor = AutoImageProcessor.from_pretrained(content_image_encoder_path)
else:
self.content_image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
self.device, dtype=torch.float16
)
self.content_image_processor = CLIPImageProcessor()
# model.requires_grad_(False)
# if content_image_encoder_path is not None:
# self.content_image_encoder = AutoModel.from_pretrained(content_image_encoder_path).to(self.device,
# dtype=torch.float16)
# self.content_image_processor = AutoImageProcessor.from_pretrained(content_image_encoder_path)
# else:
# self.content_image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
# self.device, dtype=torch.float16
# )
# self.content_image_processor = CLIPImageProcessor()
# # model.requires_grad_(False)
#
# self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
# self.device, dtype=torch.float16
# )
self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(self.image_encoder_path).to(
self.device, dtype=torch.float16
)
self.image_encoder=image_encoder_path.encode_image #comfy
self.clip_image_processor = CLIPImageProcessor()
self.content_image_encoder=image_encoder_path.encode_image #comfy
self.content_image_processor = CLIPImageProcessor()
self.encoder_config = encoder_config
# image_encoder_projection_dim = image_encoder_config["vision_config"]["projection_dim"]
# if self.use_CSD is not None:
# self.style_image_encoder = CSD_CLIP("vit_large", "default",self.use_CSD+"/ViT-L-14.pt")
# model_path = self.use_CSD+"/checkpoint.pth"
@@ -307,7 +325,7 @@ class IPAdapter_CS:
# normalize,
# ])
self.clip_image_processor = CLIPImageProcessor()
# image proj model
self.content_image_proj_model = self.init_proj(self.num_content_tokens, content_or_style_='content',
model_resampler=self.content_model_resampler)
@@ -319,17 +337,29 @@ class IPAdapter_CS:
def init_proj(self, num_tokens, content_or_style_='content', model_resampler=False):
# print('@@@@',self.pipe.unet.config.cross_attention_dim,self.image_encoder.config.projection_dim)
# if content_or_style_ == 'content' and self.content_image_encoder_path is not None:
# image_proj_model = ImageProjModel(
# cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
# clip_embeddings_dim=self.content_image_encoder.config.projection_dim,
# clip_extra_context_tokens=num_tokens,
# ).to(self.device, dtype=torch.float16)
# return image_proj_model
# image_proj_model = ImageProjModel(
# cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
# clip_embeddings_dim=self.image_encoder.config.projection_dim,
# clip_extra_context_tokens=num_tokens,
# ).to(self.device, dtype=torch.float16)
if content_or_style_ == 'content' and self.content_image_encoder_path is not None:
image_proj_model = ImageProjModel(
cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
clip_embeddings_dim=self.content_image_encoder.config.projection_dim,
clip_embeddings_dim=self.encoder_config["projection_dim"],
clip_extra_context_tokens=num_tokens,
).to(self.device, dtype=torch.float16)
return image_proj_model
image_proj_model = ImageProjModel(
cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
clip_embeddings_dim=self.image_encoder.config.projection_dim,
clip_embeddings_dim=self.encoder_config["projection_dim"],
clip_extra_context_tokens=num_tokens,
).to(self.device, dtype=torch.float16)
return image_proj_model
@@ -517,36 +547,57 @@ class IPAdapter_CS:
# clip_image_embeds = clip_image_embeds - content_prompt_embeds
if content_or_style_ == 'content':
if pil_image is not None:
if isinstance(pil_image, Image.Image):
pil_image = [pil_image]
# if pil_image is not None:
# if isinstance(pil_image, Image.Image):
# pil_image = [pil_image]
# if self.content_image_proj_model is not None:
# clip_image = self.content_image_processor(images=pil_image, return_tensors="pt").pixel_values
# clip_image_embeds = self.content_image_encoder(
# clip_image.to(self.device, dtype=torch.float16)).image_embeds
# else:
# clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
# clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
if isinstance(pil_image, torch.Tensor):
pil_image = pil_image
if self.content_image_proj_model is not None:
clip_image = self.content_image_processor(images=pil_image, return_tensors="pt").pixel_values
clip_image_embeds = self.content_image_encoder(
clip_image.to(self.device, dtype=torch.float16)).image_embeds
# clip_image = self.content_image_processor(images=pil_image, return_tensors="pt").pixel_values
# clip_image_embeds = self.content_image_encoder(
# clip_image.to(self.device, dtype=torch.float16)).image_embeds
clip_image_embeds = self.image_encoder(pil_image)["image_embeds"]
#print(clip_image_embeds.shape,11)
else:
clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
#clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
#clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
clip_image_embeds=self.image_encoder(pil_image)["image_embeds"]
#print(clip_image_embeds.shape,12)
else:
clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
#clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
#print(clip_image_embeds.shape, 13)
image_prompt_embeds = self.content_image_proj_model(clip_image_embeds)
uncond_image_prompt_embeds = self.content_image_proj_model(torch.zeros_like(clip_image_embeds))
return image_prompt_embeds, uncond_image_prompt_embeds
if content_or_style_ == 'style':
if pil_image is not None:
# if pil_image is not None:
if isinstance(pil_image, torch.Tensor):
if self.use_CSD is not None:
clip_image = self.style_preprocess(pil_image).unsqueeze(0).to(self.device, dtype=torch.float32)
clip_image_embeds = self.style_image_encoder(clip_image)
# clip_image = self.style_preprocess(pil_image).unsqueeze(0).to(self.device, dtype=torch.float32)
# clip_image_embeds = self.style_image_encoder(clip_image)
clip_image_embeds = self.image_encoder(pil_image)["penultimate_hidden_states"]
#print(clip_image_embeds.shape,2)
else:
if isinstance(pil_image, Image.Image):
pil_image = [pil_image]
clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
# if isinstance(pil_image, Image.Image):
# pil_image = [pil_image]
# clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
# clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
clip_image_embeds = self.image_encoder(pil_image)["image_embeds"]
print(clip_image_embeds.shape,3)
else:
clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
#clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
print(clip_image_embeds.shape, 4)
image_prompt_embeds = self.style_image_proj_model(clip_image_embeds)
uncond_image_prompt_embeds = self.style_image_proj_model(torch.zeros_like(clip_image_embeds))
return image_prompt_embeds, uncond_image_prompt_embeds
@@ -589,9 +640,10 @@ class IPAdapter_CS:
**kwargs,
):
self.set_scale(content_scale, style_scale)
if pil_content_image is not None:
num_prompts = 1 if isinstance(pil_content_image, Image.Image) else len(pil_content_image)
d1, _, _, _ = pil_content_image.size()
if isinstance(pil_content_image,torch.Tensor) :
num_prompts = d1
else:
num_prompts = clip_content_image_embeds.size(0)
@@ -678,8 +730,8 @@ class IPAdapterXL_CS(IPAdapter_CS):
):
self.set_scale(content_scale, style_scale)
self.device=device
num_prompts = 1 if isinstance(pil_content_image, Image.Image) else len(pil_content_image)
d1, _, _, _ = pil_content_image.size()
num_prompts =d1
if prompt is None:
prompt = "best quality, high quality"
@@ -700,7 +752,10 @@ class IPAdapterXL_CS(IPAdapter_CS):
style_image_prompt_embeds, uncond_style_image_prompt_embeds = self.get_image_embeds(pil_style_image,
style_image_embeds,
content_or_style_='style')
torch.cuda.empty_cache()
bs_embed, seq_len, _ = content_image_prompt_embeds.shape
content_image_prompt_embeds = content_image_prompt_embeds.repeat(1, num_samples, 1)
@@ -753,6 +808,24 @@ class CSGO(IPAdapterXL_CS):
def init_proj(self, num_tokens, content_or_style_='content', model_resampler=False):
if content_or_style_ == 'content':
# if model_resampler:
# image_proj_model = Resampler(
# dim=self.pipe.unet.config.cross_attention_dim,
# depth=4,
# dim_head=64,
# heads=12,
# num_queries=num_tokens,
# embedding_dim=self.content_image_encoder.config.hidden_size,
# output_dim=self.pipe.unet.config.cross_attention_dim,
# ff_mult=4,
# ).to(self.device, dtype=torch.float16)
# else:
# image_proj_model = ImageProjModel(
# cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
# clip_embeddings_dim=self.image_encoder.config.projection_dim,
# clip_extra_context_tokens=num_tokens,
# ).to(self.device, dtype=torch.float16)
if model_resampler:
image_proj_model = Resampler(
dim=self.pipe.unet.config.cross_attention_dim,
@@ -760,16 +833,34 @@ class CSGO(IPAdapterXL_CS):
dim_head=64,
heads=12,
num_queries=num_tokens,
embedding_dim=self.content_image_encoder.config.hidden_size,
embedding_dim=self.encoder_config["hidden_size"],
output_dim=self.pipe.unet.config.cross_attention_dim,
ff_mult=4,
).to(self.device, dtype=torch.float16)
else:
image_proj_model = ImageProjModel(
cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
clip_embeddings_dim=self.image_encoder.config.projection_dim,
clip_embeddings_dim=self.encoder_config["projection_dim"],
clip_extra_context_tokens=num_tokens,
).to(self.device, dtype=torch.float16)
# if content_or_style_ == 'style':
# if model_resampler:
# image_proj_model = Resampler(
# dim=self.pipe.unet.config.cross_attention_dim,
# depth=4,
# dim_head=64,
# heads=12,
# num_queries=num_tokens,
# embedding_dim=self.content_image_encoder.config.hidden_size,
# output_dim=self.pipe.unet.config.cross_attention_dim,
# ff_mult=4,
# ).to(self.device, dtype=torch.float16)
# else:
# image_proj_model = ImageProjModel(
# cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
# clip_embeddings_dim=self.image_encoder.config.projection_dim,
# clip_extra_context_tokens=num_tokens,
# ).to(self.device, dtype=torch.float16)
if content_or_style_ == 'style':
if model_resampler:
image_proj_model = Resampler(
@@ -778,47 +869,55 @@ class CSGO(IPAdapterXL_CS):
dim_head=64,
heads=12,
num_queries=num_tokens,
embedding_dim=self.content_image_encoder.config.hidden_size,
embedding_dim=self.encoder_config["hidden_size"],
output_dim=self.pipe.unet.config.cross_attention_dim,
ff_mult=4,
).to(self.device, dtype=torch.float16)
else:
image_proj_model = ImageProjModel(
cross_attention_dim=self.pipe.unet.config.cross_attention_dim,
clip_embeddings_dim=self.image_encoder.config.projection_dim,
clip_embeddings_dim=self.encoder_config["projection_dim"],
clip_extra_context_tokens=num_tokens,
).to(self.device, dtype=torch.float16)
return image_proj_model
@torch.inference_mode()
def get_image_embeds(self, pil_image=None, clip_image_embeds=None, content_or_style_=''):
if isinstance(pil_image, Image.Image):
pil_image = [pil_image]
if content_or_style_ == 'style':
if isinstance(pil_image, Image.Image):
pil_image = [pil_image]
if isinstance(pil_image, torch.Tensor):
pil_image = pil_image
if self.style_model_resampler:
clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16),
output_hidden_states=True).hidden_states[-2]
# clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
# clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16),
# output_hidden_states=True).hidden_states[-2]
clip_image_embeds=self.image_encoder(pil_image)["penultimate_hidden_states"]
clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
image_prompt_embeds = self.style_image_proj_model(clip_image_embeds)
uncond_image_prompt_embeds = self.style_image_proj_model(torch.zeros_like(clip_image_embeds))
else:
clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
# clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
# clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
clip_image_embeds = self.image_encoder(pil_image)["image_embeds"]
clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
image_prompt_embeds = self.style_image_proj_model(clip_image_embeds)
uncond_image_prompt_embeds = self.style_image_proj_model(torch.zeros_like(clip_image_embeds))
return image_prompt_embeds, uncond_image_prompt_embeds
else:
if isinstance(pil_image, torch.Tensor):
pil_image = pil_image
if self.content_image_encoder_path is not None:
clip_image = self.content_image_processor(images=pil_image, return_tensors="pt").pixel_values
outputs = self.content_image_encoder(clip_image.to(self.device, dtype=torch.float16),
output_hidden_states=True)
clip_image_embeds = outputs.last_hidden_state
# clip_image = self.content_image_processor(images=pil_image, return_tensors="pt").pixel_values
# outputs = self.content_image_encoder(clip_image.to(self.device, dtype=torch.float16),
# output_hidden_states=True)
# clip_image_embeds = outputs.last_hidden_state
clip_image_embeds = self.image_encoder(pil_image)["last_hidden_state"]
clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
image_prompt_embeds = self.content_image_proj_model(clip_image_embeds)
# uncond_clip_image_embeds = self.image_encoder(
@@ -830,10 +929,13 @@ class CSGO(IPAdapterXL_CS):
else:
if self.content_model_resampler:
clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
clip_image = clip_image.to(self.device, dtype=torch.float16)
clip_image_embeds = self.image_encoder(clip_image, output_hidden_states=True).hidden_states[-2]
# clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
#
# clip_image = clip_image.to(self.device, dtype=torch.float16)
# clip_image_embeds = self.image_encoder(clip_image, output_hidden_states=True).hidden_states[-2]
clip_image_embeds = self.image_encoder(pil_image)["penultimate_hidden_states"]
clip_image_embeds = clip_image_embeds.clone().detach().to(self.device,dtype=torch.float16)
# clip_image_embeds = clip_image_embeds.to(self.device, dtype=torch.float16)
image_prompt_embeds = self.content_image_proj_model(clip_image_embeds)
# uncond_clip_image_embeds = self.image_encoder(
@@ -841,11 +943,13 @@ class CSGO(IPAdapterXL_CS):
# ).hidden_states[-2]
uncond_image_prompt_embeds = self.content_image_proj_model(torch.zeros_like(clip_image_embeds))
else:
clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
# clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
# clip_image_embeds = self.image_encoder(clip_image.to(self.device, dtype=torch.float16)).image_embeds
clip_image_embeds = self.image_encoder(pil_image)["image_embeds"]
clip_image_embeds = clip_image_embeds.clone().detach().to(self.device, dtype=torch.float16)
image_prompt_embeds = self.content_image_proj_model(clip_image_embeds)
uncond_image_prompt_embeds = self.content_image_proj_model(torch.zeros_like(clip_image_embeds))
return image_prompt_embeds, uncond_image_prompt_embeds
# # clip_image = self.clip_image_processor(images=pil_image, return_tensors="pt").pixel_values
README.md
+24 -17
View File
@@ -3,6 +3,12 @@ You can using InstantX's CSGO in comfyUI
**CSGO From: [link](https://github.com/instantX-research/CSGO)**
Update
---
**2024/09/07**
* fix runway loader error, using single clip_vision weight now；
* 修复runway跑路导致的diffuser加载报错，限制直接使用IP的SDXL 图片解码模型，请放在clip_vision目录下；
1.Installation
-----
@@ -19,10 +25,16 @@ pip install -r requirements.txt
```
3 Need model
----
3.1 base SDXl ckpt
3.1 base SDXl ckpt and vae and clip_vision
h94/IP-Adapter/sdxl_models [link](https://huggingface.co/h94/IP-Adapter/tree/main/sdxl_models/image_encoder)
no need config
```
├── ComfyUI/models/checkpoints/
| ├──any SDXL weights #for example: Jumpernaut XL_v9-RunDiffusionPhoto_v2.safetensors
| ├──any SDXL weights # for example: Jumpernaut XL_v9-RunDiffusionPhoto_v2.safetensors
├── ComfyUI/models/vae/
| ├──any SDXL vae weights # sdxl.vae.safetensors
├── ComfyUI/models/clip_vision/
| ├──model.safetensors # h94/IP-Adapter/sdxl_models/model.safetensors
```
3.2 main ckpt
CSGO models [link](https://huggingface.co/InstantX/CSGO/tree/main)
@@ -31,18 +43,7 @@ CSGO models [link](https://huggingface.co/InstantX/CSGO/tree/main)
| ├──acsgo_4_32.bin #need token 4/32
| ├──acsgo.bin #need token 4/16
```
3.3 base vae
```
├── ComfyUI/models/vae/
| ├──any SDXL vae weights #sdxl.vae.safetensors
```
3.4 image encoder
h94/IP-Adapter/sdxl_models [link](https://huggingface.co/h94/IP-Adapter/tree/main/sdxl_models/image_encoder)
```
├── any path dir/
| ├── model.safetensors
| ├── config.json
```
3.5 controlnet
TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic [link](https://huggingface.co/TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic)
```
@@ -50,7 +51,7 @@ TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic [link](https://huggingface.co/T
| ├── diffusion_pytorch_model.safetensors #rename from "TTPLANET_Controlnet_Tile_realistic_v2_fp16.safetensors"
| ├── config.json
```
3.6 if using LLM
3.6 if using LLM (unnecessary)
Salesforce/blip-image-captioning-large [link](https://huggingface.co/Salesforce/blip-image-captioning-large/tree/main)
```
├── any path dir/
@@ -62,10 +63,16 @@ Salesforce/blip-image-captioning-large [link](https://huggingface.co/Salesforce
| ├── tokenizer_config.json
| ├── vocab.txt
```
or change prompt input using any other LL.
4 Example
----
![](https://github.com/smthemex/ComfyUI_CSGO_Wrapper/blob/main/example/exampleA.png)
![](https://github.com/smthemex/ComfyUI_CSGO_Wrapper/blob/main/example/example.png)
conternt+style img
![](https://github.com/smthemex/ComfyUI_CSGO_Wrapper/blob/main/example/content_style_img.png)
style img + prompt
![](https://github.com/smthemex/ComfyUI_CSGO_Wrapper/blob/main/example/txt_only.png)
conternt+style img +llm
![](https://github.com/smthemex/ComfyUI_CSGO_Wrapper/blob/main/example/using blip.png)
5 Function Description of Nodes
__pycache__/__init__.cpython-311.pyc
BIN
View File
Binary file not shown.
__pycache__/csgo_node.cpython-311.pyc
BIN
View File
Binary file not shown.
configs/config.json
+81
View File
@@ -0,0 +1,81 @@
{
"architectures": [
"CLIPVisionModelWithProjection"
],
"_name_or_path": "",
"add_cross_attention": false,
"architectures_": null,
"attention_dropout": 0.0,
"bad_words_ids": null,
"begin_suppress_tokens": null,
"bos_token_id": null,
"chunk_size_feed_forward": 0,
"cross_attention_hidden_size": null,
"decoder_start_token_id": null,
"diversity_penalty": 0.0,
"do_sample": false,
"dropout": 0.0,
"early_stopping": false,
"encoder_no_repeat_ngram_size": 0,
"eos_token_id": null,
"exponential_decay_length_penalty": null,
"finetuning_task": null,
"forced_bos_token_id": null,
"forced_eos_token_id": null,
"hidden_act": "gelu",
"hidden_size": 1664,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"image_size": 224,
"initializer_factor": 1.0,
"initializer_range": 0.02,
"intermediate_size": 8192,
"is_decoder": false,
"is_encoder_decoder": false,
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"layer_norm_eps": 1e-05,
"length_penalty": 1.0,
"max_length": 20,
"min_length": 0,
"model_type": "clip_vision_model",
"no_repeat_ngram_size": 0,
"num_attention_heads": 16,
"num_beam_groups": 1,
"num_beams": 1,
"num_channels": 3,
"num_hidden_layers": 48,
"num_return_sequences": 1,
"output_attentions": false,
"output_hidden_states": false,
"output_scores": false,
"pad_token_id": null,
"patch_size": 14,
"prefix": null,
"problem_type": null,
"pruned_heads": {},
"remove_invalid_values": false,
"repetition_penalty": 1.0,
"return_dict": true,
"return_dict_in_generate": false,
"sep_token_id": null,
"suppress_tokens": null,
"task_specific_params": null,
"temperature": 1.0,
"tf_legacy_loss": false,
"tie_encoder_decoder": false,
"tie_word_embeddings": true,
"tokenizer_class": null,
"top_k": 50,
"top_p": 1.0,
"torch_dtype": null,
"torchscript": false,
"transformers_version": "4.24.0",
"typical_p": 1.0,
"use_bfloat16": false,
"projection_dim": 1280
}
csgo_node.py
+37 -24
View File
@@ -3,10 +3,8 @@
import os
import numpy as np
import torch
import cv2
from PIL import Image
from transformers import AutoImageProcessor, AutoModel
from omegaconf import OmegaConf
from diffusers import (
AutoencoderKL,
ControlNetModel,
@@ -19,12 +17,16 @@ from .CSGO.ip_adapter.utils import BLOCKS as BLOCKS
from .CSGO.ip_adapter.utils import controlnet_BLOCKS as controlnet_BLOCKS
from .CSGO.ip_adapter.utils import resize_content
from .CSGO.ip_adapter import CSGO
MAX_SEED = np.iinfo(np.int32).max
import folder_paths
from comfy.utils import common_upscale
from comfy.clip_vision import load
MAX_SEED = np.iinfo(np.int32).max
node_cur_path = os.path.dirname(os.path.abspath(__file__))
#device = torch.device("cuda" if torch.cuda.is_available() else "cpu") #cuda:0
device = (
"cuda"
if torch.cuda.is_available()
@@ -69,7 +71,7 @@ class Blip_Loader:
RETURN_TYPES = ("MODEL", "MODEL",)
RETURN_NAMES = ("blip_processor", "blip_model",)
FUNCTION = "test"
CATEGORY = "CSGO_Example_Test"
CATEGORY = "CSGO_Wrapper"
def test(self, blip_repo,):
blip_processor = BlipProcessor.from_pretrained(blip_repo)
@@ -86,7 +88,7 @@ class CSGO_Loader:
return {
"required": {
"base_cpkt":(["none"]+folder_paths.get_filename_list("checkpoints"),),
"image_encoder_repo": ("STRING", {"default": "h94/IP-Adapter"}),
"clip_vision":(["none"]+folder_paths.get_filename_list("clip_vision"),),
"vae_id":(["none"]+folder_paths.get_filename_list("vae"),),
"controlnet_repo": ("STRING", {"default": "TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic" }),
"csgo_ckpt": (["none"]+folder_paths.get_filename_list("checkpoints"),),
@@ -110,21 +112,22 @@ class CSGO_Loader:
RETURN_TYPES = ("MODEL",)
RETURN_NAMES = ("csgo",)
FUNCTION = "test"
CATEGORY = "CSGO_Example_Test"
CATEGORY = "CSGO_Wrapper"
def test(self,base_cpkt,image_encoder_repo,vae_id,controlnet_repo,csgo_ckpt,num_content_tokens,num_style_tokens):
def test(self,base_cpkt,clip_vision,vae_id,controlnet_repo,csgo_ckpt,num_content_tokens,num_style_tokens):
if csgo_ckpt=="none" or base_cpkt=="none" :
raise "need weight"
csgo_ckpt=folder_paths.get_full_path("checkpoints",csgo_ckpt) #获取绝对路径
ckpt_path = folder_paths.get_full_path("checkpoints", base_cpkt) # 获取绝对路径
#vae = AutoencoderKL.from_pretrained(vae, torch_dtype=torch.float16)
clip_vision=folder_paths.get_full_path("clip_vision", clip_vision) # 获取绝对路径
img_encoder=load(clip_vision)
model_config = os.path.join(node_cur_path, "local_repo")
original_config_file = os.path.join(node_cur_path, 'configs', 'sd_xl_base.yaml')
controlnet = ControlNetModel.from_pretrained(controlnet_repo, torch_dtype=torch.float16, use_safetensors=True)
vae_id = folder_paths.get_full_path("vae", vae_id)
vae = AutoencoderKL.from_single_file(vae_id, torch_dtype=torch.float16)
model_config = "stabilityai/stable-diffusion-xl-base-1.0"
original_config_file = os.path.join(node_cur_path, 'configs', 'sd_xl_base.yaml')
vae_config = os.path.join(node_cur_path, "local_repo","vae")
vae = AutoencoderKL.from_single_file(vae_id, config=vae_config,torch_dtype=torch.float16)
try:
pipe = StableDiffusionXLControlNetPipeline.from_single_file(
ckpt_path,config=model_config, original_config=original_config_file,vae=vae, controlnet=controlnet,
@@ -138,7 +141,6 @@ class CSGO_Loader:
raise "load pipe error!,check you diffusers"
pipe.enable_vae_tiling()
#pipe.to(device)
pipe.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2)
if device != "mps":
pipe.enable_model_cpu_offload()
@@ -147,8 +149,10 @@ class CSGO_Loader:
target_style_blocks = BLOCKS['style']
controlnet_target_content_blocks = controlnet_BLOCKS['content']
controlnet_target_style_blocks = controlnet_BLOCKS['style']
csgo = CSGO(pipe, image_encoder_repo, csgo_ckpt, device, num_content_tokens=num_content_tokens, num_style_tokens=num_style_tokens,
config_path = os.path.join(node_cur_path, "configs", "config.json")
image_encoder_config = OmegaConf.load(config_path)
csgo = CSGO(pipe, img_encoder, csgo_ckpt, device, image_encoder_config,num_content_tokens=num_content_tokens, num_style_tokens=num_style_tokens,
target_content_blocks=target_content_blocks, target_style_blocks=target_style_blocks,
controlnet_adapter=True,
controlnet_target_content_blocks=controlnet_target_content_blocks,
@@ -220,7 +224,7 @@ class CSGO_Sampler:
"display": "number",
}),
"steps": ("INT", {
"default": 50,
"default": 25,
"min": 1, # Minimum value
"max": 100, # Maximum value
"step": 1, # Slider's step
@@ -251,7 +255,7 @@ class CSGO_Sampler:
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("images",)
FUNCTION = "test"
CATEGORY = "CSGO_Example_Test"
CATEGORY = "CSGO_Wrapper"
def test(self,content_image,style_image, csgo, prompt,negative_prompt,num_sample, width,height,content_scale,style_scale,cfg,steps,seed,controlnet_scale,text_only,**kwargs):
@@ -259,9 +263,15 @@ class CSGO_Sampler:
blip_model=kwargs.get("blip_model",None)
#向量转图片
style_image= tensor2pil(style_image)
#style_image= tensor2pil(style_image)
#content_image= tensor2pil(content_image)
content_image = tensor_upscale2pil(content_image, width, height)
content_image_pil = tensor_upscale2pil(content_image, width, height)
style_image =tensor_upscale(style_image, width, height) # torch tensor
content_image=tensor_upscale(content_image, width, height) # torch tensor
#clip vison
if blip_processor and blip_model: #判断模型是否存在，不存在则跑文本驱动流程
with torch.no_grad():
@@ -274,8 +284,11 @@ class CSGO_Sampler:
else:
caption = prompt
if text_only:
content_image = Image.fromarray(
np.zeros((content_image.size[0], content_image.size[1], 3), dtype=np.uint8)).convert('RGB')
content_image_pil = Image.fromarray(
np.zeros((content_image_pil.size[0], content_image_pil.size[1], 3), dtype=np.uint8)).convert('RGB')
content_image=pil2narry(content_image_pil)
#width, height, content_image = resize_content(content_image)#?
@@ -291,7 +304,7 @@ class CSGO_Sampler:
num_samples=num_sample,
num_inference_steps=steps,
seed=seed,
image=content_image.convert('RGB'),
image= content_image_pil,
controlnet_conditioning_scale=controlnet_scale,
)[0]
images=pil2narry(images)
example/content_style_img.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 545 KiB

example/example.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 489 KiB

example/exampleA.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 494 KiB

example/txt_only.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 523 KiB

example/using blip.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 503 KiB

example/workflow.json
+314
View File
@@ -0,0 +1,314 @@
{
"last_node_id": 10,
"last_link_id": 11,
"nodes": [
{
"id": 10,
"type": "Blip_Loader",
"pos": [
3337,
270
],
"size": {
"0": 315,
"1": 78
},
"flags": {},
"order": 0,
"mode": 4,
"outputs": [
{
"name": "blip_processor",
"type": "MODEL",
"links": null,
"shape": 3,
"label": "blip_processor"
},
{
"name": "blip_model",
"type": "MODEL",
"links": null,
"shape": 3,
"label": "blip_model"
}
],
"properties": {
"Node name for S&R": "Blip_Loader"
},
"widgets_values": [
"Salesforce/blip-image-captioning-large"
]
},
{
"id": 8,
"type": "CSGO_Loader",
"pos": [
3313,
-9
],
"size": {
"0": 365.1694030761719,
"1": 202
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "csgo",
"type": "MODEL",
"links": [
8
],
"slot_index": 0,
"shape": 3,
"label": "csgo"
}
],
"properties": {
"Node name for S&R": "CSGO_Loader"
},
"widgets_values": [
"0SDXL\\sd_xl_base_1.0.safetensors",
"IP_img_encoder_sdxl_model.safetensors",
"sdxl.vae.safetensors",
"F:/test/ComfyUI/models/diffusers/TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic",
"csgo_4_32.bin",
4,
32
]
},
{
"id": 5,
"type": "LoadImage",
"pos": [
2952,
-55
],
"size": {
"0": 315,
"1": 314
},
"flags": {},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
9
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3,
"label": "MASK"
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"girl.jpeg",
"image"
]
},
{
"id": 4,
"type": "LoadImage",
"pos": [
2952,
-434
],
"size": {
"0": 315,
"1": 314
},
"flags": {},
"order": 3,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
10
],
"slot_index": 0,
"shape": 3,
"label": "IMAGE"
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3,
"label": "MASK"
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"3736c69fb7ce5b795ff78a607d3bda37.jpeg",
"image"
]
},
{
"id": 6,
"type": "SaveImage",
"pos": [
4169,
-145
],
"size": {
"0": 400.16595458984375,
"1": 459.12664794921875
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 11,
"label": "images"
}
],
"properties": {},
"widgets_values": [
"ComfyUI"
]
},
{
"id": 9,
"type": "CSGO_Sampler",
"pos": [
3729,
-164
],
"size": {
"0": 400,
"1": 470
},
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "content_image",
"type": "IMAGE",
"link": 10,
"label": "content_image"
},
{
"name": "style_image",
"type": "IMAGE",
"link": 9,
"label": "style_image"
},
{
"name": "csgo",
"type": "MODEL",
"link": 8,
"label": "csgo"
},
{
"name": "blip_processor",
"type": "MODEL",
"link": null,
"label": "blip_processor"
},
{
"name": "blip_model",
"type": "MODEL",
"link": null,
"label": "blip_model"
}
],
"outputs": [
{
"name": "images",
"type": "IMAGE",
"links": [
11
],
"slot_index": 0,
"shape": 3,
"label": "images"
}
],
"properties": {
"Node name for S&R": "CSGO_Sampler"
},
"widgets_values": [
"a girl",
"text, watermark, lowres, low quality, worst quality, deformed, glitch, low contrast, noisy, saturation, blurry",
1,
1024,
1024,
0.5,
1,
10,
25,
919524022,
"randomize",
0.8,
false
]
}
],
"links": [
[
8,
8,
0,
9,
2,
"MODEL"
],
[
9,
5,
0,
9,
1,
"IMAGE"
],
[
10,
4,
0,
9,
0,
"IMAGE"
],
[
11,
9,
0,
6,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.9646149645000137,
"offset": [
-2800.1905049143675,
506.1924603210764
]
}
},
"version": 0.4
}
example/workflow.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1020 KiB

local_repo/model_index.json
+34
View File
@@ -0,0 +1,34 @@
{
"_class_name": "StableDiffusionXLPipeline",
"_diffusers_version": "0.19.0.dev0",
"force_zeros_for_empty_prompt": true,
"add_watermarker": null,
"scheduler": [
"diffusers",
"EulerDiscreteScheduler"
],
"text_encoder": [
"transformers",
"CLIPTextModel"
],
"text_encoder_2": [
"transformers",
"CLIPTextModelWithProjection"
],
"tokenizer": [
"transformers",
"CLIPTokenizer"
],
"tokenizer_2": [
"transformers",
"CLIPTokenizer"
],
"unet": [
"diffusers",
"UNet2DConditionModel"
],
"vae": [
"diffusers",
"AutoencoderKL"
]
}
local_repo/scheduler/scheduler_config.json
+18
View File
@@ -0,0 +1,18 @@
{
"_class_name": "EulerDiscreteScheduler",
"_diffusers_version": "0.19.0.dev0",
"beta_end": 0.012,
"beta_schedule": "scaled_linear",
"beta_start": 0.00085,
"clip_sample": false,
"interpolation_type": "linear",
"num_train_timesteps": 1000,
"prediction_type": "epsilon",
"sample_max_value": 1.0,
"set_alpha_to_one": false,
"skip_prk_steps": true,
"steps_offset": 1,
"timestep_spacing": "leading",
"trained_betas": null,
"use_karras_sigmas": false
}
local_repo/text_encoder/config.json
+24
View File
@@ -0,0 +1,24 @@
{
"architectures": [
"CLIPTextModel"
],
"attention_dropout": 0.0,
"bos_token_id": 0,
"dropout": 0.0,
"eos_token_id": 2,
"hidden_act": "quick_gelu",
"hidden_size": 768,
"initializer_factor": 1.0,
"initializer_range": 0.02,
"intermediate_size": 3072,
"layer_norm_eps": 1e-05,
"max_position_embeddings": 77,
"model_type": "clip_text_model",
"num_attention_heads": 12,
"num_hidden_layers": 12,
"pad_token_id": 1,
"projection_dim": 768,
"torch_dtype": "float16",
"transformers_version": "4.32.0.dev0",
"vocab_size": 49408
}
local_repo/text_encoder_2/config.json
+24
View File
@@ -0,0 +1,24 @@
{
"architectures": [
"CLIPTextModelWithProjection"
],
"attention_dropout": 0.0,
"bos_token_id": 0,
"dropout": 0.0,
"eos_token_id": 2,
"hidden_act": "gelu",
"hidden_size": 1280,
"initializer_factor": 1.0,
"initializer_range": 0.02,
"intermediate_size": 5120,
"layer_norm_eps": 1e-05,
"max_position_embeddings": 77,
"model_type": "clip_text_model",
"num_attention_heads": 20,
"num_hidden_layers": 32,
"pad_token_id": 1,
"projection_dim": 1280,
"torch_dtype": "float16",
"transformers_version": "4.32.0.dev0",
"vocab_size": 49408
}
local_repo/tokenizer/merges.txt
+48895
View File
File diff suppressed because it is too large Load Diff
local_repo/tokenizer/special_tokens_map.json
+24
View File
@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<|startoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<|endoftext|>",
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}
local_repo/tokenizer/tokenizer_config.json
+33
View File
@@ -0,0 +1,33 @@
{
"add_prefix_space": false,
"bos_token": {
"__type": "AddedToken",
"content": "<|startoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"clean_up_tokenization_spaces": true,
"do_lower_case": true,
"eos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"errors": "replace",
"model_max_length": 77,
"pad_token": "<|endoftext|>",
"tokenizer_class": "CLIPTokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}
local_repo/tokenizer/vocab.json
+49410
View File
File diff suppressed because it is too large Load Diff
local_repo/tokenizer_2/merges.txt
+48895
View File
File diff suppressed because it is too large Load Diff
local_repo/tokenizer_2/special_tokens_map.json
+24
View File
@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<|startoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "!",
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}
local_repo/tokenizer_2/tokenizer_config.json
+33
View File
@@ -0,0 +1,33 @@
{
"add_prefix_space": false,
"bos_token": {
"__type": "AddedToken",
"content": "<|startoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"clean_up_tokenization_spaces": true,
"do_lower_case": true,
"eos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"errors": "replace",
"model_max_length": 77,
"pad_token": "!",
"tokenizer_class": "CLIPTokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}
local_repo/tokenizer_2/vocab.json
+49410
View File
File diff suppressed because it is too large Load Diff
local_repo/unet/config.json
+69
View File
@@ -0,0 +1,69 @@
{
"_class_name": "UNet2DConditionModel",
"_diffusers_version": "0.19.0.dev0",
"act_fn": "silu",
"addition_embed_type": "text_time",
"addition_embed_type_num_heads": 64,
"addition_time_embed_dim": 256,
"attention_head_dim": [
5,
10,
20
],
"block_out_channels": [
320,
640,
1280
],
"center_input_sample": false,
"class_embed_type": null,
"class_embeddings_concat": false,
"conv_in_kernel": 3,
"conv_out_kernel": 3,
"cross_attention_dim": 2048,
"cross_attention_norm": null,
"down_block_types": [
"DownBlock2D",
"CrossAttnDownBlock2D",
"CrossAttnDownBlock2D"
],
"downsample_padding": 1,
"dual_cross_attention": false,
"encoder_hid_dim": null,
"encoder_hid_dim_type": null,
"flip_sin_to_cos": true,
"freq_shift": 0,
"in_channels": 4,
"layers_per_block": 2,
"mid_block_only_cross_attention": null,
"mid_block_scale_factor": 1,
"mid_block_type": "UNetMidBlock2DCrossAttn",
"norm_eps": 1e-05,
"norm_num_groups": 32,
"num_attention_heads": null,
"num_class_embeds": null,
"only_cross_attention": false,
"out_channels": 4,
"projection_class_embeddings_input_dim": 2816,
"resnet_out_scale_factor": 1.0,
"resnet_skip_time_act": false,
"resnet_time_scale_shift": "default",
"sample_size": 128,
"time_cond_proj_dim": null,
"time_embedding_act_fn": null,
"time_embedding_dim": null,
"time_embedding_type": "positional",
"timestep_post_act": null,
"transformer_layers_per_block": [
1,
2,
10
],
"up_block_types": [
"CrossAttnUpBlock2D",
"CrossAttnUpBlock2D",
"UpBlock2D"
],
"upcast_attention": null,
"use_linear_projection": true
}
local_repo/vae/config.json
+32
View File
@@ -0,0 +1,32 @@
{
"_class_name": "AutoencoderKL",
"_diffusers_version": "0.20.0.dev0",
"_name_or_path": "../sdxl-vae/",
"act_fn": "silu",
"block_out_channels": [
128,
256,
512,
512
],
"down_block_types": [
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D"
],
"force_upcast": true,
"in_channels": 3,
"latent_channels": 4,
"layers_per_block": 2,
"norm_num_groups": 32,
"out_channels": 3,
"sample_size": 1024,
"scaling_factor": 0.13025,
"up_block_types": [
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D"
]
}
local_repo/vae_1_0/config.json
+31
View File
@@ -0,0 +1,31 @@
{
"_class_name": "AutoencoderKL",
"_diffusers_version": "0.19.0.dev0",
"act_fn": "silu",
"block_out_channels": [
128,
256,
512,
512
],
"down_block_types": [
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D"
],
"force_upcast": true,
"in_channels": 3,
"latent_channels": 4,
"layers_per_block": 2,
"norm_num_groups": 32,
"out_channels": 3,
"sample_size": 1024,
"scaling_factor": 0.13025,
"up_block_types": [
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D"
]
}
local_repo/vae_decoder/config.json
+31
View File
@@ -0,0 +1,31 @@
{
"_class_name": "AutoencoderKL",
"_diffusers_version": "0.19.0.dev0",
"act_fn": "silu",
"block_out_channels": [
128,
256,
512,
512
],
"down_block_types": [
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D"
],
"force_upcast": true,
"in_channels": 3,
"latent_channels": 4,
"layers_per_block": 2,
"norm_num_groups": 32,
"out_channels": 3,
"sample_size": 1024,
"scaling_factor": 0.13025,
"up_block_types": [
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D"
]
}
local_repo/vae_encoder/config.json
+31
View File
@@ -0,0 +1,31 @@
{
"_class_name": "AutoencoderKL",
"_diffusers_version": "0.19.0.dev0",
"act_fn": "silu",
"block_out_channels": [
128,
256,
512,
512
],
"down_block_types": [
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D",
"DownEncoderBlock2D"
],
"force_upcast": true,
"in_channels": 3,
"latent_channels": 4,
"layers_per_block": 2,
"norm_num_groups": 32,
"out_channels": 3,
"sample_size": 1024,
"scaling_factor": 0.13025,
"up_block_types": [
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D",
"UpDecoderBlock2D"
]
}