init
This commit is contained in:
smthemex
@@ -1,316 +0,0 @@
|
||||
{
|
||||
"last_node_id": 6,
|
||||
"last_link_id": 6,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 4,
|
||||
"type": "SaveImage",
|
||||
"pos": [
|
||||
24757.544921875,
|
||||
-1843.1082763671875
|
||||
],
|
||||
"size": [
|
||||
315,
|
||||
270
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 3
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"ComfyUI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "LoadImage",
|
||||
"pos": [
|
||||
23483.703125,
|
||||
-1840.22021484375
|
||||
],
|
||||
"size": [
|
||||
315,
|
||||
314
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": [],
|
||||
"slot_index": 1
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
"background (1).png",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "ImageToMask",
|
||||
"pos": [
|
||||
24199.099609375,
|
||||
-1488.7823486328125
|
||||
],
|
||||
"size": [
|
||||
315,
|
||||
58
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"type": "IMAGE",
|
||||
"link": 6
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": [
|
||||
5
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ImageToMask"
|
||||
},
|
||||
"widgets_values": [
|
||||
"red"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "LoadImage",
|
||||
"pos": [
|
||||
23857.28515625,
|
||||
-1701.5709228515625
|
||||
],
|
||||
"size": [
|
||||
315,
|
||||
314
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
6
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImage"
|
||||
},
|
||||
"widgets_values": [
|
||||
"mask (4).png",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"type": "Personalize_Anything_Load",
|
||||
"pos": [
|
||||
23754.28125,
|
||||
-2035.958984375
|
||||
],
|
||||
"size": [
|
||||
315,
|
||||
150
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MODEL",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MODEL_PERSONALIZE_ANYTHING",
|
||||
"links": [
|
||||
1
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Personalize_Anything_Load"
|
||||
},
|
||||
"widgets_values": [
|
||||
"none",
|
||||
"F:/test/ComfyUI/models/diffusers/black-forest-labs/FLUX.1-dev",
|
||||
"nf4",
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "Personalize_Anything_Sampler",
|
||||
"pos": [
|
||||
24227.978515625,
|
||||
-2057.822265625
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
398
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MODEL_PERSONALIZE_ANYTHING",
|
||||
"link": 1
|
||||
},
|
||||
{
|
||||
"name": "iamge",
|
||||
"type": "IMAGE",
|
||||
"link": 2
|
||||
},
|
||||
{
|
||||
"name": "mask",
|
||||
"type": "MASK",
|
||||
"link": 5
|
||||
},
|
||||
{
|
||||
"name": "bg_image",
|
||||
"type": "IMAGE",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
3
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Personalize_Anything_Sampler"
|
||||
},
|
||||
"widgets_values": [
|
||||
"A teddy bear",
|
||||
"A teddy bear waving its right hand on a nighttime street, positioned on the left side of the frame, with an empty road on the right.",
|
||||
1434828011,
|
||||
"randomize",
|
||||
512,
|
||||
512,
|
||||
28,
|
||||
50,
|
||||
0,
|
||||
"single_personalize"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
2,
|
||||
0,
|
||||
"MODEL_PERSONALIZE_ANYTHING"
|
||||
],
|
||||
[
|
||||
2,
|
||||
3,
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
3,
|
||||
2,
|
||||
0,
|
||||
4,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
5,
|
||||
5,
|
||||
0,
|
||||
2,
|
||||
2,
|
||||
"MASK"
|
||||
],
|
||||
[
|
||||
6,
|
||||
6,
|
||||
0,
|
||||
5,
|
||||
0,
|
||||
"IMAGE"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 0.9441746356323133,
|
||||
"offset": [
|
||||
-23352.781257049,
|
||||
2171.1353098562176
|
||||
]
|
||||
}
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -1,245 +0,0 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import os
|
||||
from PIL import Image
|
||||
import torch
|
||||
import numpy as np
|
||||
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig,FluxTransformer2DModel, AutoencoderKL
|
||||
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig,T5EncoderModel
|
||||
|
||||
from .src.utils import *
|
||||
from .scripts.grounding_sam import *
|
||||
from .node_utils import cleanup,pil2narry,process_image_with_mask,tensor2pil_upscale
|
||||
from .src.pipeline import RFInversionParallelFluxPipeline
|
||||
from .src.pipeline_wrapper import RFInversionParallelFluxPipeline as RFInversionParallelFluxPipeline_Wrapper
|
||||
from .gradio_demo import generate_image_out,generate_image_in,generate_image_personalize_single,generate_image_reconstruction,generate_image_Composition
|
||||
|
||||
import folder_paths
|
||||
|
||||
MAX_SEED = np.iinfo(np.int32).max
|
||||
current_node_path = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
device = torch.device(
|
||||
"cuda:0") if torch.cuda.is_available() else torch.device(
|
||||
"mps") if torch.backends.mps.is_available() else torch.device(
|
||||
"cpu")
|
||||
|
||||
|
||||
class Personalize_Anything_Load:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"transformer": (["none"]+folder_paths.get_filename_list("diffusion_models"),),
|
||||
"flux_repo": ("STRING", {"default": "F:/test/ComfyUI/models/diffusers/black-forest-labs/FLUX.1-dev"},),
|
||||
"quantization":(["none","fp8","nf4"],),
|
||||
"quantize_T5":("BOOLEAN",{"default":True}),
|
||||
},
|
||||
"optional": { "model":("MODEL",),
|
||||
"vae":("VAE",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL_PERSONALIZE_ANYTHING", )
|
||||
RETURN_NAMES = ("model",)
|
||||
FUNCTION = "main"
|
||||
CATEGORY = "Personalize_Anything"
|
||||
|
||||
def main(self, transformer,flux_repo,quantization,quantize_T5,**kwargs):
|
||||
cf_model=kwargs.get("model",None)
|
||||
cf_vae=kwargs.get("vae",None)
|
||||
|
||||
flux_repo_local=os.path.join(current_node_path, 'src/FLUX.1-dev')
|
||||
|
||||
if flux_repo:
|
||||
|
||||
if quantization=="none":
|
||||
if quantize_T5:
|
||||
quant_config = TransformersBitsAndBytesConfig(load_in_8bit=True,) # 8bit default
|
||||
text_encoder_2_8bit = T5EncoderModel.from_pretrained(
|
||||
flux_repo,
|
||||
subfolder="text_encoder_2",
|
||||
quantization_config=quant_config,
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
pipeline = RFInversionParallelFluxPipeline.from_pretrained(flux_repo,text_encoder_2=text_encoder_2_8bit,torch_dtype=torch.bfloat16,)
|
||||
else:
|
||||
pipeline = RFInversionParallelFluxPipeline.from_pretrained(flux_repo,torch_dtype=torch.bfloat16,)
|
||||
else:
|
||||
|
||||
if quantization=="fp8":
|
||||
if quantize_T5:
|
||||
quant_config = TransformersBitsAndBytesConfig(load_in_8bit=True,) # 8bit
|
||||
text_encoder_2_8bit = T5EncoderModel.from_pretrained(
|
||||
flux_repo,
|
||||
subfolder="text_encoder_2",
|
||||
quantization_config=quant_config,
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
transformer = FluxTransformer2DModel.from_pretrained(
|
||||
flux_repo,
|
||||
subfolder="transformer",
|
||||
quantization_config=DiffusersBitsAndBytesConfig(load_in_8bit=True,),
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
if quantize_T5:
|
||||
pipeline =RFInversionParallelFluxPipeline.from_pretrained(flux_repo,transformer=transformer,text_encoder_2=text_encoder_2_8bit,torch_dtype=torch.bfloat16,)
|
||||
else:
|
||||
pipeline = RFInversionParallelFluxPipeline.from_pretrained(flux_repo,transformer=transformer, torch_dtype=torch.bfloat16,)
|
||||
else: #nf4
|
||||
if quantize_T5:
|
||||
quant_config = TransformersBitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_quant_type="nf4",
|
||||
)
|
||||
|
||||
text_encoder_2_4bit = T5EncoderModel.from_pretrained(
|
||||
flux_repo,
|
||||
subfolder="text_encoder_2",
|
||||
quantization_config=quant_config,
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
transformer = FluxTransformer2DModel.from_pretrained(
|
||||
flux_repo,
|
||||
subfolder="transformer",
|
||||
quantization_config=DiffusersBitsAndBytesConfig(
|
||||
load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
|
||||
),
|
||||
torch_dtype=torch.bfloat16,)
|
||||
if quantize_T5:
|
||||
pipeline =RFInversionParallelFluxPipeline.from_pretrained(flux_repo,transformer=transformer,text_encoder_2=text_encoder_2_4bit,torch_dtype=torch.bfloat16,)
|
||||
else:
|
||||
pipeline = RFInversionParallelFluxPipeline.from_pretrained(flux_repo,transformer=transformer, torch_dtype=torch.bfloat16,)
|
||||
else:
|
||||
if cf_model is None and cf_vae is None:
|
||||
if transformer != "none":
|
||||
flux_transformer_path = folder_paths.get_full_path("diffusion_models", transformer)
|
||||
pipeline = RFInversionParallelFluxPipeline.from_single_file(
|
||||
flux_transformer_path,config=flux_repo_local, torch_dtype=torch.bfloat16,)
|
||||
else:
|
||||
raise Exception("Please select a transformer model")
|
||||
else:
|
||||
ae_dic=cf_vae.get_sd()
|
||||
# vae_path = folder_paths.get_full_path("vae", vae)
|
||||
vae_config=os.path.join(flux_repo_local, 'vae')
|
||||
ae = AutoencoderKL.from_single_file(ae_dic,config=vae_config, torch_dtype=torch.bfloat16)
|
||||
|
||||
config_file = os.path.join(flux_repo_local,"transformer/config.json")
|
||||
if transformer != "none":
|
||||
flux_transformer_path = folder_paths.get_full_path("diffusion_models", transformer)
|
||||
transformer_ = FluxTransformer2DModel.from_single_file(
|
||||
flux_transformer_path,
|
||||
config=config_file,
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
if cf_model is not None:
|
||||
|
||||
cf_state_dict = cf_model.model.diffusion_model.state_dict()
|
||||
del cf_model
|
||||
cleanup()
|
||||
transformer_=FluxTransformer2DModel.from_single_file(cf_state_dict,config=config_file,torch_dtype=torch.bfloat16)
|
||||
# unet_config = FluxTransformer2DModel.load_config(config_file)
|
||||
# transformer_ = FluxTransformer2DModel.from_config(unet_config).to(torch.bfloat16)
|
||||
# transformer_.load_state_dict(cf_state_dict, strict=False)
|
||||
del cf_state_dict
|
||||
cleanup()
|
||||
else:
|
||||
raise ValueError("No transformer model found")
|
||||
pipeline = RFInversionParallelFluxPipeline_Wrapper.from_pretrained(
|
||||
flux_repo_local,vae=ae,transformer=transformer_, torch_dtype=torch.bfloat16,)
|
||||
cleanup()
|
||||
pipeline.enable_model_cpu_offload()
|
||||
return (pipeline,)
|
||||
|
||||
|
||||
class Personalize_Anything_Sampler:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL_PERSONALIZE_ANYTHING",),
|
||||
"iamge": ("IMAGE",), # B H W C
|
||||
"mask": ("MASK",), # B H W
|
||||
"prompt": ("STRING", {"default": "A teddy bear", "multiline": True,}),
|
||||
"personalize_prompt": ("STRING", {"default": "A teddy bear waving its right hand on a nighttime street, positioned on the left side of the frame, with an empty road on the right.", "multiline": True,}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": MAX_SEED, "step": 1, "display": "number"}),
|
||||
"width": ("INT", {"default": 512, "min": 256, "max": 4096, "step": 16, "display": "number"}),
|
||||
"height": ("INT", {"default": 512, "min": 256, "max": 4096, "step": 16, "display": "number"}),
|
||||
"timestep": ("INT", {"default": 28, "min": 1, "max": 1024, "step": 1, "display": "number"}),
|
||||
"tau": ("INT", {"default": 50, "min": 0, "max": 100, "step": 1, "display": "number"}),
|
||||
"shift": ("INT", {"default": 0, "min": -4096, "max": 4096, "step": 1, "display": "number"}),
|
||||
"infer_mode":(["inpainting","outpainting","single_personalize","multi_personalize","subject_reconstruction","scene_composition"],),
|
||||
},
|
||||
"optional": { "bg_image":("IMAGE",),
|
||||
"clip": ("CLIP",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", )
|
||||
RETURN_NAMES = ("image",)
|
||||
FUNCTION = "main"
|
||||
CATEGORY = "Personalize_Anything"
|
||||
|
||||
def main(self,model, iamge,mask,prompt,personalize_prompt,seed,width,height,timestep,tau,shift,infer_mode,**kwargs):
|
||||
bg_image=kwargs.get("bg_image")
|
||||
cf_clip=kwargs.get("clip")
|
||||
|
||||
|
||||
if infer_mode=="single_personalize" or infer_mode=="subject_reconstruction" or infer_mode=="scene_composition":
|
||||
init_image,mask_pil=process_image_with_mask(iamge,mask,width,height,False) #false
|
||||
else:
|
||||
init_image,mask_pil=process_image_with_mask(iamge,mask,width,height,True) # outpainting and inpainting need Ture
|
||||
|
||||
if infer_mode=="scene_composition":
|
||||
if "[" in prompt:
|
||||
prompt=prompt.split("[")
|
||||
else:
|
||||
raise ValueError("scene_composition need a [ in prompt")
|
||||
|
||||
if isinstance(bg_image, torch.Tensor) :
|
||||
bg_image=tensor2pil_upscale(bg_image,width,height)
|
||||
else:
|
||||
raise ValueError("bg_image must link a image")
|
||||
|
||||
|
||||
latent_h = height // 16
|
||||
latent_w = width // 16
|
||||
img_dims = latent_h * latent_w
|
||||
|
||||
mask = create_mask(mask_pil, latent_w, latent_h)
|
||||
bg_mask=1-mask
|
||||
# inverse
|
||||
|
||||
if infer_mode=="inpainting":
|
||||
img=generate_image_in(model,prompt, seed, timestep, tau, init_image,mask,height,width,img_dims,device,shift,cf_clip)
|
||||
elif infer_mode=="outpainting":
|
||||
img=generate_image_out(model,prompt,personalize_prompt, seed, timestep, tau,init_image,mask,height,width,img_dims,device,shift,cf_clip)
|
||||
elif infer_mode=="single_personalize":
|
||||
img=generate_image_personalize_single(model,prompt,personalize_prompt, seed, timestep, tau, init_image,mask,height,width,img_dims,device,shift,cf_clip)
|
||||
elif infer_mode=="multi_personalize":
|
||||
pass
|
||||
elif infer_mode=="scene_composition":
|
||||
img=generate_image_Composition(model,prompt,personalize_prompt, seed, timestep, tau, init_image,bg_image,mask,bg_mask,height,width,img_dims,device,shift,cf_clip)
|
||||
else: # subject_reconstruction
|
||||
img=generate_image_reconstruction(model,prompt,personalize_prompt, seed, timestep, tau, init_image,mask,height,width,img_dims,device,shift,cf_clip)
|
||||
cleanup()
|
||||
return (pil2narry(img),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Personalize_Anything_Load": Personalize_Anything_Load,
|
||||
"Personalize_Anything_Sampler":Personalize_Anything_Sampler,
|
||||
|
||||
}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Personalize_Anything_Load": "Personalize_Anything_Load",
|
||||
"Personalize_Anything_Sampler":"Personalize_Anything_Sampler",
|
||||
}
|
||||
@@ -2,28 +2,18 @@
|
||||
[Personalize Anything](https://github.com/fenghora/personalize-anything) for Free with Diffusion Transformer,use it in comfyUI with wrapper mode
|
||||
|
||||
# Update
|
||||
* fix inpainting bug, inpainting only use prompt line one /修复内绘的bug，注意内绘只用第一行的prompt
|
||||
* 改用comfyUI内核复现方法，去掉diffuser和transformer的版本限制/ Use comfyUI core replay method, remove the version limit of diffuser and transformer
|
||||
|
||||
# 1. Installation
|
||||
|
||||
In the ./ComfyUI/custom_node directory, run the following:
|
||||
In the ./ComfyUI/custom_nodes directory, run the following:
|
||||
```
|
||||
git clone https://github.com/smthemex/ComfyUI_Personalize_Anything
|
||||
```
|
||||
---
|
||||
|
||||
# 2. Requirements
|
||||
|
||||
```
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
# 3.Model
|
||||
**3.1 flux dev repo （preference/推荐）**
|
||||
* 3.1 use flux dev [diffusser repo](https://huggingface.co/black-forest-labs/FLUX.1-dev/tree/main)
|
||||
if OOM chocie nf4 or fp8 type 如果OOM，开启nf4或者fp8量化
|
||||
|
||||
**3.2 single model**
|
||||
* 3.1.1 download kijai 'flux1-dev.safetensors' fp8 from [here](https://huggingface.co/Kijai/flux-fp8/tree/main) 下载kijai的flux单体模型模型,文件结构如下图
|
||||
* 3.1.2 download 'clip_l.safetensors' and 't5xxl_fp8_e4m3fn.safetensors' from [here](https://huggingface.co/comfyanonymous/flux_text_encoders/tree/main)下载T5和clip l单体模型模型,文件结构如下图
|
||||
|
||||
@@ -37,23 +27,16 @@ if OOM chocie nf4 or fp8 type 如果OOM，开启nf4或者fp8量化
|
||||
├── ae.safetensor
|
||||
```
|
||||
|
||||
|
||||
# 4.Example
|
||||
* Inpainting , use up prompt only
|
||||
|
||||
* Outpainting
|
||||
|
||||
* Inpainting
|
||||
|
||||
* subject
|
||||
|
||||
* Reconstruction
|
||||
|
||||
* Single_personalize
|
||||
|
||||
* comfy single model
|
||||
|
||||
|
||||
|
||||
# 5.Tips
|
||||
* 使用方法请参考示例图片/Please refer to the example image for usage instructions
|
||||
|
||||
# 6. Citation
|
||||
# 5. Citation
|
||||
```
|
||||
@article{feng2025personalize,
|
||||
title={Personalize Anything for Free with Diffusion Transformer},
|
||||
|
||||
+23
-1
@@ -1,4 +1,26 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""Personalize Anything on comfy's **native** FLUX DiT stack.
|
||||
|
||||
from .Personalize_Anything_node import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
|
||||
The legacy diffusers wrapper that used to live here (``Personalize_Anything_node.py``,
|
||||
``src/``, ``gradio_demo.py``) is kept in the folder for archaeology but is *not*
|
||||
imported any more: it dies on transformers>=5 (``diffusers.loaders.peft`` imports
|
||||
``HybridCache``), which used to make the whole node package vanish from the comfy
|
||||
menu. The new implementation needs nothing but torch + comfy itself.
|
||||
"""
|
||||
|
||||
from .pa_nodes import (PersonalizeAnythingInvert, PersonalizeAnythingSample,
|
||||
PersonalizeAnythingCompose)
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"PersonalizeAnythingInvert": PersonalizeAnythingInvert,
|
||||
"PersonalizeAnythingSample": PersonalizeAnythingSample,
|
||||
"PersonalizeAnythingCompose": PersonalizeAnythingCompose,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"PersonalizeAnythingInvert": "Personalize Anything Invert (RF-Inversion)",
|
||||
"PersonalizeAnythingSample": "Personalize Anything Sample",
|
||||
"PersonalizeAnythingCompose": "Personalize Anything Compose (fg+bg)",
|
||||
}
|
||||
|
||||
__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 342 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 398 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 5.1 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 373 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 6.4 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 411 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 420 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.0 MiB |
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 432 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 387 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 399 KiB |
-494
@@ -1,494 +0,0 @@
|
||||
import os
|
||||
#import gradio as gr
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
from .src.utils import *
|
||||
from .scripts.grounding_sam import *
|
||||
from diffusers.models.attention_processor import FluxAttnProcessor2_0
|
||||
from .src.attn_processor import (
|
||||
PersonalizeAnythingAttnProcessor,MultiPersonalizeAnythingAttnProcessor,
|
||||
set_flux_transformer_attn_processor,
|
||||
)
|
||||
from .node_utils import cf_prompt_clip,cleanup,cf_unload
|
||||
|
||||
|
||||
def generate_image_out(pipe,prompt,new_prompt, seed, timestep, tau, init_image,mask,height,width,img_dims,device,shift,cf_clip=None):
|
||||
shift_mask = shift_tensor(mask, shift)
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
|
||||
if cf_clip is not None:
|
||||
|
||||
inv_prompt_embeds,inv_pooled_prompt_embeds,inv_text_ids=cf_prompt_clip(cf_clip,"")
|
||||
inf_prompt_embeds_,inf_pooled_prompt_embeds_,_= cf_prompt_clip(cf_clip,prompt)
|
||||
new_prompt_embeds,new_pooled_prompt_embeds,_=cf_prompt_clip(cf_clip,new_prompt)
|
||||
|
||||
pipe_prompt_embeds=torch.cat([inf_prompt_embeds_,new_prompt_embeds],dim=0)# batch size 2
|
||||
pipe_pooled_prompt_embeds=torch.cat([inf_pooled_prompt_embeds_,new_pooled_prompt_embeds],dim=0)
|
||||
|
||||
cf_clip=None
|
||||
cf_unload()
|
||||
cleanup()
|
||||
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
prompt_embeds= inv_prompt_embeds,
|
||||
pooled_prompt_embeds= inv_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
)
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, mask=mask, shift_mask=shift_mask, device=device, img_dims=img_dims),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
None,
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
prompt_embeds= pipe_prompt_embeds,
|
||||
pooled_prompt_embeds= pipe_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
).images[1]
|
||||
|
||||
else:
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0)
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, mask=mask, shift_mask=shift_mask, device=device, img_dims=img_dims),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
[prompt, new_prompt],
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
).images[1]
|
||||
|
||||
return image
|
||||
|
||||
def generate_image_in(pipe,prompt, seed, timestep, tau, init_image,mask,height,width,img_dims,device,shift,cf_clip=None):
|
||||
shift_mask = shift_tensor(mask, shift)
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
|
||||
if cf_clip is not None:
|
||||
|
||||
inv_prompt_embeds,inv_pooled_prompt_embeds,inv_text_ids=cf_prompt_clip(cf_clip,"")
|
||||
inf_prompt_embeds_,inf_pooled_prompt_embeds_,_= cf_prompt_clip(cf_clip,prompt)
|
||||
|
||||
pipe_prompt_embeds=torch.cat([inf_prompt_embeds_,inv_prompt_embeds],dim=0)# batch size 2
|
||||
pipe_pooled_prompt_embeds=torch.cat([inf_pooled_prompt_embeds_,inv_pooled_prompt_embeds],dim=0)
|
||||
|
||||
cf_clip=None
|
||||
cf_unload()
|
||||
cleanup()
|
||||
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
prompt_embeds= inv_prompt_embeds,
|
||||
pooled_prompt_embeds= inv_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
)
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, mask=mask, shift_mask=shift_mask, device=device, img_dims=img_dims,token_len=inf_prompt_embeds_.shape[1]),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
None,
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height=height,
|
||||
width=width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
prompt_embeds= pipe_prompt_embeds,
|
||||
pooled_prompt_embeds= pipe_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
).images[1]
|
||||
|
||||
else:
|
||||
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
|
||||
)
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, mask=mask, shift_mask=shift_mask, device=device, img_dims=img_dims),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
["", prompt],
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height=height,
|
||||
width=width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
).images[1]
|
||||
|
||||
|
||||
return image
|
||||
|
||||
|
||||
|
||||
|
||||
def generate_image_personalize_single(pipe,prompt,personalize_prompt, seed, timestep, tau, init_image,mask,height,width,img_dims,device,shift,cf_clip=None):
|
||||
|
||||
shift_mask = shift_tensor(mask, shift)
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
|
||||
if cf_clip is not None:
|
||||
inv_prompt_embeds,inv_pooled_prompt_embeds,inv_text_ids=cf_prompt_clip(cf_clip,"")
|
||||
inf_prompt_embeds_,inf_pooled_prompt_embeds_,_= cf_prompt_clip(cf_clip,prompt)
|
||||
new_prompt_embeds,new_pooled_prompt_embeds,_=cf_prompt_clip(cf_clip,personalize_prompt)
|
||||
|
||||
pipe_prompt_embeds=torch.cat([inf_prompt_embeds_,new_prompt_embeds],dim=0)# batch size 2
|
||||
pipe_pooled_prompt_embeds=torch.cat([inf_pooled_prompt_embeds_,new_pooled_prompt_embeds],dim=0)
|
||||
|
||||
|
||||
cf_clip=None
|
||||
cf_unload()
|
||||
cleanup()
|
||||
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
prompt_embeds= inv_prompt_embeds,
|
||||
pooled_prompt_embeds= inv_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
)
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, mask=mask, shift_mask=shift_mask, device=device, img_dims=img_dims),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
None,
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
prompt_embeds= pipe_prompt_embeds,
|
||||
pooled_prompt_embeds= pipe_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
).images[1]
|
||||
else:
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
)
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, mask=mask, shift_mask=shift_mask, device=device, img_dims=img_dims),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
[prompt, personalize_prompt],
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
).images[1]
|
||||
return image
|
||||
|
||||
|
||||
def generate_image_reconstruction(pipe,prompt,new_prompt, seed, timestep, tau, init_image,mask,height,width,img_dims,device,shift,cf_clip=None):
|
||||
|
||||
shift_mask = shift_tensor(mask, shift)
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
|
||||
if cf_clip is not None:
|
||||
inv_prompt_embeds,inv_pooled_prompt_embeds,inv_text_ids=cf_prompt_clip(cf_clip,"")
|
||||
inf_prompt_embeds_,inf_pooled_prompt_embeds_,_= cf_prompt_clip(cf_clip,prompt)
|
||||
new_prompt_embeds,new_pooled_prompt_embeds,_=cf_prompt_clip(cf_clip,new_prompt)
|
||||
|
||||
pipe_prompt_embeds=torch.cat([inf_prompt_embeds_,new_prompt_embeds],dim=0)# batch size 2
|
||||
pipe_pooled_prompt_embeds=torch.cat([inf_pooled_prompt_embeds_,new_pooled_prompt_embeds],dim=0)
|
||||
|
||||
|
||||
cf_clip=None
|
||||
cf_unload()
|
||||
cleanup()
|
||||
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
prompt_embeds= inv_prompt_embeds,
|
||||
pooled_prompt_embeds= inv_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
)
|
||||
|
||||
|
||||
shift_mask = shift_tensor(mask, shift)
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name,
|
||||
mask=mask,
|
||||
shift_mask=shift_mask,
|
||||
tau=tau/100,
|
||||
device=device,
|
||||
img_dims=img_dims,
|
||||
concept_process=False),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
None,
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
prompt_embeds= pipe_prompt_embeds,
|
||||
pooled_prompt_embeds= pipe_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
).images[1]
|
||||
else:
|
||||
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents, image_latents, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=init_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0)
|
||||
|
||||
|
||||
shift_mask = shift_tensor(mask, shift)
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: PersonalizeAnythingAttnProcessor(
|
||||
name=name,
|
||||
mask=mask,
|
||||
shift_mask=shift_mask,
|
||||
tau=tau/100,
|
||||
device=device,
|
||||
img_dims=img_dims,
|
||||
concept_process=False),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
[prompt, new_prompt],
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
).images[1]
|
||||
|
||||
return image
|
||||
|
||||
def generate_image_Composition(pipe,prompt,new_prompt, seed, timestep, tau, fg_image,bg_image,fg_mask,bg_mask,height,width,img_dims,device,shift,cf_clip=None):
|
||||
|
||||
fg_prompt, bg_prompt = prompt[0], prompt[1]
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
|
||||
|
||||
if cf_clip is not None:
|
||||
#inv_prompt_embeds,inv_pooled_prompt_embeds,inv_text_ids=cf_prompt_clip(cf_clip,"")
|
||||
fg_prompt_embeds_,fg_pooled_prompt_embeds_,inv_text_ids= cf_prompt_clip(cf_clip,fg_prompt)
|
||||
bg_prompt_embeds_,bg_pooled_prompt_embeds_,_= cf_prompt_clip(cf_clip,bg_prompt)
|
||||
new_prompt_embeds,new_pooled_prompt_embeds,_=cf_prompt_clip(cf_clip,new_prompt)
|
||||
|
||||
|
||||
pipe_prompt_embeds=torch.cat([bg_prompt_embeds_,fg_prompt_embeds_,new_prompt_embeds],dim=0)# batch size 2
|
||||
pipe_pooled_prompt_embeds=torch.cat([bg_pooled_prompt_embeds_,fg_pooled_prompt_embeds_,new_pooled_prompt_embeds],dim=0)
|
||||
|
||||
|
||||
cf_clip=None
|
||||
cf_unload()
|
||||
cleanup()
|
||||
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents_fg, image_latents_fg, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=fg_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
prompt_embeds= fg_prompt_embeds_,
|
||||
pooled_prompt_embeds= fg_pooled_prompt_embeds_,
|
||||
text_ids= inv_text_ids,
|
||||
)
|
||||
|
||||
inverted_latents_bg, image_latents_bg, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=bg_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
prompt_embeds= bg_prompt_embeds_,
|
||||
pooled_prompt_embeds= bg_pooled_prompt_embeds_,
|
||||
text_ids= inv_text_ids,
|
||||
)
|
||||
|
||||
inverted_latents = torch.cat([inverted_latents_fg, inverted_latents_bg], dim=0)
|
||||
image_latents = torch.cat([image_latents_fg, image_latents_bg], dim=0)
|
||||
masks = [fg_mask, bg_mask]
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: MultiPersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, masks=masks, shift_masks=None, device=device, img_dims=img_dims),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
None,
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
prompt_embeds= pipe_prompt_embeds,
|
||||
pooled_prompt_embeds= pipe_pooled_prompt_embeds,
|
||||
text_ids= inv_text_ids,
|
||||
).images[-1]
|
||||
else:
|
||||
|
||||
set_flux_transformer_attn_processor(pipe.transformer, set_attn_proc_func=lambda name, dh, nh, ap:FluxAttnProcessor2_0())
|
||||
inverted_latents_fg, image_latents_fg, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=fg_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
)
|
||||
|
||||
inverted_latents_bg, image_latents_bg, latent_image_ids = pipe.invert(
|
||||
source_prompt="",
|
||||
image=bg_image,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inversion_steps=timestep,
|
||||
gamma=1.0,
|
||||
)
|
||||
|
||||
inverted_latents = torch.cat([inverted_latents_fg, inverted_latents_bg], dim=0)
|
||||
image_latents = torch.cat([image_latents_fg, image_latents_bg], dim=0)
|
||||
masks = [fg_mask, bg_mask]
|
||||
|
||||
set_flux_transformer_attn_processor(
|
||||
pipe.transformer,
|
||||
set_attn_proc_func=lambda name, dh, nh, ap: MultiPersonalizeAnythingAttnProcessor(
|
||||
name=name, tau=tau/100, masks=masks, shift_masks=None, device=device, img_dims=img_dims),
|
||||
)
|
||||
|
||||
image = pipe(
|
||||
[fg_prompt, bg_prompt, new_prompt],
|
||||
inverted_latents=inverted_latents,
|
||||
image_latents=image_latents,
|
||||
latent_image_ids=latent_image_ids,
|
||||
height = height,
|
||||
width = width,
|
||||
start_timestep=0.0,
|
||||
stop_timestep=0.99,
|
||||
num_inference_steps=timestep,
|
||||
eta=1.0,
|
||||
generator=generator,
|
||||
).images[-1]
|
||||
|
||||
return image
|
||||
-267
@@ -1,267 +0,0 @@
|
||||
# !/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import os
|
||||
import torch
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import cv2
|
||||
import gc
|
||||
import torch
|
||||
from comfy.utils import common_upscale,ProgressBar
|
||||
from huggingface_hub import hf_hub_download
|
||||
import comfy.model_management
|
||||
|
||||
cur_path = os.path.dirname(os.path.abspath(__file__))
|
||||
device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
|
||||
|
||||
|
||||
def cf_unload():
|
||||
comfy.model_management.unload_all_models()
|
||||
comfy.model_management.soft_empty_cache()
|
||||
|
||||
|
||||
def cf_prompt_clip(cf_clip,prompt):
|
||||
tokens = cf_clip.tokenize(prompt)
|
||||
tokens["t5xxl"] = cf_clip.tokenize(prompt)["t5xxl"]
|
||||
prompt_embeds = cf_clip.encode_from_tokens(tokens, return_pooled=True, return_dict=True).pop("cond")
|
||||
tokens["l"] = cf_clip.tokenize(prompt)["l"]
|
||||
pooled_prompt_embeds = cf_clip.encode_from_tokens(tokens, return_dict=True).pop("pooled_output")
|
||||
prompt_embeds=prompt_embeds.to(device,torch.bfloat16)
|
||||
pooled_prompt_embeds=pooled_prompt_embeds.to(device,torch.bfloat16)
|
||||
text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=torch.bfloat16)
|
||||
return prompt_embeds,pooled_prompt_embeds,text_ids
|
||||
|
||||
|
||||
def latent_crop( samples, width, height, x, y):
|
||||
s = samples.copy()
|
||||
samples = samples['samples']
|
||||
x = x // 8
|
||||
y = y // 8
|
||||
|
||||
#enfonce minimum size of 64
|
||||
if x > (samples.shape[3] - 8):
|
||||
x = samples.shape[3] - 8
|
||||
if y > (samples.shape[2] - 8):
|
||||
y = samples.shape[2] - 8
|
||||
|
||||
new_height = height // 8
|
||||
new_width = width // 8
|
||||
to_x = new_width + x
|
||||
to_y = new_height + y
|
||||
s['samples'] = samples[:,:,y:to_y, x:to_x]
|
||||
return s
|
||||
|
||||
def cleanup():
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def process_image_with_mask(image, mask,width, height,invert=False):
|
||||
if invert:
|
||||
mask = 1.0 - mask
|
||||
mask_tensor=mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])).movedim(1, -1).expand(-1, -1, -1, 3)
|
||||
|
||||
input_image_resized = tensor2pil_upscale(image, width, height)
|
||||
mask_pil=tensor2pil_upscale(mask_tensor, width, height)
|
||||
|
||||
return input_image_resized,mask_pil
|
||||
|
||||
def cv2pil(cv_image):
|
||||
"""
|
||||
将OpenCV图像转换为PIL图像
|
||||
:param cv_image: OpenCV图像
|
||||
:return: PIL图像
|
||||
"""
|
||||
# 将图像从BGR转换为RGB
|
||||
rgb_image = cv2.cvtColor(cv_image, cv2.COLOR_BGR2RGB)
|
||||
# 使用PIL的Image.fromarray方法将NumPy数组转换为PIL图像
|
||||
pil_image = Image.fromarray(rgb_image)
|
||||
return pil_image
|
||||
|
||||
|
||||
def tensor_to_pil(tensor):
|
||||
image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
|
||||
image = Image.fromarray(image_np, mode='RGB')
|
||||
return image
|
||||
|
||||
def tensor2pil_list(image,width,height):
|
||||
B,_,_,_=image.size()
|
||||
if B==1:
|
||||
ref_image_list=[tensor2pil_upscale(image,width,height)]
|
||||
else:
|
||||
img_list = list(torch.chunk(image, chunks=B))
|
||||
ref_image_list = [tensor2pil_upscale(img,width,height) for img in img_list]
|
||||
return ref_image_list
|
||||
|
||||
|
||||
def tensor_upscale(img_tensor, width, height):
|
||||
samples = img_tensor.movedim(-1, 1)
|
||||
img = common_upscale(samples, width, height, "nearest-exact", "center")
|
||||
samples = img.movedim(1, -1)
|
||||
return samples
|
||||
|
||||
def tensor2pil_upscale(img_tensor, width, height):
|
||||
samples = img_tensor.movedim(-1, 1)
|
||||
img = common_upscale(samples, width, height, "nearest-exact", "center")
|
||||
samples = img.movedim(1, -1)
|
||||
img_pil = tensor_to_pil(samples)
|
||||
return img_pil
|
||||
|
||||
def tensor2np_upscale(img_tensor, width, height):
|
||||
samples = img_tensor.movedim(-1, 1)
|
||||
img = common_upscale(samples, width, height, "nearest-exact", "center")
|
||||
samples = img.movedim(1, -1)
|
||||
image_np = samples.squeeze().mul(255).clamp(0, 255).byte().numpy()
|
||||
return image_np
|
||||
|
||||
|
||||
def tensor2cv(tensor_image,RGB2BGR=True):
|
||||
if len(tensor_image.shape)==4:#bhwc to hwc
|
||||
tensor_image=tensor_image.squeeze(0)
|
||||
if tensor_image.is_cuda:
|
||||
tensor_image = tensor_image.cpu().detach()
|
||||
tensor_image=tensor_image.numpy()
|
||||
#反归一化
|
||||
maxValue=tensor_image.max()
|
||||
tensor_image=tensor_image*255/maxValue
|
||||
img_cv2=np.uint8(tensor_image)#32 to uint8
|
||||
if RGB2BGR:
|
||||
img_cv2=cv2.cvtColor(img_cv2,cv2.COLOR_RGB2BGR)
|
||||
return img_cv2
|
||||
|
||||
def cvargb2tensor(img):
|
||||
assert type(img) == np.ndarray, 'the img type is {}, but ndarry expected'.format(type(img))
|
||||
img = torch.from_numpy(img.transpose((2, 0, 1)))
|
||||
return img.float().div(255).unsqueeze(0) # 255也可以改为256
|
||||
|
||||
def cv2tensor(img):
|
||||
assert type(img) == np.ndarray, 'the img type is {}, but ndarry expected'.format(type(img))
|
||||
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
||||
img = torch.from_numpy(img.transpose((2, 0, 1)))
|
||||
return img.float().div(255).unsqueeze(0) # 255也可以改为256
|
||||
|
||||
def images_generator(img_list: list,):
|
||||
#get img size
|
||||
sizes = {}
|
||||
for image_ in img_list:
|
||||
if isinstance(image_,Image.Image):
|
||||
count = sizes.get(image_.size, 0)
|
||||
sizes[image_.size] = count + 1
|
||||
elif isinstance(image_,np.ndarray):
|
||||
count = sizes.get(image_.shape[:2][::-1], 0)
|
||||
sizes[image_.shape[:2][::-1]] = count + 1
|
||||
else:
|
||||
raise "unsupport image list,must be pil or cv2!!!"
|
||||
size = max(sizes.items(), key=lambda x: x[1])[0]
|
||||
yield size[0], size[1]
|
||||
|
||||
# any to tensor
|
||||
def load_image(img_in):
|
||||
if isinstance(img_in, Image.Image):
|
||||
img_in=img_in.convert("RGB")
|
||||
i = np.array(img_in, dtype=np.float32)
|
||||
i = torch.from_numpy(i).div_(255)
|
||||
if i.shape[0] != size[1] or i.shape[1] != size[0]:
|
||||
i = torch.from_numpy(i).movedim(-1, 0).unsqueeze(0)
|
||||
i = common_upscale(i, size[0], size[1], "lanczos", "center")
|
||||
i = i.squeeze(0).movedim(0, -1).numpy()
|
||||
return i
|
||||
elif isinstance(img_in,np.ndarray):
|
||||
i=cv2.cvtColor(img_in,cv2.COLOR_BGR2RGB).astype(np.float32)
|
||||
i = torch.from_numpy(i).div_(255)
|
||||
#print(i.shape)
|
||||
return i
|
||||
else:
|
||||
raise "unsupport image list,must be pil,cv2 or tensor!!!"
|
||||
|
||||
total_images = len(img_list)
|
||||
processed_images = 0
|
||||
pbar = ProgressBar(total_images)
|
||||
images = map(load_image, img_list)
|
||||
try:
|
||||
prev_image = next(images)
|
||||
while True:
|
||||
next_image = next(images)
|
||||
yield prev_image
|
||||
processed_images += 1
|
||||
pbar.update_absolute(processed_images, total_images)
|
||||
prev_image = next_image
|
||||
except StopIteration:
|
||||
pass
|
||||
if prev_image is not None:
|
||||
yield prev_image
|
||||
|
||||
def load_images(img_list: list,):
|
||||
gen = images_generator(img_list)
|
||||
(width, height) = next(gen)
|
||||
images = torch.from_numpy(np.fromiter(gen, np.dtype((np.float32, (height, width, 3)))))
|
||||
if len(images) == 0:
|
||||
raise FileNotFoundError(f"No images could be loaded .")
|
||||
return images
|
||||
|
||||
def tensor2pil(tensor):
|
||||
image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
|
||||
image = Image.fromarray(image_np, mode='RGB')
|
||||
return image
|
||||
|
||||
def pil2narry(img):
|
||||
narry = torch.from_numpy(np.array(img).astype(np.float32) / 255.0).unsqueeze(0)
|
||||
return narry
|
||||
|
||||
def equalize_lists(list1, list2):
|
||||
"""
|
||||
比较两个列表的长度，如果不一致，则将较短的列表复制以匹配较长列表的长度。
|
||||
|
||||
参数:
|
||||
list1 (list): 第一个列表
|
||||
list2 (list): 第二个列表
|
||||
|
||||
返回:
|
||||
tuple: 包含两个长度相等的列表的元组
|
||||
"""
|
||||
len1 = len(list1)
|
||||
len2 = len(list2)
|
||||
|
||||
if len1 == len2:
|
||||
pass
|
||||
elif len1 < len2:
|
||||
print("list1 is shorter than list2, copying list1 to match list2's length.")
|
||||
list1.extend(list1 * ((len2 // len1) + 1)) # 复制list1以匹配list2的长度
|
||||
list1 = list1[:len2] # 确保长度一致
|
||||
else:
|
||||
print("list2 is shorter than list1, copying list2 to match list1's length.")
|
||||
list2.extend(list2 * ((len1 // len2) + 1)) # 复制list2以匹配list1的长度
|
||||
list2 = list2[:len1] # 确保长度一致
|
||||
|
||||
return list1, list2
|
||||
|
||||
def file_exists(directory, filename):
|
||||
# 构建文件的完整路径
|
||||
file_path = os.path.join(directory, filename)
|
||||
# 检查文件是否存在
|
||||
return os.path.isfile(file_path)
|
||||
|
||||
def download_weights(file_dir,repo_id,subfolder="",pt_name=""):
|
||||
if subfolder:
|
||||
file_path = os.path.join(file_dir,subfolder, pt_name)
|
||||
sub_dir=os.path.join(file_dir,subfolder)
|
||||
if not os.path.exists(sub_dir):
|
||||
os.makedirs(sub_dir)
|
||||
if not os.path.exists(file_path):
|
||||
file_path = hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
subfolder=subfolder,
|
||||
filename=pt_name,
|
||||
local_dir = file_dir,
|
||||
)
|
||||
return file_path
|
||||
else:
|
||||
file_path = os.path.join(file_dir, pt_name)
|
||||
if not os.path.exists(file_dir):
|
||||
os.makedirs(file_dir)
|
||||
if not os.path.exists(file_path):
|
||||
file_path = hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
filename=pt_name,
|
||||
local_dir=file_dir,
|
||||
)
|
||||
return file_path
|
||||
+401
@@ -0,0 +1,401 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""Personalize Anything, re-implemented on comfy's *native* FLUX DiT stack.
|
||||
|
||||
Upstream (https://github.com/fenghora/personalize-anything) is a diffusers wrapper that
|
||||
re-implements three things:
|
||||
|
||||
1. a custom FLUX attention processor - "timestep-adaptive token replacement"
|
||||
(the paper's core trick: the subject tokens of the source image are copied into
|
||||
the target image's tokens while the timestep is still high),
|
||||
2. the RF-Inversion *forward* ODE that turns the source image into a noise latent
|
||||
(Eq. 8 of https://arxiv.org/pdf/2410.10792),
|
||||
3. the RF-Inversion *controlled reverse* ODE that denoises a fresh noise latent
|
||||
while a second, parallel batch row carries the source branch (Algorithm 2).
|
||||
|
||||
This module re-implements exactly those three pieces against comfy's own
|
||||
``comfy.ldm.flux`` implementation, so the plugin needs **no diffusers, no transformers,
|
||||
no peft** and is immune to upstream version drift (the old wrapper dies on
|
||||
``transformers>=5`` because ``diffusers.loaders.peft`` imports ``HybridCache``).
|
||||
|
||||
Why the attention trick survives the port
|
||||
-----------------------------------------
|
||||
Upstream replaces rows of the *hidden states* (before to_q/to_k/to_v) and then projects:
|
||||
|
||||
concept = hidden[src_row, mask] # pre-projection
|
||||
hidden[dst_row, shift_mask] = concept # in-place
|
||||
q, k, v = to_q(hidden), to_k(hidden), to_v(hidden)
|
||||
|
||||
comfy's FLUX blocks hand us q/k/v *after* the fused qkv projection and after QKNorm.
|
||||
Because the projection is linear and QKNorm is per-token elementwise (RMSNorm over
|
||||
head_dim), replacing the projected rows is *exactly* equivalent to replacing the
|
||||
hidden rows and projecting afterwards:
|
||||
|
||||
to_q(hidden[dst_row, shift_mask] = concept) == to_q(hidden)[dst_row, shift_mask]
|
||||
|
||||
so we can do the whole trick on the q/k/v tensors comfy's own attention consumes.
|
||||
|
||||
The token layout is identical between the two stacks, too:
|
||||
* diffusers FLUX: ``_pack_latents`` -> view(B,C,h/2,2,w/2,2).permute(0,2,4,1,3,5)
|
||||
-> token = 16 latent channels x 2x2 spatial patch, token grid (h/16, w/16) row major.
|
||||
* comfy FLUX : ``rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)")`` with
|
||||
patch_size=2 -> the very same token grid (h/16, w/16), row major, same 64 values.
|
||||
* the RoPE ids are row major over that same grid in both stacks.
|
||||
Therefore a mask built on the (h/16, w/16) grid indexes the same spatial positions in
|
||||
comfy as it does upstream - no index remapping needed.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
import comfy.model_management
|
||||
import comfy.utils
|
||||
from comfy.ldm.flux.math import apply_rope as _flux_apply_rope
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# attention patch
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PAState:
|
||||
"""Mutable per-step state read by the attention patch.
|
||||
|
||||
``inject`` is a list of ``(src_row, mask_idx, shift_idx)``:
|
||||
* ``src_row`` - batch row that owns the subject tokens (0 = source branch)
|
||||
* ``mask_idx`` - flat token indices (on the h_tok x w_tok grid) of the subject
|
||||
* ``shift_idx`` - flat token indices where the subject is written
|
||||
``target_row`` is the row that receives the replacement.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.enabled = False
|
||||
self.tau = 0.5 # replacement is active while model_t > tau
|
||||
self.model_t = 0.0 # model input timestep of the current step (sigma)
|
||||
self.inject = None
|
||||
self.target_row = 1
|
||||
self.concept_process = False
|
||||
self.token_len = 512
|
||||
self.fire_count = 0 # diagnostics: how often the replacement actually ran
|
||||
|
||||
def reset(self):
|
||||
self.enabled = False
|
||||
self.model_t = 0.0
|
||||
self.inject = None
|
||||
self.target_row = 1
|
||||
self.concept_process = False
|
||||
self.fire_count = 0
|
||||
|
||||
|
||||
def make_attn_patch(state: PAState):
|
||||
"""comfy ``transformer_options["patches"]["attn1_patch"]`` callable.
|
||||
|
||||
Called by every flux block (19 double + 38 single) with
|
||||
``q, k, v`` of shape ``[B, heads, seq, head_dim]`` and ``pe`` of shape
|
||||
``[1, 1, seq, head_dim/2, 2, 2]``.
|
||||
"""
|
||||
|
||||
def patch(q, k, v, pe=None, attn_mask=None, extra_options=None):
|
||||
if not state.enabled or not state.inject:
|
||||
return {}
|
||||
# timestep-adaptive gate: upstream compares the *model* timestep (sigma) with tau
|
||||
if not (state.model_t > state.tau):
|
||||
return {}
|
||||
extra_options = extra_options or {}
|
||||
img_slice = extra_options.get("img_slice", None)
|
||||
if img_slice is None:
|
||||
return {}
|
||||
txt_len = int(img_slice[0])
|
||||
|
||||
tgt = state.target_row
|
||||
for src_row, mask_idx, shift_idx in state.inject:
|
||||
src = txt_len + mask_idx
|
||||
dst = txt_len + shift_idx
|
||||
for t in (q, k, v):
|
||||
t[tgt].index_copy_(1, dst, t[src_row].index_select(1, src))
|
||||
state.fire_count += 1
|
||||
|
||||
if not state.concept_process:
|
||||
return {"q": q, "k": k, "v": v}
|
||||
|
||||
# ---- token concatenation (upstream `concept_process`) ----------------------
|
||||
# the subject tokens are appended to the sequence so that *every* token can
|
||||
# attend to them; they keep the rope of their original position.
|
||||
c_q, c_k, c_v, c_pe = [], [], [], []
|
||||
for src_row, mask_idx, _ in state.inject:
|
||||
src = txt_len + mask_idx
|
||||
c_q.append(q[src_row].index_select(1, src))
|
||||
c_k.append(k[src_row].index_select(1, src))
|
||||
c_v.append(v[src_row].index_select(1, src))
|
||||
if pe is not None:
|
||||
c_pe.append(pe[0, 0].index_select(0, src))
|
||||
c_q = torch.cat(c_q, dim=1)
|
||||
c_k = torch.cat(c_k, dim=1)
|
||||
c_v = torch.cat(c_v, dim=1)
|
||||
b = q.shape[0]
|
||||
q = torch.cat([q, c_q.unsqueeze(0).expand(b, -1, -1, -1)], dim=2)
|
||||
k = torch.cat([k, c_k.unsqueeze(0).expand(b, -1, -1, -1)], dim=2)
|
||||
v = torch.cat([v, c_v.unsqueeze(0).expand(b, -1, -1, -1)], dim=2)
|
||||
|
||||
if pe is not None:
|
||||
pe = torch.cat([pe, torch.cat(c_pe, dim=0).unsqueeze(0).unsqueeze(0)], dim=2)
|
||||
q, k = _flux_apply_rope(q, k, pe)
|
||||
# rope already applied -> hand the block a null pe so it does not rope twice
|
||||
return {"q": q, "k": k, "v": v, "pe": None}
|
||||
return {"q": q, "k": k, "v": v}
|
||||
|
||||
return patch
|
||||
|
||||
|
||||
def make_out_patch(state: PAState):
|
||||
"""comfy ``transformer_options["patches"]["attn1_output_patch"]`` callable.
|
||||
|
||||
Only needed when the sequence was extended by the token concatenation: the block
|
||||
expects the attention output to have exactly ``txt + img`` rows.
|
||||
"""
|
||||
|
||||
def patch(attn, extra_options=None):
|
||||
if not state.enabled or not state.concept_process:
|
||||
return attn
|
||||
extra_options = extra_options or {}
|
||||
img_slice = extra_options.get("img_slice", None)
|
||||
if img_slice is None:
|
||||
return attn
|
||||
seq = int(img_slice[1])
|
||||
if attn.shape[1] > seq:
|
||||
return attn[:, :seq]
|
||||
return attn
|
||||
|
||||
return patch
|
||||
|
||||
|
||||
def build_transformer_options(patcher, state: PAState):
|
||||
"""transformer_options for the DiT call: the patcher's own options + our patches.
|
||||
|
||||
The patcher's options are kept so that hooks installed by other plugins (LoRA
|
||||
patches, rope options, controlnet hooks, ...) keep working.
|
||||
"""
|
||||
base = {}
|
||||
model_options = getattr(patcher, "model_options", None) or {}
|
||||
own = model_options.get("transformer_options", None) or {}
|
||||
if own:
|
||||
base = dict(own)
|
||||
patches = dict(base.get("patches", {}) or {})
|
||||
patches["attn1_patch"] = list(patches.get("attn1_patch", []) or []) + [make_attn_patch(state)]
|
||||
patches["attn1_output_patch"] = list(patches.get("attn1_output_patch", []) or []) + [make_out_patch(state)]
|
||||
base["patches"] = patches
|
||||
return base
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# raw velocity proxy
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class RawVelocitySampling(torch.nn.Module):
|
||||
"""Proxy over ``BaseModel.model_sampling`` that returns the *raw* velocity.
|
||||
|
||||
comfy's ``BaseModel._apply_model`` ends with::
|
||||
|
||||
return self.model_sampling.calculate_denoised(sigma, model_output, x)
|
||||
|
||||
and for FLUX (CONST mixin) that is ``x - sigma * v`` - an x0 prediction. RF-Inversion
|
||||
needs the velocity ``v`` itself, and the inversion loop evaluates the model at
|
||||
``sigma = 0`` where x0 and v cannot be separated any more. So we swap that one
|
||||
conversion out and delegate everything else (calculate_input / timestep / sigma
|
||||
tables) unchanged.
|
||||
"""
|
||||
|
||||
def __init__(self, base):
|
||||
super().__init__()
|
||||
self.base = base
|
||||
|
||||
def calculate_input(self, sigma, noise):
|
||||
return self.base.calculate_input(sigma, noise)
|
||||
|
||||
def timestep(self, sigma):
|
||||
return self.base.timestep(sigma)
|
||||
|
||||
def calculate_denoised(self, sigma, model_output, model_input):
|
||||
return model_output.float()
|
||||
|
||||
def percent_to_sigma(self, percent):
|
||||
return self.base.percent_to_sigma(percent)
|
||||
|
||||
def sigma(self, timestep):
|
||||
return self.base.sigma(timestep)
|
||||
|
||||
@property
|
||||
def sigma_min(self):
|
||||
return self.base.sigma_min
|
||||
|
||||
@property
|
||||
def sigma_max(self):
|
||||
return self.base.sigma_max
|
||||
|
||||
|
||||
class raw_velocity:
|
||||
"""Context manager: swap the model's model_sampling for the raw-velocity proxy."""
|
||||
|
||||
def __init__(self, base_model):
|
||||
self.base_model = base_model
|
||||
self.orig = None
|
||||
|
||||
def __enter__(self):
|
||||
self.orig = self.base_model.model_sampling
|
||||
self.base_model.model_sampling = RawVelocitySampling(self.orig)
|
||||
return self.base_model
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.base_model.model_sampling = self.orig
|
||||
self.orig = None
|
||||
return False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# mask helpers
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def token_grid(latent_shape, patch_size=2):
|
||||
"""(h_tok, w_tok) of the image-token grid for a comfy latent [B, C, h, w]."""
|
||||
h = int(latent_shape[-2])
|
||||
w = int(latent_shape[-1])
|
||||
return ((h + (patch_size // 2)) // patch_size, (w + (patch_size // 2)) // patch_size)
|
||||
|
||||
|
||||
def mask_to_grid(mask, h_tok, w_tok, invert=False, threshold=0.5):
|
||||
"""comfy MASK ([B,H,W] in [0,1]) -> boolean grid [h_tok, w_tok].
|
||||
|
||||
Upstream resized the PIL mask with NEAREST to (w/16, h/16) and kept the *pure
|
||||
white* pixels. We use the same nearest resize but threshold at 0.5 so that soft
|
||||
masks also work.
|
||||
"""
|
||||
if mask is None:
|
||||
return torch.zeros((h_tok, w_tok), dtype=torch.bool)
|
||||
m = mask
|
||||
if m.ndim == 2:
|
||||
m = m.unsqueeze(0)
|
||||
m = m[0].detach().float().cpu()
|
||||
if invert:
|
||||
m = 1.0 - m
|
||||
m = m.unsqueeze(0).unsqueeze(0)
|
||||
m = comfy.utils.common_upscale(m, w_tok, h_tok, "nearest-exact", "center")
|
||||
return m[0, 0] > threshold
|
||||
|
||||
|
||||
def grid_to_indices(grid):
|
||||
return torch.nonzero(grid.reshape(-1), as_tuple=False).reshape(-1).long()
|
||||
|
||||
|
||||
def shift_grid(grid, shift):
|
||||
"""Horizontal shift of the token grid (upstream ``shift_tensor``)."""
|
||||
if shift == 0:
|
||||
return grid
|
||||
out = torch.zeros_like(grid)
|
||||
w = grid.shape[1]
|
||||
if shift > 0:
|
||||
s = min(shift, w)
|
||||
out[:, s:] = grid[:, : w - s]
|
||||
else:
|
||||
s = min(-shift, w)
|
||||
out[:, : w - s] = grid[:, s:]
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# the two ODEs
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def rf_invert(apply_fn, image_latents, sigmas, gamma=1.0, progress=None):
|
||||
"""RF-Inversion forward ODE (Eq. 8 of https://arxiv.org/pdf/2410.10792).
|
||||
|
||||
``apply_fn(x, t) -> velocity`` must return the *raw* velocity (the guidance
|
||||
embedding travels inside the conditioning, as in the stock sampler).
|
||||
|
||||
Note the upstream asymmetry (reproduced verbatim):
|
||||
* the model is evaluated at the **raw** time ``t_i = i / (steps + 1)`` (NOT the
|
||||
shifted sigma), while the ODE step uses the **shifted** sigmas;
|
||||
* the source branch uses guidance 0.0 (set on the conditioning by the node).
|
||||
"""
|
||||
y_t = image_latents
|
||||
y_1 = torch.randn_like(y_t)
|
||||
n = len(sigmas) - 1 # number of inversion steps
|
||||
for i in range(n):
|
||||
t_i = i / (n + 1)
|
||||
u_t_i = apply_fn(y_t, t_i)
|
||||
u_t_i_cond = (y_1 - y_t) / (1.0 - t_i)
|
||||
u_hat = u_t_i + gamma * (u_t_i_cond - u_t_i)
|
||||
y_t = y_t + u_hat * (sigmas[i] - sigmas[i + 1])
|
||||
if progress is not None:
|
||||
progress(i + 1, n)
|
||||
return y_t
|
||||
|
||||
|
||||
def parallel_reverse(apply_fn, latents, y_0, sigmas, eta=1.0, start=0, stop=None,
|
||||
state: PAState = None, progress=None):
|
||||
"""Controlled reverse ODE (Algorithm 2 of RF-Inversion).
|
||||
|
||||
``latents`` is ``[source_branch, target_branch]`` (or ``[fg, bg, target]``):
|
||||
* every row but the last one follows the *conditional* field towards ``y_0``
|
||||
(the encoded source image) with strength ``eta``;
|
||||
* the last row is the plain euler step of the target image.
|
||||
|
||||
The model is evaluated at ``sigma`` (the shifted sigma) - exactly like upstream
|
||||
(``timestep / 1000`` with ``timestep = sigma * 1000``).
|
||||
"""
|
||||
if stop is None:
|
||||
stop = len(sigmas) - 1
|
||||
n = len(sigmas) - 1
|
||||
for i in range(n):
|
||||
sigma = sigmas[i]
|
||||
sigma_next = sigmas[i + 1]
|
||||
if state is not None:
|
||||
state.model_t = float(sigma)
|
||||
noise_pred = apply_fn(latents, float(sigma))
|
||||
|
||||
v_t = -noise_pred[:-1]
|
||||
v_t_cond = (y_0 - latents[:-1]) / sigma
|
||||
eta_t = eta if start <= i < stop else 0.0
|
||||
v_hat = v_t + eta_t * (v_t_cond - v_t)
|
||||
latents[:-1] = latents[:-1] + v_hat * (sigma - sigma_next)
|
||||
latents[-1] = latents[-1] + (sigma_next - sigma) * noise_pred[-1]
|
||||
if progress is not None:
|
||||
progress(i + 1, n)
|
||||
return latents
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# prompt / image helpers
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def encode_prompt(clip, text, min_length=512, device=None):
|
||||
"""comfy CLIP -> (t5 context [1, T, 4096], clip_l pooled [1, 768]).
|
||||
|
||||
``min_length`` = 512 reproduces the official diffusers padding
|
||||
(``padding="max_length", max_length=512``); comfy's flux t5xxl tokenizer defaults
|
||||
to 256, which would put the model off its training distribution.
|
||||
"""
|
||||
options = {}
|
||||
if min_length:
|
||||
options["t5xxl_min_length"] = int(min_length)
|
||||
tokens = clip.tokenize(text, tokenizer_options=options)
|
||||
out = clip.encode_from_tokens(tokens, return_pooled=True, return_dict=True)
|
||||
cond = out["cond"]
|
||||
pooled = out.get("pooled_output", None)
|
||||
if device is not None:
|
||||
cond = cond.to(device)
|
||||
if pooled is not None:
|
||||
pooled = pooled.to(device)
|
||||
return cond, pooled
|
||||
|
||||
|
||||
def encode_prompts(clip, texts, min_length=512, device=None):
|
||||
"""Encode several prompts, one call each (keeps the per-prompt padding exact)."""
|
||||
ctxs, pools = [], []
|
||||
for t in texts:
|
||||
c, p = encode_prompt(clip, t, min_length=min_length, device=device)
|
||||
ctxs.append(c)
|
||||
pools.append(p)
|
||||
return torch.cat(ctxs, dim=0), (torch.cat(pools, dim=0) if pools[0] is not None else None)
|
||||
+423
@@ -0,0 +1,423 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""ComfyUI nodes for Personalize Anything (native FLUX DiT stack).
|
||||
|
||||
Architecture follows comfy's own conventions - this plugin does **not** load or touch
|
||||
the CLIP and the VAE at all:
|
||||
|
||||
Load Diffusion Model -> MODEL ──────────────┐
|
||||
Load CLIP + CLIPTextEncode("") -> CONDITIONING (source branch) ────┐
|
||||
Load CLIP + CLIPTextEncode + FluxGuidance -> CONDITIONING (target) ─┤
|
||||
Load Image -> VAEEncode -> LATENT (source image) ───────────────────┤
|
||||
▼
|
||||
PersonalizeAnythingInvert
|
||||
-> inverted_latent + source_latent
|
||||
▼
|
||||
PersonalizeAnythingSample -> LATENT
|
||||
▼
|
||||
VAEDecode -> IMAGE
|
||||
|
||||
Because CLIP / VAE / DiT live in different nodes, comfy's model management can unload
|
||||
each of them when the next one needs the VRAM - the old wrapper kept all of them in one
|
||||
process, which is what blew up the 12 GB card.
|
||||
|
||||
All LATENT inputs/outputs follow the stock convention: **raw (unscaled) latents**, the
|
||||
same space VAEEncode / EmptySD3LatentImage / KSampler / VAEDecode use. The scaling into
|
||||
the DiT's sampling space happens inside the nodes via ``model.process_latent_in/out``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import os
|
||||
import comfy.model_management
|
||||
import comfy.sample
|
||||
import comfy.samplers
|
||||
import comfy.utils
|
||||
|
||||
from . import pa_core
|
||||
from . import pa_sampling
|
||||
|
||||
MODES = [
|
||||
"inpainting",
|
||||
"outpainting",
|
||||
"single_personalize",
|
||||
"subject_reconstruction",
|
||||
]
|
||||
|
||||
# modes where the comfy MASK means "region to replace" and therefore has to be
|
||||
# inverted to obtain the upstream "region to copy" mask
|
||||
INVERT_MASK_MODES = ("inpainting", "outpainting")
|
||||
|
||||
|
||||
def _activation_budget(patcher, latent_shape):
|
||||
"""VRAM budget to keep free for the activations of one forward.
|
||||
|
||||
comfy's own estimate (BaseModel.memory_required) is tuned for the stock KSampler
|
||||
and is far too small for our batch-2/3 parallel forward on a 12 GB card, where the
|
||||
11.9 GB fp8 flux weights and the activations have to share the card. We keep
|
||||
comfy's formula (scaled) with a 2 GB floor.
|
||||
"""
|
||||
base_model = patcher.model
|
||||
batch = max(1, int(latent_shape[0]))
|
||||
area = batch * int(latent_shape[-2]) * int(latent_shape[-1])
|
||||
dtype_size = comfy.model_management.dtype_size(base_model.get_dtype_inference())
|
||||
mem_factor = getattr(base_model, "memory_usage_factor", 3.1) or 3.1
|
||||
est = int(area * dtype_size * 0.01 * mem_factor * (1024 * 1024))
|
||||
floor = int(float(os.environ.get("PA_VRAM_BUDGET", "2")) * (1024 ** 3))
|
||||
return max(est * 2, floor)
|
||||
|
||||
|
||||
def _prepare_model(patcher, latent_shape):
|
||||
"""Load the model onto the GPU and return (BaseModel, device).
|
||||
|
||||
Memory rules learned on this machine (DiffuEraser / PiD notes):
|
||||
* ``free_memory(need)`` only *partially* unloads - flux would keep ~10 GB
|
||||
resident. ``unload_all_models()`` detaches everything first (the CLIP and the
|
||||
VAE reload on demand when their own nodes run).
|
||||
* the amount of model kept in VRAM is ``free - minimum_memory_required``, so the
|
||||
budget passed here directly decides how much of the fp8 flux weights stay
|
||||
resident and how much streams from RAM per layer.
|
||||
"""
|
||||
need = _activation_budget(patcher, latent_shape)
|
||||
comfy.model_management.unload_all_models()
|
||||
comfy.model_management.soft_empty_cache()
|
||||
comfy.model_management.load_models_gpu([patcher], memory_required=need,
|
||||
minimum_memory_required=need)
|
||||
patcher.pre_run()
|
||||
free, total = torch.cuda.mem_get_info()
|
||||
print(f"[PersonalizeAnything] model loaded: budget={need/2**30:.2f}GB "
|
||||
f"alloc={torch.cuda.memory_allocated()/2**30:.2f}GB "
|
||||
f"dev_free={free/2**30:.2f}/{total/2**30:.2f}GB")
|
||||
return patcher.model, patcher.load_device
|
||||
|
||||
|
||||
def _cond_rows(conditioning):
|
||||
"""comfy CONDITIONING ([[emb, dict], ...]) -> cond dict for ``model.extra_conds``."""
|
||||
if conditioning is None or len(conditioning) == 0:
|
||||
raise ValueError("empty conditioning input")
|
||||
if len(conditioning) > 1:
|
||||
print(f"[PersonalizeAnything] warning: conditioning has {len(conditioning)} entries, "
|
||||
"only the first one is used")
|
||||
emb, extra = conditioning[0][0], conditioning[0][1]
|
||||
row = dict(extra)
|
||||
row["cross_attn"] = emb
|
||||
return row
|
||||
|
||||
|
||||
def _rows_to_model_conds(base_model, rows, noise, device):
|
||||
"""One cond dict per batch row -> merged kwargs for ``apply_model``.
|
||||
|
||||
Uses the model's own ``extra_conds`` (exactly what ``process_conds`` does for the
|
||||
stock sampler) so every conditioning key (pooled_output, guidance, attention_mask,
|
||||
reference_latents, ...) keeps its standard meaning, then batches the rows with
|
||||
comfy's own ``cond_cat``.
|
||||
"""
|
||||
per_row = []
|
||||
for row in rows:
|
||||
d = dict(row)
|
||||
d.setdefault("noise", noise)
|
||||
d.setdefault("device", device)
|
||||
per_row.append(base_model.extra_conds(**d))
|
||||
return comfy.samplers.cond_cat(per_row)
|
||||
|
||||
|
||||
def _make_apply_fn(real_model, transformer_options, model_conds):
|
||||
"""Build the ``x, t -> raw velocity`` closure used by both ODEs.
|
||||
|
||||
The guidance embedding is *not* passed here - it lives in the conditioning
|
||||
(FluxGuidance node / the invert node's `guidance` input) exactly like the stock
|
||||
sampler, so it survives the cond_cat batching.
|
||||
"""
|
||||
|
||||
def apply_fn(x, t):
|
||||
t_tensor = torch.full((x.shape[0],), float(t), device=x.device, dtype=torch.float32)
|
||||
out = real_model.apply_model(
|
||||
x,
|
||||
t_tensor,
|
||||
transformer_options=transformer_options,
|
||||
**model_conds,
|
||||
)
|
||||
return out.float()
|
||||
|
||||
return apply_fn
|
||||
|
||||
|
||||
def _run_reverse(model, latents, y_0, rows, inject, target_row, seed, steps, tau, eta,
|
||||
mode, concept_process=False):
|
||||
"""Shared driver of the controlled reverse ODE for Sample / Compose.
|
||||
|
||||
``latents`` is in the *scaled* space, the LAST row being the target branch and the
|
||||
preceding rows the source branches. ``rows`` is one cond dict per batch row, in
|
||||
the same order. Returns the target row (scaled space).
|
||||
"""
|
||||
device = model.load_device
|
||||
base_model = model.model
|
||||
h_tok, w_tok = pa_core.token_grid(latents.shape)
|
||||
|
||||
sigmas, mu = pa_sampling.official_sigmas(steps, h_tok * w_tok, device=device)
|
||||
|
||||
state = pa_core.PAState()
|
||||
state.inject = inject
|
||||
state.target_row = target_row
|
||||
state.tau = float(tau) / 100.0
|
||||
state.concept_process = bool(concept_process)
|
||||
transformer_options = pa_core.build_transformer_options(model, state)
|
||||
|
||||
real_model, device = _prepare_model(model, latents.shape)
|
||||
# comfy's prepare_noise is intentionally generated on CPU -> move it onto the
|
||||
# latents' device (the stock sampler does the same move inside inner_sample)
|
||||
noise = comfy.sample.prepare_noise(torch.zeros_like(latents[-1:]), seed).to(
|
||||
device=latents.device, dtype=torch.float32)
|
||||
model_conds = _rows_to_model_conds(real_model, rows, noise, device)
|
||||
apply_fn = _make_apply_fn(real_model, transformer_options, model_conds)
|
||||
|
||||
pbar = comfy.utils.ProgressBar(steps)
|
||||
state.enabled = True
|
||||
|
||||
with comfy.model_management.cuda_device_context(device):
|
||||
with pa_core.raw_velocity(real_model):
|
||||
latents = pa_core.parallel_reverse(
|
||||
apply_fn, latents, y_0, sigmas, eta=eta, start=0,
|
||||
stop=len(sigmas) - 1, state=state,
|
||||
progress=lambda i, n: pbar.update_absolute(i, n),
|
||||
)
|
||||
|
||||
target = latents[-1:].contiguous()
|
||||
n_subj = sum(int(m.numel()) for _, m, _ in (state.inject or []))
|
||||
print(f"[PersonalizeAnything] {mode} | {pa_sampling.describe(steps, h_tok * w_tok, mu)} | "
|
||||
f"tau={tau} eta={eta} seed={seed} | subject tokens={n_subj} | "
|
||||
f"patch fired {state.fire_count}x")
|
||||
return target
|
||||
|
||||
|
||||
class PersonalizeAnythingInvert:
|
||||
"""RF-Inversion forward ODE: source image latent -> noise latent (source branch)."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"conditioning": ("CONDITIONING", {"tooltip": "Prompt of the source branch. "
|
||||
"The original paper uses an empty prompt."}),
|
||||
"latent_image": ("LATENT", {"tooltip": "Raw latent of the source image "
|
||||
"(e.g. from VAEEncode)."}),
|
||||
"steps": ("INT", {"default": 28, "min": 1, "max": 100, "step": 1}),
|
||||
"gamma": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05}),
|
||||
"guidance": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 30.0, "step": 0.1,
|
||||
"tooltip": "Inversion guidance; the original pipeline "
|
||||
"hardcodes source_guidance_scale = 0.0."}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LATENT", "LATENT")
|
||||
RETURN_NAMES = ("inverted_latent", "source_latent")
|
||||
FUNCTION = "run"
|
||||
CATEGORY = "PersonalizeAnything"
|
||||
DESCRIPTION = (
|
||||
"RF-Inversion (https://arxiv.org/pdf/2410.10792) of the source image on comfy's native "
|
||||
"FLUX DiT. Produces the 'source branch' latent that the sampler parallelises with the "
|
||||
"fresh noise latent. Keep `steps` equal to the sampler's `steps` (the original algorithm "
|
||||
"uses the same discretisation for both directions). No CLIP / VAE inside - they stay in "
|
||||
"their own nodes so comfy can unload them."
|
||||
)
|
||||
|
||||
def run(self, model, conditioning, latent_image, steps, gamma, guidance):
|
||||
device = model.load_device
|
||||
base_model = model.model
|
||||
|
||||
raw = latent_image["samples"].to(device=device, dtype=torch.float32)
|
||||
scaled = base_model.process_latent_in(raw)
|
||||
|
||||
h_tok, w_tok = pa_core.token_grid(scaled.shape)
|
||||
sigmas, mu = pa_sampling.official_sigmas(steps, h_tok * w_tok, device=device)
|
||||
|
||||
row = _cond_rows(conditioning)
|
||||
row["guidance"] = float(guidance) # upstream: source_guidance_scale = 0.0
|
||||
noise = torch.zeros_like(scaled)
|
||||
|
||||
state = pa_core.PAState()
|
||||
transformer_options = pa_core.build_transformer_options(model, state)
|
||||
|
||||
real_model, device = _prepare_model(model, scaled.shape)
|
||||
model_conds = _rows_to_model_conds(real_model, [row], noise, device)
|
||||
apply_fn = _make_apply_fn(real_model, transformer_options, model_conds)
|
||||
|
||||
pbar = comfy.utils.ProgressBar(steps)
|
||||
state.enabled = False # the inversion itself is *not* personalized
|
||||
|
||||
with comfy.model_management.cuda_device_context(device):
|
||||
with pa_core.raw_velocity(real_model):
|
||||
inverted = pa_core.rf_invert(
|
||||
apply_fn, scaled, sigmas, gamma=gamma,
|
||||
progress=lambda i, n: pbar.update_absolute(i, n),
|
||||
)
|
||||
|
||||
print(f"[PersonalizeAnything] inverted {tuple(scaled.shape)} | "
|
||||
f"{pa_sampling.describe(steps, h_tok * w_tok, mu)} | gamma={gamma}")
|
||||
# LATENT convention: raw (unscaled) latents, like VAEEncode / KSampler / VAEDecode
|
||||
return ({"samples": base_model.process_latent_out(inverted)},
|
||||
{"samples": raw})
|
||||
|
||||
|
||||
class PersonalizeAnythingSample:
|
||||
"""Controlled reverse ODE with the timestep-adaptive token replacement (1 source branch)."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"conditioning": ("CONDITIONING", {"tooltip": "Prompt of the generated image "
|
||||
"(target branch, last row)."}),
|
||||
"source_conditioning": ("CONDITIONING", {"tooltip": "Prompt of the source branch "
|
||||
"(row 0, the inverted latent). "
|
||||
"Must match the Invert node's "
|
||||
"conditioning (upstream: empty)."}),
|
||||
"inverted_latent": ("LATENT", {"tooltip": "From PersonalizeAnythingInvert."}),
|
||||
"source_latent": ("LATENT", {"tooltip": "From PersonalizeAnythingInvert."}),
|
||||
"mask": ("MASK",),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "display": "number"}),
|
||||
"steps": ("INT", {"default": 28, "min": 1, "max": 100, "step": 1}),
|
||||
"tau": ("FLOAT", {"default": 50.0, "min": 0.0, "max": 100.0, "step": 1.0,
|
||||
"tooltip": "Timestep (0-100) above which the subject tokens are "
|
||||
"replaced. Smaller tau = stronger local consistency, "
|
||||
"less global harmony. 50 = upstream default."}),
|
||||
"shift": ("INT", {"default": 0, "min": -512, "max": 512, "step": 1,
|
||||
"tooltip": "Horizontal shift of the target region, in image tokens "
|
||||
"(1 token = 16 px). Positive moves the subject left."}),
|
||||
"eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "Strength of the source branch (0 = ignore the source, "
|
||||
"1 = fully follow it). 1.0 = upstream default."}),
|
||||
"mode": (MODES,),
|
||||
},
|
||||
"optional": {
|
||||
"concept_process": ("BOOLEAN", {"default": False,
|
||||
"tooltip": "Also concatenate the subject tokens to the "
|
||||
"sequence (upstream `concept_process`). "
|
||||
"The released upstream app runs with it OFF."}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LATENT",)
|
||||
RETURN_NAMES = ("latent",)
|
||||
FUNCTION = "run"
|
||||
CATEGORY = "PersonalizeAnything"
|
||||
DESCRIPTION = (
|
||||
"Personalize Anything: the source branch (inverted latent) and the target branch (fresh "
|
||||
"noise) are denoised in parallel; while the timestep is above `tau`, the subject tokens of "
|
||||
"the source branch replace the target tokens selected by the mask.\n"
|
||||
"mask semantics: for inpainting / outpainting the MASK is the region to REPLACE and is "
|
||||
"inverted internally (the kept region is what gets copied, like upstream); for the "
|
||||
"personalization modes the MASK is the SUBJECT region itself.\n"
|
||||
"Plug the output into VAEDecode. Guidance comes from the conditioning (insert a "
|
||||
"FluxGuidance node for flux-dev; default 3.5).\n"
|
||||
"For multi-subject composition (two source images) use PersonalizeAnythingCompose."
|
||||
)
|
||||
|
||||
def run(self, model, conditioning, source_conditioning, inverted_latent, source_latent,
|
||||
mask, seed, steps, tau, shift, eta, mode, concept_process=False):
|
||||
device = model.load_device
|
||||
base_model = model.model
|
||||
|
||||
inv = base_model.process_latent_in(
|
||||
inverted_latent["samples"].to(device=device, dtype=torch.float32))
|
||||
src = base_model.process_latent_in(
|
||||
source_latent["samples"].to(device=device, dtype=torch.float32))
|
||||
h_tok, w_tok = pa_core.token_grid(inv.shape)
|
||||
|
||||
grid = pa_core.mask_to_grid(mask, h_tok, w_tok, invert=(mode in INVERT_MASK_MODES))
|
||||
shift_grid = pa_core.shift_grid(grid, shift)
|
||||
# index tensors must live on the model device (used by index_copy_ in the patch)
|
||||
mask_idx = pa_core.grid_to_indices(grid).to(device)
|
||||
shift_idx = pa_core.grid_to_indices(shift_grid).to(device)
|
||||
|
||||
noise = comfy.sample.prepare_noise(torch.zeros_like(inv), seed).to(
|
||||
device=inv.device, dtype=torch.float32)
|
||||
latents = torch.cat([inv, noise], dim=0) # row 0 = source, row 1 = target
|
||||
y_0 = src
|
||||
rows = [_cond_rows(source_conditioning), _cond_rows(conditioning)]
|
||||
|
||||
target = _run_reverse(model, latents, y_0, rows,
|
||||
inject=[(0, mask_idx, shift_idx)], target_row=1,
|
||||
seed=seed, steps=steps, tau=tau, eta=eta, mode=mode,
|
||||
concept_process=concept_process)
|
||||
# LATENT convention: raw (unscaled) latent -> plug straight into VAEDecode
|
||||
return ({"samples": base_model.process_latent_out(target)},)
|
||||
|
||||
|
||||
class PersonalizeAnythingCompose:
|
||||
"""Scene composition: two inverted branches (foreground + background) -> one image."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"conditioning": ("CONDITIONING", {"tooltip": "Prompt of the composed image "
|
||||
"(target branch, last row)."}),
|
||||
"fg_conditioning": ("CONDITIONING", {"tooltip": "Prompt of the foreground branch "
|
||||
"(row 0). Must match the foreground "
|
||||
"Invert node's conditioning."}),
|
||||
"bg_conditioning": ("CONDITIONING", {"tooltip": "Prompt of the background branch "
|
||||
"(row 1). Must match the background "
|
||||
"Invert node's conditioning."}),
|
||||
"fg_inverted_latent": ("LATENT", {"tooltip": "Invert node #1 (foreground image)."}),
|
||||
"fg_source_latent": ("LATENT", {"tooltip": "Invert node #1 (foreground image)."}),
|
||||
"bg_inverted_latent": ("LATENT", {"tooltip": "Invert node #2 (background image)."}),
|
||||
"bg_source_latent": ("LATENT", {"tooltip": "Invert node #2 (background image)."}),
|
||||
"fg_mask": ("MASK", {"tooltip": "Subject region of the FOREGROUND image; the "
|
||||
"background branch fills the complement."}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "display": "number"}),
|
||||
"steps": ("INT", {"default": 28, "min": 1, "max": 100, "step": 1}),
|
||||
"tau": ("FLOAT", {"default": 50.0, "min": 0.0, "max": 100.0, "step": 1.0}),
|
||||
"eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05}),
|
||||
},
|
||||
"optional": {
|
||||
"concept_process": ("BOOLEAN", {"default": False}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LATENT",)
|
||||
RETURN_NAMES = ("latent",)
|
||||
FUNCTION = "run"
|
||||
CATEGORY = "PersonalizeAnything"
|
||||
DESCRIPTION = (
|
||||
"Personalize Anything scene composition (upstream MultiPersonalizeAnythingAttnProcessor): "
|
||||
"TWO inverted branches run in parallel with the target - batch = [foreground, background, "
|
||||
"target]. Foreground subject tokens are copied into the target at the fg_mask positions, "
|
||||
"background tokens at the complement. Wire one PersonalizeAnythingInvert per source image "
|
||||
"(fg and bg), keep the prompt of each branch identical between its Invert node and here."
|
||||
)
|
||||
|
||||
def run(self, model, conditioning, fg_conditioning, bg_conditioning,
|
||||
fg_inverted_latent, fg_source_latent, bg_inverted_latent, bg_source_latent,
|
||||
fg_mask, seed, steps, tau, eta, concept_process=False):
|
||||
device = model.load_device
|
||||
base_model = model.model
|
||||
|
||||
inv_fg = base_model.process_latent_in(
|
||||
fg_inverted_latent["samples"].to(device=device, dtype=torch.float32))
|
||||
src_fg = base_model.process_latent_in(
|
||||
fg_source_latent["samples"].to(device=device, dtype=torch.float32))
|
||||
inv_bg = base_model.process_latent_in(
|
||||
bg_inverted_latent["samples"].to(device=device, dtype=torch.float32))
|
||||
src_bg = base_model.process_latent_in(
|
||||
bg_source_latent["samples"].to(device=device, dtype=torch.float32))
|
||||
h_tok, w_tok = pa_core.token_grid(inv_fg.shape)
|
||||
|
||||
fg_grid = pa_core.mask_to_grid(fg_mask, h_tok, w_tok, invert=False)
|
||||
bg_grid = ~fg_grid
|
||||
fg_idx = pa_core.grid_to_indices(fg_grid).to(device)
|
||||
bg_idx = pa_core.grid_to_indices(bg_grid).to(device)
|
||||
|
||||
noise = comfy.sample.prepare_noise(torch.zeros_like(inv_fg), seed).to(
|
||||
device=inv_fg.device, dtype=torch.float32)
|
||||
latents = torch.cat([inv_fg, inv_bg, noise], dim=0) # rows: fg, bg, target
|
||||
y_0 = torch.cat([src_fg, src_bg], dim=0)
|
||||
rows = [_cond_rows(fg_conditioning), _cond_rows(bg_conditioning),
|
||||
_cond_rows(conditioning)]
|
||||
|
||||
target = _run_reverse(model, latents, y_0, rows,
|
||||
inject=[(0, fg_idx, fg_idx), (1, bg_idx, bg_idx)],
|
||||
target_row=2, seed=seed, steps=steps, tau=tau, eta=eta,
|
||||
mode="scene_composition", concept_process=concept_process)
|
||||
return ({"samples": base_model.process_latent_out(target)},)
|
||||
@@ -0,0 +1,70 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""Official FLUX sigma grid, replayed on comfy's native stack.
|
||||
|
||||
Source of truth = the scheduler config that shipped with the original
|
||||
personalize-anything repo (``src/FLUX.1-dev/scheduler/scheduler_config.json``)::
|
||||
|
||||
FlowMatchEulerDiscreteScheduler
|
||||
base_image_seq_len 256 / max_image_seq_len 4096
|
||||
base_shift 0.5 / max_shift 1.15
|
||||
use_dynamic_shifting = True
|
||||
|
||||
diffusers 0.37 ``FlowMatchEulerDiscreteScheduler.set_timesteps(sigmas=..., mu=...)``::
|
||||
|
||||
sigmas = linspace(1.0, 1/steps, steps) # raw grid
|
||||
sigmas = time_shift(mu, 1.0, sigmas) # exp(mu)/(exp(mu)+(1/t-1))
|
||||
sigmas = cat([sigmas, 0.0]) # terminal
|
||||
|
||||
``mu = calculate_shift(image_seq_len, 256, 4096, 0.5, 1.15)`` - i.e. the shift is
|
||||
*resolution dependent* (256 tokens -> mu 0.5, 4096 tokens -> mu 1.15).
|
||||
|
||||
comfy's own ``comfy.model_sampling.flux_time_shift`` has the identical formula, but
|
||||
comfy's ``ModelSamplingFlux`` hard-codes ``shift=1.15`` for *every* resolution, so the
|
||||
stock KSampler is NOT equivalent to the official pipeline below 1024px. The whole
|
||||
Personalize-Anything algebra (RF-Inversion) is written against the *official* grid, so
|
||||
we recompute it here and drive the model ourselves.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
# scheduler_config.json of the original repo (FLUX.1-dev)
|
||||
BASE_SEQ = 256
|
||||
MAX_SEQ = 4096
|
||||
BASE_SHIFT = 0.5
|
||||
MAX_SHIFT = 1.15
|
||||
|
||||
|
||||
def calculate_shift(seq_len, base_seq=BASE_SEQ, max_seq=MAX_SEQ,
|
||||
base_shift=BASE_SHIFT, max_shift=MAX_SHIFT):
|
||||
"""diffusers.pipelines.flux.pipeline_flux.calculate_shift (identical in comfy)."""
|
||||
m = (max_shift - base_shift) / (max_seq - base_seq)
|
||||
b = base_shift - m * base_seq
|
||||
return seq_len * m + b
|
||||
|
||||
|
||||
def time_shift(mu, sigma, t):
|
||||
"""FlowMatchEulerDiscreteScheduler.time_shift (== comfy's flux_time_shift)."""
|
||||
return math.exp(mu) / (math.exp(mu) + (1.0 / t - 1.0) ** sigma)
|
||||
|
||||
|
||||
def official_sigmas(steps, tokens, device=None, dtype=torch.float32):
|
||||
"""Full sigma table (len = steps+1, terminal 0), diffusers-exact.
|
||||
|
||||
`tokens` = image_seq_len = (latent_h/2) * (latent_w/2) = number of image tokens.
|
||||
"""
|
||||
raw = np.linspace(1.0, 1.0 / steps, steps).astype(np.float32)
|
||||
mu = calculate_shift(tokens)
|
||||
shifted = np.array([time_shift(mu, 1.0, float(t)) for t in raw], dtype=np.float32)
|
||||
sig = np.concatenate([shifted, np.zeros(1, dtype=np.float32)])
|
||||
out = torch.from_numpy(sig).to(dtype=dtype)
|
||||
if device is not None:
|
||||
out = out.to(device)
|
||||
return out, mu
|
||||
|
||||
|
||||
def describe(steps, tokens, mu):
|
||||
return (f"steps={steps} tokens={tokens} mu={mu:.4f} "
|
||||
f"shift={math.exp(mu):.3f} sigma[0]={1.0:.4f}")
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
[project]
|
||||
name = "personalize_anything"
|
||||
description = "Personalize Anything for Free with Diffusion Transformer,use it in comfyUI with wrapper mode"
|
||||
version = "1.0.0"
|
||||
version = "2.0.0"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = ["accelerate", "diffusers>=0.32.2", "#ipykernel", "#gradio", "#matplotlib", "opencv-python", "protobuf", "sentencepiece", "transformers>=4.49.0"]
|
||||
dependencies = ["torch"]
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/smthemex/ComfyUI_Personalize_Anything"
|
||||
|
||||
+5
-9
@@ -1,9 +1,5 @@
|
||||
accelerate
|
||||
diffusers>=0.32.2
|
||||
#ipykernel
|
||||
#gradio
|
||||
#matplotlib
|
||||
opencv-python
|
||||
protobuf
|
||||
sentencepiece
|
||||
transformers>=4.49.0
|
||||
# Personalize Anything - native comfy port.
|
||||
# No diffusers / transformers / peft / accelerate / opencv needed any more:
|
||||
# the plugin runs on comfy's own FLUX DiT, CLIP and VAE implementations.
|
||||
# (The legacy diffusers wrapper is still in the folder for archaeology only.)
|
||||
torch
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"_class_name": "FluxTransformer2DModel",
|
||||
"_diffusers_version": "0.30.0.dev0",
|
||||
"_name_or_path": "../checkpoints/flux-dev/transformer",
|
||||
"attention_head_dim": 128,
|
||||
"axes_dims_rope": [
|
||||
16,
|
||||
56,
|
||||
56
|
||||
],
|
||||
"guidance_embeds": true,
|
||||
"in_channels": 64,
|
||||
"joint_attention_dim": 4096,
|
||||
"num_attention_heads": 24,
|
||||
"num_layers": 19,
|
||||
"num_single_layers": 38,
|
||||
"patch_size": 1,
|
||||
"pooled_projection_dim": 768
|
||||
}
|
||||
@@ -1,287 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import requests
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoModelForMaskGeneration, AutoProcessor, pipeline
|
||||
|
||||
|
||||
def create_palette():
|
||||
palette = [0, 0, 0, 255, 255, 255]
|
||||
palette += [0] * (768 - len(palette))
|
||||
|
||||
return palette
|
||||
|
||||
|
||||
PALETTE = create_palette()
|
||||
|
||||
|
||||
# Result Utils
|
||||
@dataclass
|
||||
class BoundingBox:
|
||||
xmin: int
|
||||
ymin: int
|
||||
xmax: int
|
||||
ymax: int
|
||||
|
||||
@property
|
||||
def xyxy(self) -> List[float]:
|
||||
return [self.xmin, self.ymin, self.xmax, self.ymax]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectionResult:
|
||||
score: Optional[float] = None
|
||||
label: Optional[str] = None
|
||||
box: Optional[BoundingBox] = None
|
||||
mask: Optional[np.array] = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, detection_dict: Dict) -> "DetectionResult":
|
||||
return cls(
|
||||
score=detection_dict["score"],
|
||||
label=detection_dict["label"],
|
||||
box=BoundingBox(
|
||||
xmin=detection_dict["box"]["xmin"],
|
||||
ymin=detection_dict["box"]["ymin"],
|
||||
xmax=detection_dict["box"]["xmax"],
|
||||
ymax=detection_dict["box"]["ymax"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Utils
|
||||
def mask_to_polygon(mask: np.ndarray) -> List[List[int]]:
|
||||
# Find contours in the binary mask
|
||||
contours, _ = cv2.findContours(
|
||||
mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
|
||||
)
|
||||
|
||||
# Find the contour with the largest area
|
||||
largest_contour = max(contours, key=cv2.contourArea)
|
||||
|
||||
# Extract the vertices of the contour
|
||||
polygon = largest_contour.reshape(-1, 2).tolist()
|
||||
|
||||
return polygon
|
||||
|
||||
|
||||
def polygon_to_mask(
|
||||
polygon: List[Tuple[int, int]], image_shape: Tuple[int, int]
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Convert a polygon to a segmentation mask.
|
||||
|
||||
Args:
|
||||
- polygon (list): List of (x, y) coordinates representing the vertices of the polygon.
|
||||
- image_shape (tuple): Shape of the image (height, width) for the mask.
|
||||
|
||||
Returns:
|
||||
- np.ndarray: Segmentation mask with the polygon filled.
|
||||
"""
|
||||
# Create an empty mask
|
||||
mask = np.zeros(image_shape, dtype=np.uint8)
|
||||
|
||||
# Convert polygon to an array of points
|
||||
pts = np.array(polygon, dtype=np.int32)
|
||||
|
||||
# Fill the polygon with white color (0)
|
||||
cv2.fillPoly(mask, [pts], color=255)
|
||||
|
||||
return mask
|
||||
|
||||
|
||||
def load_image(image_str: str) -> Image.Image:
|
||||
if image_str.startswith("http"):
|
||||
image = Image.open(requests.get(image_str, stream=True).raw).convert("RGB")
|
||||
else:
|
||||
image = Image.open(image_str).convert("RGB")
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def get_boxes(results: DetectionResult) -> List[List[List[float]]]:
|
||||
boxes = []
|
||||
for result in results:
|
||||
xyxy = result.box.xyxy
|
||||
boxes.append(xyxy)
|
||||
|
||||
return [boxes]
|
||||
|
||||
|
||||
def refine_masks(
|
||||
masks: torch.BoolTensor, polygon_refinement: bool = False
|
||||
) -> List[np.ndarray]:
|
||||
masks = masks.cpu().float()
|
||||
masks = masks.permute(0, 2, 3, 1)
|
||||
masks = masks.mean(axis=-1)
|
||||
masks = (masks > 0).int()
|
||||
masks = masks.numpy().astype(np.uint8)
|
||||
masks = list(masks)
|
||||
|
||||
if polygon_refinement:
|
||||
for idx, mask in enumerate(masks):
|
||||
shape = mask.shape
|
||||
polygon = mask_to_polygon(mask)
|
||||
mask = polygon_to_mask(polygon, shape)
|
||||
masks[idx] = mask
|
||||
|
||||
return masks
|
||||
|
||||
|
||||
# Post-processing Utils
|
||||
def generate_colored_segmentation(label_image):
|
||||
# Create a PIL Image from the label image (assuming it's a 2D numpy array)
|
||||
label_image_pil = Image.fromarray(label_image.astype(np.uint8), mode="P")
|
||||
|
||||
# Apply the palette to the image
|
||||
palette = create_palette()
|
||||
label_image_pil.putpalette(palette)
|
||||
|
||||
return label_image_pil
|
||||
|
||||
|
||||
def plot_segmentation(image, detections):
|
||||
seg_map = np.zeros(image.size[::-1], dtype=np.uint8)
|
||||
for i, detection in enumerate(detections):
|
||||
mask = detection.mask
|
||||
seg_map[mask > 0] = i + 1
|
||||
seg_map_pil = generate_colored_segmentation(seg_map)
|
||||
return seg_map_pil
|
||||
|
||||
|
||||
# Grounded SAM
|
||||
def prepare_model(
|
||||
device: str = "cuda",
|
||||
detector_id: Optional[str] = None,
|
||||
segmenter_id: Optional[str] = None,
|
||||
):
|
||||
detector_id = (
|
||||
detector_id if detector_id is not None else "IDEA-Research/grounding-dino-tiny"
|
||||
)
|
||||
object_detector = pipeline(
|
||||
model=detector_id, task="zero-shot-object-detection", device=device
|
||||
)
|
||||
|
||||
segmenter_id = segmenter_id if segmenter_id is not None else "facebook/sam-vit-base"
|
||||
processor = AutoProcessor.from_pretrained(segmenter_id)
|
||||
segmentator = AutoModelForMaskGeneration.from_pretrained(segmenter_id).to(device)
|
||||
|
||||
return object_detector, processor, segmentator
|
||||
|
||||
|
||||
def detect(
|
||||
object_detector: Any,
|
||||
image: Image.Image,
|
||||
labels: List[str],
|
||||
threshold: float = 0.3,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Use Grounding DINO to detect a set of labels in an image in a zero-shot fashion.
|
||||
"""
|
||||
labels = [label if label.endswith(".") else label + "." for label in labels]
|
||||
|
||||
results = object_detector(image, candidate_labels=labels, threshold=threshold)
|
||||
results = [DetectionResult.from_dict(result) for result in results]
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def segment(
|
||||
processor: Any,
|
||||
segmentator: Any,
|
||||
image: Image.Image,
|
||||
boxes: Optional[List[List[List[float]]]] = None,
|
||||
detection_results: Optional[List[Dict[str, Any]]] = None,
|
||||
polygon_refinement: bool = False,
|
||||
) -> List[DetectionResult]:
|
||||
"""
|
||||
Use Segment Anything (SAM) to generate masks given an image + a set of bounding boxes.
|
||||
"""
|
||||
if detection_results is None and boxes is None:
|
||||
raise ValueError(
|
||||
"Either detection_results or detection_boxes must be provided."
|
||||
)
|
||||
|
||||
if boxes is None:
|
||||
boxes = get_boxes(detection_results)
|
||||
print(boxes)
|
||||
inputs = processor(images=image, input_boxes=boxes, return_tensors="pt").to(
|
||||
segmentator.device, segmentator.dtype
|
||||
)
|
||||
|
||||
outputs = segmentator(**inputs)
|
||||
masks = processor.post_process_masks(
|
||||
masks=outputs.pred_masks,
|
||||
original_sizes=inputs.original_sizes,
|
||||
reshaped_input_sizes=inputs.reshaped_input_sizes,
|
||||
)[0]
|
||||
|
||||
masks = refine_masks(masks, polygon_refinement)
|
||||
|
||||
if detection_results is None:
|
||||
detection_results = [DetectionResult() for _ in masks]
|
||||
|
||||
for detection_result, mask in zip(detection_results, masks):
|
||||
detection_result.mask = mask
|
||||
|
||||
return detection_results
|
||||
|
||||
|
||||
def grounded_segmentation(
|
||||
object_detector,
|
||||
processor,
|
||||
segmentator,
|
||||
image: Union[Image.Image, str],
|
||||
labels: Union[str, List[str]],
|
||||
threshold: float = 0.3,
|
||||
polygon_refinement: bool = False,
|
||||
) -> Tuple[np.ndarray, List[DetectionResult], Image.Image]:
|
||||
if isinstance(image, str):
|
||||
image = load_image(image)
|
||||
if isinstance(labels, str):
|
||||
labels = labels.split(",")
|
||||
|
||||
detections = detect(object_detector, image, labels, threshold)
|
||||
print(detections)
|
||||
detections = segment(processor, segmentator, image, None, detections, polygon_refinement)
|
||||
|
||||
seg_map_pil = plot_segmentation(image, detections)
|
||||
|
||||
return np.array(image), detections, seg_map_pil
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--image", type=str, required=True)
|
||||
parser.add_argument("--labels", type=str, nargs="+", required=True)
|
||||
parser.add_argument("--threshold", type=float, default=0.3)
|
||||
parser.add_argument(
|
||||
"--detector_id", type=str, default="IDEA-Research/grounding-dino-tiny"
|
||||
)
|
||||
parser.add_argument("--segmenter_id", type=str, default="facebook/sam-vit-base")
|
||||
args = parser.parse_args()
|
||||
|
||||
device = "cuda:0" if torch.cuda.is_available() else "cpu"
|
||||
object_detector, processor, segmentator = prepare_model(
|
||||
device=device, detector_id=args.detector_id, segmenter_id=args.segmenter_id
|
||||
)
|
||||
|
||||
image_array, detections, seg_map_pil = grounded_segmentation(
|
||||
object_detector,
|
||||
processor,
|
||||
segmentator,
|
||||
image=args.image,
|
||||
labels=args.labels,
|
||||
threshold=args.threshold,
|
||||
polygon_refinement=True,
|
||||
)
|
||||
|
||||
prefix_path = os.path.dirname(args.image)
|
||||
seg_map_pil.save(os.path.join(prefix_path, "mask.png"))
|
||||
@@ -1,82 +0,0 @@
|
||||
**FLUX.1 \[dev\] Non-Commercial License**
|
||||
|
||||
Black Forest Labs, Inc. (“**we**” or “**our**” or “**Company**”) is pleased to make available the weights, parameters and inference code for the FLUX.1 \[dev\] Model (as defined below) freely available for your non-commercial and non-production use as set forth in this FLUX.1 \[dev\] Non-Commercial License (“**License**”). The “**FLUX.1 \[dev\] Model**” means the FLUX.1 \[dev\] text-to-image AI model and its elements which includes algorithms, software, checkpoints, parameters, source code (inference code, evaluation code, and if applicable, fine-tuning code) and any other materials associated with the FLUX.1 \[dev\] AI model made available by Company under this License, including if any, the technical documentation, manuals and instructions for the use and operation thereof (collectively, “**FLUX.1 \[dev\] Model**”).
|
||||
|
||||
By downloading, accessing, use, Distributing (as defined below), or creating a Derivative (as defined below) of the FLUX.1 \[dev\] Model, you agree to the terms of this License. If you do not agree to this License, then you do not have any rights to access, use, Distribute or create a Derivative of the FLUX.1 \[dev\] Model and you must immediately cease using the FLUX.1 \[dev\] Model. If you are agreeing to be bound by the terms of this License on behalf of your employer or other entity, you represent and warrant to us that you have full legal authority to bind your employer or such entity to this License. If you do not have the requisite authority, you may not accept the License or access the FLUX.1 \[dev\] Model on behalf of your employer or other entity.
|
||||
|
||||
1. **Definitions**. Capitalized terms used in this License but not defined herein have the following meanings:
|
||||
|
||||
1. “**Derivative**” means any (i) modified version of the FLUX.1 \[dev\] Model (including but not limited to any customized or fine-tuned version thereof), (ii) work based on the FLUX.1 \[dev\] Model, or (iii) any other derivative work thereof. For the avoidance of doubt, Outputs are not considered Derivatives under this License.
|
||||
|
||||
1. “**Distribution**” or “**Distribute**” or “**Distributing**” means providing or making available, by any means, a copy of the FLUX.1 \[dev\] Models and/or the Derivatives as the case may be.
|
||||
|
||||
1. “**Non-Commercial Purpose**” means any of the following uses, but only so far as you do not receive any direct or indirect payment arising from the use of the model or its output: (i) personal use for research, experiment, and testing for the benefit of public knowledge, personal study, private entertainment, hobby projects, or otherwise not directly or indirectly connected to any commercial activities, business operations, or employment responsibilities; (ii) use by commercial or for-profit entities for testing, evaluation, or non-commercial research and development in a non-production environment, (iii) use by any charitable organization for charitable purposes, or for testing or evaluation. For clarity, use for revenue-generating activity or direct interactions with or impacts on end users, or use to train, fine tune or distill other models for commercial use is not a Non-Commercial purpose.
|
||||
|
||||
1. “**Outputs**” means any content generated by the operation of the FLUX.1 \[dev\] Models or the Derivatives from a prompt (i.e., text instructions) provided by users. For the avoidance of doubt, Outputs do not include any components of a FLUX.1 \[dev\] Models, such as any fine-tuned versions of the FLUX.1 \[dev\] Models, the weights, or parameters.
|
||||
|
||||
1. “**you**” or “**your**” means the individual or entity entering into this License with Company.
|
||||
|
||||
1. **License Grant**.
|
||||
|
||||
1. License. Subject to your compliance with this License, Company grants you a non-exclusive, worldwide, non-transferable, non-sublicensable, revocable, royalty free and limited license to access, use, create Derivatives of, and Distribute the FLUX.1 \[dev\] Models solely for your Non-Commercial Purposes. The foregoing license is personal to you, and you may not assign or sublicense this License or any other rights or obligations under this License without Company’s prior written consent; any such assignment or sublicense will be void and will automatically and immediately terminate this License. Any restrictions set forth herein in regarding the FLUX.1 \[dev\] Model also applies to any Derivative you create or that are created on your behalf.
|
||||
|
||||
1. Non-Commercial Use Only. You may only access, use, Distribute, or creative Derivatives of or the FLUX.1 \[dev\] Model or Derivatives for Non-Commercial Purposes. If You want to use a FLUX.1 \[dev\] Model a Derivative for any purpose that is not expressly authorized under this License, such as for a commercial activity, you must request a license from Company, which Company may grant to you in Company’s sole discretion and which additional use may be subject to a fee, royalty or other revenue share. Please contact Company at the following e-mail address if you want to discuss such a license: info@blackforestlabs.ai.
|
||||
|
||||
1. Reserved Rights. The grant of rights expressly set forth in this License are the complete grant of rights to you in the FLUX.1 \[dev\] Model, and no other licenses are granted, whether by waiver, estoppel, implication, equity or otherwise. Company and its licensors reserve all rights not expressly granted by this License.
|
||||
|
||||
1. Outputs. We claim no ownership rights in and to the Outputs. You are solely responsible for the Outputs you generate and their subsequent uses in accordance with this License. You may use Output for any purpose (including for commercial purposes), except as expressly prohibited herein. You may not use the Output to train, fine-tune or distill a model that is competitive with the FLUX.1 \[dev\] Model.
|
||||
|
||||
1. **Distribution**. Subject to this License, you may Distribute copies of the FLUX.1 \[dev\] Model and/or Derivatives made by you, under the following conditions:
|
||||
|
||||
1. you must make available a copy of this License to third-party recipients of the FLUX.1 \[dev\] Models and/or Derivatives you Distribute, and specify that any rights to use the FLUX.1 \[dev\] Models and/or Derivatives shall be directly granted by Company to said third-party recipients pursuant to this License;
|
||||
|
||||
1. you must make prominently display the following notice alongside the Distribution of the FLUX.1 \[dev\] Model or Derivative (such as via a “Notice” text file distributed as part of such FLUX.1 \[dev\] Model or Derivative) (the “**Attribution Notice**”):
|
||||
|
||||
“The FLUX.1 \[dev\] Model is licensed by Black Forest Labs. Inc. under the FLUX.1 \[dev\] Non-Commercial License. Copyright Black Forest Labs. Inc.
|
||||
|
||||
IN NO EVENT SHALL BLACK FOREST LABS, INC. BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH USE OF THIS MODEL.”
|
||||
|
||||
1. in the case of Distribution of Derivatives made by you, you must also include in the Attribution Notice a statement that you have modified the applicable FLUX.1 \[dev\] Model; and
|
||||
|
||||
1. in the case of Distribution of Derivatives made by you, any terms and conditions you impose on any third-party recipients relating to Derivatives made by or for you shall neither limit such third-party recipients’ use of the FLUX.1 \[dev\] Model or any Derivatives made by or for Company in accordance with this License nor conflict with any of its terms and conditions.
|
||||
|
||||
1. In the case of Distribution of Derivatives made by you, you must not misrepresent or imply, through any means, that the Derivatives made by or for you and/or any modified version of the FLUX.1 \[dev\] Model you Distribute under your name and responsibility is an official product of the Company or has been endorsed, approved or validated by the Company, unless you are authorized by Company to do so in writing.
|
||||
|
||||
1. **Restrictions**. You will not, and will not permit, assist or cause any third party to
|
||||
|
||||
1. use, modify, copy, reproduce, create Derivatives of, or Distribute the FLUX.1 \[dev\] Model (or any Derivative thereof, or any data produced by the FLUX.1 \[dev\] Model), in whole or in part, for (i) any commercial or production purposes, (ii) military purposes, (iii) purposes of surveillance, including any research or development relating to surveillance, (iv) biometric processing, (v) in any manner that infringes, misappropriates, or otherwise violates any third-party rights, or (vi) in any manner that violates any applicable law and violating any privacy or security laws, rules, regulations, directives, or governmental requirements (including the General Data Privacy Regulation (Regulation (EU) 2016/679), the California Consumer Privacy Act, and any and all laws governing the processing of biometric information), as well as all amendments and successor laws to any of the foregoing;
|
||||
|
||||
1. alter or remove copyright and other proprietary notices which appear on or in any portion of the FLUX.1 \[dev\] Model;
|
||||
|
||||
1. utilize any equipment, device, software, or other means to circumvent or remove any security or protection used by Company in connection with the FLUX.1 \[dev\] Model, or to circumvent or remove any usage restrictions, or to enable functionality disabled by FLUX.1 \[dev\] Model; or
|
||||
|
||||
1. offer or impose any terms on the FLUX.1 \[dev\] Model that alter, restrict, or are inconsistent with the terms of this License.
|
||||
|
||||
1. violate any applicable U.S. and non-U.S. export control and trade sanctions laws (“**Export Laws**”) in connection with your use or Distribution of any FLUX.1 \[dev\] Model;
|
||||
|
||||
1. directly or indirectly Distribute, export, or otherwise transfer FLUX.1 \[dev\] Model (a) to any individual, entity, or country prohibited by Export Laws; (b) to anyone on U.S. or non-U.S. government restricted parties lists; or (c) for any purpose prohibited by Export Laws, including nuclear, chemical or biological weapons, or missile technology applications; 3\) use or download FLUX.1 \[dev\] Model if you or they are (a) located in a comprehensively sanctioned jurisdiction, (b) currently listed on any U.S. or non-U.S. restricted parties list, or (c) for any purpose prohibited by Export Laws; and (4) will not disguise your location through IP proxying or other methods.
|
||||
|
||||
1. **DISCLAIMERS**. THE FLUX.1 \[dev\] MODEL IS PROVIDED “AS IS” AND “WITH ALL FAULTS” WITH NO WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. COMPANY EXPRESSLY DISCLAIMS ALL REPRESENTATIONS AND WARRANTIES, EXPRESS OR IMPLIED, WHETHER BY STATUTE, CUSTOM, USAGE OR OTHERWISE AS TO ANY MATTERS RELATED TO THE FLUX.1 \[dev\] MODEL, INCLUDING BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE, SATISFACTORY QUALITY, OR NON-INFRINGEMENT. COMPANY MAKES NO WARRANTIES OR REPRESENTATIONS THAT THE FLUX.1 \[dev\] MODEL WILL BE ERROR FREE OR FREE OF VIRUSES OR OTHER HARMFUL COMPONENTS, OR PRODUCE ANY PARTICULAR RESULTS.
|
||||
|
||||
1. **LIMITATION OF LIABILITY**. TO THE FULLEST EXTENT PERMITTED BY LAW, IN NO EVENT WILL COMPANY BE LIABLE TO YOU OR YOUR EMPLOYEES, AFFILIATES, USERS, OFFICERS OR DIRECTORS (A) UNDER ANY THEORY OF LIABILITY, WHETHER BASED IN CONTRACT, TORT, NEGLIGENCE, STRICT LIABILITY, WARRANTY, OR OTHERWISE UNDER THIS LICENSE, OR (B) FOR ANY INDIRECT, CONSEQUENTIAL, EXEMPLARY, INCIDENTAL, PUNITIVE OR SPECIAL DAMAGES OR LOST PROFITS, EVEN IF COMPANY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. THE FLUX.1 \[dev\] MODEL, ITS CONSTITUENT COMPONENTS, AND ANY OUTPUT (COLLECTIVELY, “MODEL MATERIALS”) ARE NOT DESIGNED OR INTENDED FOR USE IN ANY APPLICATION OR SITUATION WHERE FAILURE OR FAULT OF THE MODEL MATERIALS COULD REASONABLY BE ANTICIPATED TO LEAD TO SERIOUS INJURY OF ANY PERSON, INCLUDING POTENTIAL DISCRIMINATION OR VIOLATION OF AN INDIVIDUAL’S PRIVACY RIGHTS, OR TO SEVERE PHYSICAL, PROPERTY, OR ENVIRONMENTAL DAMAGE (EACH, A “**HIGH-RISK USE**”). IF YOU ELECT TO USE ANY OF THE MODEL MATERIALS FOR A HIGH-RISK USE, YOU DO SO AT YOUR OWN RISK. YOU AGREE TO DESIGN AND IMPLEMENT APPROPRIATE DECISION-MAKING AND RISK-MITIGATION PROCEDURES AND POLICIES IN CONNECTION WITH A HIGH-RISK USE SUCH THAT EVEN IF THERE IS A FAILURE OR FAULT IN ANY OF THE MODEL MATERIALS, THE SAFETY OF PERSONS OR PROPERTY AFFECTED BY THE ACTIVITY STAYS AT A LEVEL THAT IS REASONABLE, APPROPRIATE, AND LAWFUL FOR THE FIELD OF THE HIGH-RISK USE.
|
||||
|
||||
1. **INDEMNIFICATION**
|
||||
|
||||
You will indemnify, defend and hold harmless Company and our subsidiaries and affiliates, and each of our respective shareholders, directors, officers, employees, agents, successors, and assigns (collectively, the “**Company Parties**”) from and against any losses, liabilities, damages, fines, penalties, and expenses (including reasonable attorneys’ fees) incurred by any Company Party in connection with any claim, demand, allegation, lawsuit, proceeding, or investigation (collectively, “**Claims**”) arising out of or related to (a) your access to or use of the FLUX.1 \[dev\] Model (as well as any Output, results or data generated from such access or use), including any High-Risk Use (defined below); (b) your violation of this License; or (c) your violation, misappropriation or infringement of any rights of another (including intellectual property or other proprietary rights and privacy rights). You will promptly notify the Company Parties of any such Claims, and cooperate with Company Parties in defending such Claims. You will also grant the Company Parties sole control of the defense or settlement, at Company’s sole option, of any Claims. This indemnity is in addition to, and not in lieu of, any other indemnities or remedies set forth in a written agreement between you and Company or the other Company Parties.
|
||||
|
||||
1. **Termination; Survival**.
|
||||
|
||||
1. This License will automatically terminate upon any breach by you of the terms of this License.
|
||||
|
||||
1. We may terminate this License, in whole or in part, at any time upon notice (including electronic) to you.
|
||||
|
||||
1. If You initiate any legal action or proceedings against Company or any other entity (including a cross-claim or counterclaim in a lawsuit), alleging that the FLUX.1 \[dev\] Model or any Derivative, or any part thereof, infringe upon intellectual property or other rights owned or licensable by you, then any licenses granted to you under this License will immediately terminate as of the date such legal action or claim is filed or initiated.
|
||||
|
||||
1. Upon termination of this License, you must cease all use, access or Distribution of the FLUX.1 \[dev\] Model and any Derivatives. The following sections survive termination of this License 2(c), 2(d), 4-11.
|
||||
|
||||
1. **Third Party Materials**. The FLUX.1 \[dev\] Model may contain third-party software or other components (including free and open source software) (all of the foregoing, “**Third Party Materials**”), which are subject to the license terms of the respective third-party licensors. Your dealings or correspondence with third parties and your use of or interaction with any Third Party Materials are solely between you and the third party. Company does not control or endorse, and makes no representations or warranties regarding, any Third Party Materials, and your access to and use of such Third Party Materials are at your own risk.
|
||||
|
||||
1. **Trademarks**. You have not been granted any trademark license as part of this License and may not use any name or mark associated with Company without the prior written permission of Company, except to the extent necessary to make the reference required in the Attribution Notice as specified above or as is reasonably necessary in describing the FLUX.1 \[dev\] Model and its creators.
|
||||
|
||||
1. **General**. This License will be governed and construed under the laws of the State of Delaware without regard to conflicts of law provisions. If any provision or part of a provision of this License is unlawful, void or unenforceable, that provision or part of the provision is deemed severed from this License, and will not affect the validity and enforceability of any remaining provisions. The failure of Company to exercise or enforce any right or provision of this License will not operate as a waiver of such right or provision. This License does not confer any third-party beneficiary rights upon any other person or entity. This License, together with the Documentation, contains the entire understanding between you and Company regarding the subject matter of this License, and supersedes all other written or oral agreements and understandings between you and Company regarding such subject matter. No change or addition to any provision of this License will be binding unless it is in writing and signed by an authorized representative of both you and Company.
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
---
|
||||
language:
|
||||
- en
|
||||
license: other
|
||||
license_name: flux-1-dev-non-commercial-license
|
||||
license_link: LICENSE.md
|
||||
extra_gated_prompt: By clicking "Agree", you agree to the [FluxDev Non-Commercial License Agreement](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md)
|
||||
and acknowledge the [Acceptable Use Policy](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/POLICY.md).
|
||||
tags:
|
||||
- text-to-image
|
||||
- image-generation
|
||||
- flux
|
||||
---
|
||||
|
||||
![FLUX.1 [dev] Grid](./dev_grid.jpg)
|
||||
|
||||
`FLUX.1 [dev]` is a 12 billion parameter rectified flow transformer capable of generating images from text descriptions.
|
||||
For more information, please read our [blog post](https://blackforestlabs.ai/announcing-black-forest-labs/).
|
||||
|
||||
# Key Features
|
||||
1. Cutting-edge output quality, second only to our state-of-the-art model `FLUX.1 [pro]`.
|
||||
2. Competitive prompt following, matching the performance of closed source alternatives .
|
||||
3. Trained using guidance distillation, making `FLUX.1 [dev]` more efficient.
|
||||
4. Open weights to drive new scientific research, and empower artists to develop innovative workflows.
|
||||
5. Generated outputs can be used for personal, scientific, and commercial purposes as described in the [flux-1-dev-non-commercial-license](./licence.md).
|
||||
|
||||
# Usage
|
||||
We provide a reference implementation of `FLUX.1 [dev]`, as well as sampling code, in a dedicated [github repository](https://github.com/black-forest-labs/flux).
|
||||
Developers and creatives looking to build on top of `FLUX.1 [dev]` are encouraged to use this as a starting point.
|
||||
|
||||
## API Endpoints
|
||||
The FLUX.1 models are also available via API from the following sources
|
||||
1. [bfl.ml](https://docs.bfl.ml/) (currently `FLUX.1 [pro]`)
|
||||
2. [replicate.com](https://replicate.com/collections/flux)
|
||||
3. [fal.ai](https://fal.ai/models/fal-ai/flux/dev)
|
||||
|
||||
## ComfyUI
|
||||
`FLUX.1 [dev]` is also available in [Comfy UI](https://github.com/comfyanonymous/ComfyUI) for local inference with a node-based workflow.
|
||||
|
||||
## Diffusers
|
||||
|
||||
To use `FLUX.1 [dev]` with the 🧨 diffusers python library, first install or upgrade diffusers
|
||||
|
||||
```shell
|
||||
pip install git+https://github.com/huggingface/diffusers.git
|
||||
```
|
||||
|
||||
Then you can use `FluxPipeline` to run the model
|
||||
|
||||
```python
|
||||
import torch
|
||||
from diffusers import FluxPipeline
|
||||
|
||||
pipe = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-dev", torch_dtype=torch.bfloat16)
|
||||
pipe.enable_model_cpu_offload() #save some VRAM by offloading the model to CPU. Remove this if you have enough GPU power
|
||||
|
||||
prompt = "A cat holding a sign that says hello world"
|
||||
image = pipe(
|
||||
prompt,
|
||||
height=1024,
|
||||
width=1024,
|
||||
guidance_scale=3.5,
|
||||
output_type="pil",
|
||||
num_inference_steps=50,
|
||||
max_sequence_length=512,
|
||||
generator=torch.Generator("cpu").manual_seed(0)
|
||||
).images[0]
|
||||
image.save("flux-dev.png")
|
||||
```
|
||||
|
||||
To learn more check out the [diffusers](https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux) documentation
|
||||
|
||||
---
|
||||
# Limitations
|
||||
- This model is not intended or able to provide factual information.
|
||||
- As a statistical model this checkpoint might amplify existing societal biases.
|
||||
- The model may fail to generate output that matches the prompts.
|
||||
- Prompt following is heavily influenced by the prompting-style.
|
||||
|
||||
# Out-of-Scope Use
|
||||
The model and its derivatives may not be used
|
||||
|
||||
- In any way that violates any applicable national, federal, state, local or international law or regulation.
|
||||
- For the purpose of exploiting, harming or attempting to exploit or harm minors in any way; including but not limited to the solicitation, creation, acquisition, or dissemination of child exploitative content.
|
||||
- To generate or disseminate verifiably false information and/or content with the purpose of harming others.
|
||||
- To generate or disseminate personal identifiable information that can be used to harm an individual.
|
||||
- To harass, abuse, threaten, stalk, or bully individuals or groups of individuals.
|
||||
- To create non-consensual nudity or illegal pornographic content.
|
||||
- For fully automated decision making that adversely impacts an individual's legal rights or otherwise creates or modifies a binding, enforceable obligation.
|
||||
- Generating or facilitating large-scale disinformation campaigns.
|
||||
|
||||
# License
|
||||
This model falls under the [`FLUX.1 [dev]` Non-Commercial License](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md).
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 MiB |
@@ -1,32 +0,0 @@
|
||||
{
|
||||
"_class_name": "FluxPipeline",
|
||||
"_diffusers_version": "0.30.0.dev0",
|
||||
"scheduler": [
|
||||
"diffusers",
|
||||
"FlowMatchEulerDiscreteScheduler"
|
||||
],
|
||||
"text_encoder": [
|
||||
"transformers",
|
||||
"CLIPTextModel"
|
||||
],
|
||||
"text_encoder_2": [
|
||||
"transformers",
|
||||
"T5EncoderModel"
|
||||
],
|
||||
"tokenizer": [
|
||||
"transformers",
|
||||
"CLIPTokenizer"
|
||||
],
|
||||
"tokenizer_2": [
|
||||
"transformers",
|
||||
"T5TokenizerFast"
|
||||
],
|
||||
"transformer": [
|
||||
"diffusers",
|
||||
"FluxTransformer2DModel"
|
||||
],
|
||||
"vae": [
|
||||
"diffusers",
|
||||
"AutoencoderKL"
|
||||
]
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"_class_name": "FlowMatchEulerDiscreteScheduler",
|
||||
"_diffusers_version": "0.30.0.dev0",
|
||||
"base_image_seq_len": 256,
|
||||
"base_shift": 0.5,
|
||||
"max_image_seq_len": 4096,
|
||||
"max_shift": 1.15,
|
||||
"num_train_timesteps": 1000,
|
||||
"shift": 3.0,
|
||||
"use_dynamic_shifting": true
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"_name_or_path": "openai/clip-vit-large-patch14",
|
||||
"architectures": [
|
||||
"CLIPTextModel"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 0,
|
||||
"dropout": 0.0,
|
||||
"eos_token_id": 2,
|
||||
"hidden_act": "quick_gelu",
|
||||
"hidden_size": 768,
|
||||
"initializer_factor": 1.0,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3072,
|
||||
"layer_norm_eps": 1e-05,
|
||||
"max_position_embeddings": 77,
|
||||
"model_type": "clip_text_model",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 12,
|
||||
"pad_token_id": 1,
|
||||
"projection_dim": 768,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.43.3",
|
||||
"vocab_size": 49408
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
{
|
||||
"_name_or_path": "google/t5-v1_1-xxl",
|
||||
"architectures": [
|
||||
"T5EncoderModel"
|
||||
],
|
||||
"classifier_dropout": 0.0,
|
||||
"d_ff": 10240,
|
||||
"d_kv": 64,
|
||||
"d_model": 4096,
|
||||
"decoder_start_token_id": 0,
|
||||
"dense_act_fn": "gelu_new",
|
||||
"dropout_rate": 0.1,
|
||||
"eos_token_id": 1,
|
||||
"feed_forward_proj": "gated-gelu",
|
||||
"initializer_factor": 1.0,
|
||||
"is_encoder_decoder": true,
|
||||
"is_gated_act": true,
|
||||
"layer_norm_epsilon": 1e-06,
|
||||
"model_type": "t5",
|
||||
"num_decoder_layers": 24,
|
||||
"num_heads": 64,
|
||||
"num_layers": 24,
|
||||
"output_past": true,
|
||||
"pad_token_id": 0,
|
||||
"relative_attention_max_distance": 128,
|
||||
"relative_attention_num_buckets": 32,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.43.3",
|
||||
"use_cache": true,
|
||||
"vocab_size": 32128
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 9524621312
|
||||
},
|
||||
"weight_map": {
|
||||
"encoder.block.0.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.0.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.1.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.10.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.11.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.12.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.12.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.12.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.12.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.12.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.12.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.12.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.12.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.12.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.13.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.14.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.15.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.16.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.17.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.18.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.19.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.2.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.2.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.20.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.20.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.21.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.22.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.0.SelfAttention.k.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.0.SelfAttention.o.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.0.SelfAttention.q.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.0.SelfAttention.v.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.0.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.1.DenseReluDense.wo.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.23.layer.1.layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"encoder.block.3.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.3.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.4.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.5.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.6.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.7.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.8.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.0.SelfAttention.k.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.0.SelfAttention.o.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.0.SelfAttention.q.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.0.SelfAttention.v.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.0.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.1.DenseReluDense.wo.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.block.9.layer.1.layer_norm.weight": "model-00001-of-00002.safetensors",
|
||||
"encoder.final_layer_norm.weight": "model-00002-of-00002.safetensors",
|
||||
"shared.weight": "model-00001-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<|startoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"49406": {
|
||||
"content": "<|startoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"49407": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"bos_token": "<|startoftext|>",
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"do_lower_case": true,
|
||||
"eos_token": "<|endoftext|>",
|
||||
"errors": "replace",
|
||||
"model_max_length": 77,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"tokenizer_class": "CLIPTokenizer",
|
||||
"unk_token": "<|endoftext|>"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,125 +0,0 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<extra_id_0>",
|
||||
"<extra_id_1>",
|
||||
"<extra_id_2>",
|
||||
"<extra_id_3>",
|
||||
"<extra_id_4>",
|
||||
"<extra_id_5>",
|
||||
"<extra_id_6>",
|
||||
"<extra_id_7>",
|
||||
"<extra_id_8>",
|
||||
"<extra_id_9>",
|
||||
"<extra_id_10>",
|
||||
"<extra_id_11>",
|
||||
"<extra_id_12>",
|
||||
"<extra_id_13>",
|
||||
"<extra_id_14>",
|
||||
"<extra_id_15>",
|
||||
"<extra_id_16>",
|
||||
"<extra_id_17>",
|
||||
"<extra_id_18>",
|
||||
"<extra_id_19>",
|
||||
"<extra_id_20>",
|
||||
"<extra_id_21>",
|
||||
"<extra_id_22>",
|
||||
"<extra_id_23>",
|
||||
"<extra_id_24>",
|
||||
"<extra_id_25>",
|
||||
"<extra_id_26>",
|
||||
"<extra_id_27>",
|
||||
"<extra_id_28>",
|
||||
"<extra_id_29>",
|
||||
"<extra_id_30>",
|
||||
"<extra_id_31>",
|
||||
"<extra_id_32>",
|
||||
"<extra_id_33>",
|
||||
"<extra_id_34>",
|
||||
"<extra_id_35>",
|
||||
"<extra_id_36>",
|
||||
"<extra_id_37>",
|
||||
"<extra_id_38>",
|
||||
"<extra_id_39>",
|
||||
"<extra_id_40>",
|
||||
"<extra_id_41>",
|
||||
"<extra_id_42>",
|
||||
"<extra_id_43>",
|
||||
"<extra_id_44>",
|
||||
"<extra_id_45>",
|
||||
"<extra_id_46>",
|
||||
"<extra_id_47>",
|
||||
"<extra_id_48>",
|
||||
"<extra_id_49>",
|
||||
"<extra_id_50>",
|
||||
"<extra_id_51>",
|
||||
"<extra_id_52>",
|
||||
"<extra_id_53>",
|
||||
"<extra_id_54>",
|
||||
"<extra_id_55>",
|
||||
"<extra_id_56>",
|
||||
"<extra_id_57>",
|
||||
"<extra_id_58>",
|
||||
"<extra_id_59>",
|
||||
"<extra_id_60>",
|
||||
"<extra_id_61>",
|
||||
"<extra_id_62>",
|
||||
"<extra_id_63>",
|
||||
"<extra_id_64>",
|
||||
"<extra_id_65>",
|
||||
"<extra_id_66>",
|
||||
"<extra_id_67>",
|
||||
"<extra_id_68>",
|
||||
"<extra_id_69>",
|
||||
"<extra_id_70>",
|
||||
"<extra_id_71>",
|
||||
"<extra_id_72>",
|
||||
"<extra_id_73>",
|
||||
"<extra_id_74>",
|
||||
"<extra_id_75>",
|
||||
"<extra_id_76>",
|
||||
"<extra_id_77>",
|
||||
"<extra_id_78>",
|
||||
"<extra_id_79>",
|
||||
"<extra_id_80>",
|
||||
"<extra_id_81>",
|
||||
"<extra_id_82>",
|
||||
"<extra_id_83>",
|
||||
"<extra_id_84>",
|
||||
"<extra_id_85>",
|
||||
"<extra_id_86>",
|
||||
"<extra_id_87>",
|
||||
"<extra_id_88>",
|
||||
"<extra_id_89>",
|
||||
"<extra_id_90>",
|
||||
"<extra_id_91>",
|
||||
"<extra_id_92>",
|
||||
"<extra_id_93>",
|
||||
"<extra_id_94>",
|
||||
"<extra_id_95>",
|
||||
"<extra_id_96>",
|
||||
"<extra_id_97>",
|
||||
"<extra_id_98>",
|
||||
"<extra_id_99>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
@@ -1,940 +0,0 @@
|
||||
{
|
||||
"add_prefix_space": true,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32000": {
|
||||
"content": "<extra_id_99>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32001": {
|
||||
"content": "<extra_id_98>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32002": {
|
||||
"content": "<extra_id_97>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32003": {
|
||||
"content": "<extra_id_96>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32004": {
|
||||
"content": "<extra_id_95>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32005": {
|
||||
"content": "<extra_id_94>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32006": {
|
||||
"content": "<extra_id_93>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32007": {
|
||||
"content": "<extra_id_92>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32008": {
|
||||
"content": "<extra_id_91>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32009": {
|
||||
"content": "<extra_id_90>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32010": {
|
||||
"content": "<extra_id_89>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32011": {
|
||||
"content": "<extra_id_88>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32012": {
|
||||
"content": "<extra_id_87>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32013": {
|
||||
"content": "<extra_id_86>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32014": {
|
||||
"content": "<extra_id_85>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32015": {
|
||||
"content": "<extra_id_84>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32016": {
|
||||
"content": "<extra_id_83>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32017": {
|
||||
"content": "<extra_id_82>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32018": {
|
||||
"content": "<extra_id_81>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32019": {
|
||||
"content": "<extra_id_80>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32020": {
|
||||
"content": "<extra_id_79>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32021": {
|
||||
"content": "<extra_id_78>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32022": {
|
||||
"content": "<extra_id_77>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32023": {
|
||||
"content": "<extra_id_76>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32024": {
|
||||
"content": "<extra_id_75>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32025": {
|
||||
"content": "<extra_id_74>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32026": {
|
||||
"content": "<extra_id_73>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32027": {
|
||||
"content": "<extra_id_72>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32028": {
|
||||
"content": "<extra_id_71>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32029": {
|
||||
"content": "<extra_id_70>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32030": {
|
||||
"content": "<extra_id_69>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32031": {
|
||||
"content": "<extra_id_68>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32032": {
|
||||
"content": "<extra_id_67>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32033": {
|
||||
"content": "<extra_id_66>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32034": {
|
||||
"content": "<extra_id_65>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32035": {
|
||||
"content": "<extra_id_64>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32036": {
|
||||
"content": "<extra_id_63>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32037": {
|
||||
"content": "<extra_id_62>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32038": {
|
||||
"content": "<extra_id_61>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32039": {
|
||||
"content": "<extra_id_60>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32040": {
|
||||
"content": "<extra_id_59>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32041": {
|
||||
"content": "<extra_id_58>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32042": {
|
||||
"content": "<extra_id_57>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32043": {
|
||||
"content": "<extra_id_56>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32044": {
|
||||
"content": "<extra_id_55>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32045": {
|
||||
"content": "<extra_id_54>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32046": {
|
||||
"content": "<extra_id_53>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32047": {
|
||||
"content": "<extra_id_52>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32048": {
|
||||
"content": "<extra_id_51>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32049": {
|
||||
"content": "<extra_id_50>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32050": {
|
||||
"content": "<extra_id_49>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32051": {
|
||||
"content": "<extra_id_48>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32052": {
|
||||
"content": "<extra_id_47>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32053": {
|
||||
"content": "<extra_id_46>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32054": {
|
||||
"content": "<extra_id_45>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32055": {
|
||||
"content": "<extra_id_44>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32056": {
|
||||
"content": "<extra_id_43>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32057": {
|
||||
"content": "<extra_id_42>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32058": {
|
||||
"content": "<extra_id_41>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32059": {
|
||||
"content": "<extra_id_40>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32060": {
|
||||
"content": "<extra_id_39>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32061": {
|
||||
"content": "<extra_id_38>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32062": {
|
||||
"content": "<extra_id_37>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32063": {
|
||||
"content": "<extra_id_36>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32064": {
|
||||
"content": "<extra_id_35>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32065": {
|
||||
"content": "<extra_id_34>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32066": {
|
||||
"content": "<extra_id_33>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32067": {
|
||||
"content": "<extra_id_32>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32068": {
|
||||
"content": "<extra_id_31>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32069": {
|
||||
"content": "<extra_id_30>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32070": {
|
||||
"content": "<extra_id_29>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32071": {
|
||||
"content": "<extra_id_28>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32072": {
|
||||
"content": "<extra_id_27>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32073": {
|
||||
"content": "<extra_id_26>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32074": {
|
||||
"content": "<extra_id_25>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32075": {
|
||||
"content": "<extra_id_24>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32076": {
|
||||
"content": "<extra_id_23>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32077": {
|
||||
"content": "<extra_id_22>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32078": {
|
||||
"content": "<extra_id_21>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32079": {
|
||||
"content": "<extra_id_20>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32080": {
|
||||
"content": "<extra_id_19>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32081": {
|
||||
"content": "<extra_id_18>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32082": {
|
||||
"content": "<extra_id_17>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32083": {
|
||||
"content": "<extra_id_16>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32084": {
|
||||
"content": "<extra_id_15>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32085": {
|
||||
"content": "<extra_id_14>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32086": {
|
||||
"content": "<extra_id_13>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32087": {
|
||||
"content": "<extra_id_12>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32088": {
|
||||
"content": "<extra_id_11>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32089": {
|
||||
"content": "<extra_id_10>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32090": {
|
||||
"content": "<extra_id_9>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32091": {
|
||||
"content": "<extra_id_8>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32092": {
|
||||
"content": "<extra_id_7>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32093": {
|
||||
"content": "<extra_id_6>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32094": {
|
||||
"content": "<extra_id_5>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32095": {
|
||||
"content": "<extra_id_4>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32096": {
|
||||
"content": "<extra_id_3>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32097": {
|
||||
"content": "<extra_id_2>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32098": {
|
||||
"content": "<extra_id_1>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32099": {
|
||||
"content": "<extra_id_0>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<extra_id_0>",
|
||||
"<extra_id_1>",
|
||||
"<extra_id_2>",
|
||||
"<extra_id_3>",
|
||||
"<extra_id_4>",
|
||||
"<extra_id_5>",
|
||||
"<extra_id_6>",
|
||||
"<extra_id_7>",
|
||||
"<extra_id_8>",
|
||||
"<extra_id_9>",
|
||||
"<extra_id_10>",
|
||||
"<extra_id_11>",
|
||||
"<extra_id_12>",
|
||||
"<extra_id_13>",
|
||||
"<extra_id_14>",
|
||||
"<extra_id_15>",
|
||||
"<extra_id_16>",
|
||||
"<extra_id_17>",
|
||||
"<extra_id_18>",
|
||||
"<extra_id_19>",
|
||||
"<extra_id_20>",
|
||||
"<extra_id_21>",
|
||||
"<extra_id_22>",
|
||||
"<extra_id_23>",
|
||||
"<extra_id_24>",
|
||||
"<extra_id_25>",
|
||||
"<extra_id_26>",
|
||||
"<extra_id_27>",
|
||||
"<extra_id_28>",
|
||||
"<extra_id_29>",
|
||||
"<extra_id_30>",
|
||||
"<extra_id_31>",
|
||||
"<extra_id_32>",
|
||||
"<extra_id_33>",
|
||||
"<extra_id_34>",
|
||||
"<extra_id_35>",
|
||||
"<extra_id_36>",
|
||||
"<extra_id_37>",
|
||||
"<extra_id_38>",
|
||||
"<extra_id_39>",
|
||||
"<extra_id_40>",
|
||||
"<extra_id_41>",
|
||||
"<extra_id_42>",
|
||||
"<extra_id_43>",
|
||||
"<extra_id_44>",
|
||||
"<extra_id_45>",
|
||||
"<extra_id_46>",
|
||||
"<extra_id_47>",
|
||||
"<extra_id_48>",
|
||||
"<extra_id_49>",
|
||||
"<extra_id_50>",
|
||||
"<extra_id_51>",
|
||||
"<extra_id_52>",
|
||||
"<extra_id_53>",
|
||||
"<extra_id_54>",
|
||||
"<extra_id_55>",
|
||||
"<extra_id_56>",
|
||||
"<extra_id_57>",
|
||||
"<extra_id_58>",
|
||||
"<extra_id_59>",
|
||||
"<extra_id_60>",
|
||||
"<extra_id_61>",
|
||||
"<extra_id_62>",
|
||||
"<extra_id_63>",
|
||||
"<extra_id_64>",
|
||||
"<extra_id_65>",
|
||||
"<extra_id_66>",
|
||||
"<extra_id_67>",
|
||||
"<extra_id_68>",
|
||||
"<extra_id_69>",
|
||||
"<extra_id_70>",
|
||||
"<extra_id_71>",
|
||||
"<extra_id_72>",
|
||||
"<extra_id_73>",
|
||||
"<extra_id_74>",
|
||||
"<extra_id_75>",
|
||||
"<extra_id_76>",
|
||||
"<extra_id_77>",
|
||||
"<extra_id_78>",
|
||||
"<extra_id_79>",
|
||||
"<extra_id_80>",
|
||||
"<extra_id_81>",
|
||||
"<extra_id_82>",
|
||||
"<extra_id_83>",
|
||||
"<extra_id_84>",
|
||||
"<extra_id_85>",
|
||||
"<extra_id_86>",
|
||||
"<extra_id_87>",
|
||||
"<extra_id_88>",
|
||||
"<extra_id_89>",
|
||||
"<extra_id_90>",
|
||||
"<extra_id_91>",
|
||||
"<extra_id_92>",
|
||||
"<extra_id_93>",
|
||||
"<extra_id_94>",
|
||||
"<extra_id_95>",
|
||||
"<extra_id_96>",
|
||||
"<extra_id_97>",
|
||||
"<extra_id_98>",
|
||||
"<extra_id_99>"
|
||||
],
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"eos_token": "</s>",
|
||||
"extra_ids": 100,
|
||||
"legacy": true,
|
||||
"model_max_length": 512,
|
||||
"pad_token": "<pad>",
|
||||
"sp_model_kwargs": {},
|
||||
"tokenizer_class": "T5Tokenizer",
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"_class_name": "FluxTransformer2DModel",
|
||||
"_diffusers_version": "0.30.0.dev0",
|
||||
"_name_or_path": "../checkpoints/flux-dev/transformer",
|
||||
"attention_head_dim": 128,
|
||||
"guidance_embeds": true,
|
||||
"in_channels": 64,
|
||||
"joint_attention_dim": 4096,
|
||||
"num_attention_heads": 24,
|
||||
"num_layers": 19,
|
||||
"num_single_layers": 38,
|
||||
"patch_size": 1,
|
||||
"pooled_projection_dim": 768
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,38 +0,0 @@
|
||||
{
|
||||
"_class_name": "AutoencoderKL",
|
||||
"_diffusers_version": "0.30.0.dev0",
|
||||
"_name_or_path": "../checkpoints/flux-dev",
|
||||
"act_fn": "silu",
|
||||
"block_out_channels": [
|
||||
128,
|
||||
256,
|
||||
512,
|
||||
512
|
||||
],
|
||||
"down_block_types": [
|
||||
"DownEncoderBlock2D",
|
||||
"DownEncoderBlock2D",
|
||||
"DownEncoderBlock2D",
|
||||
"DownEncoderBlock2D"
|
||||
],
|
||||
"force_upcast": true,
|
||||
"in_channels": 3,
|
||||
"latent_channels": 16,
|
||||
"latents_mean": null,
|
||||
"latents_std": null,
|
||||
"layers_per_block": 2,
|
||||
"mid_block_add_attention": true,
|
||||
"norm_num_groups": 32,
|
||||
"out_channels": 3,
|
||||
"sample_size": 1024,
|
||||
"scaling_factor": 0.3611,
|
||||
"shift_factor": 0.1159,
|
||||
"up_block_types": [
|
||||
"UpDecoderBlock2D",
|
||||
"UpDecoderBlock2D",
|
||||
"UpDecoderBlock2D",
|
||||
"UpDecoderBlock2D"
|
||||
],
|
||||
"use_post_quant_conv": false,
|
||||
"use_quant_conv": false
|
||||
}
|
||||
@@ -1,446 +0,0 @@
|
||||
from typing import Callable, Optional
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from diffusers.models.attention_processor import *
|
||||
from diffusers.models.transformers.transformer_flux import FluxTransformer2DModel
|
||||
|
||||
def default_set_attn_proc_func(
|
||||
name: str,
|
||||
hidden_size: int,
|
||||
cross_attention_dim: Optional[int],
|
||||
ori_attn_proc: object,
|
||||
) -> object:
|
||||
return ori_attn_proc
|
||||
|
||||
def set_flux_transformer_attn_processor(
|
||||
transformer: FluxTransformer2DModel,
|
||||
set_attn_proc_func: Callable = default_set_attn_proc_func,
|
||||
set_attn_module_names: Optional[list[str]] = None,
|
||||
) -> None:
|
||||
do_set_processor = lambda name, module_names: (
|
||||
any([name.startswith(module_name) for module_name in module_names])
|
||||
if module_names is not None
|
||||
else True
|
||||
) # prefix match
|
||||
|
||||
attn_procs = {}
|
||||
for name, attn_processor in transformer.attn_processors.items():
|
||||
dim_head = transformer.config.attention_head_dim
|
||||
num_heads = transformer.config.num_attention_heads
|
||||
if name.endswith("attn.processor"):
|
||||
attn_procs[name] = (
|
||||
set_attn_proc_func(name, dim_head, num_heads, attn_processor)
|
||||
if do_set_processor(name, set_attn_module_names)
|
||||
else attn_processor
|
||||
)
|
||||
|
||||
transformer.set_attn_processor(attn_procs)
|
||||
|
||||
class PersonalizeAnythingAttnProcessor:
|
||||
|
||||
def __init__(self, name, mask, device, tau=0.98, concept_process=False, shift_mask = None, img_dims=4096,token_len=512):
|
||||
if not hasattr(F, "scaled_dot_product_attention"):
|
||||
raise ImportError("FluxAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
|
||||
self.token_len=token_len
|
||||
self.name = name
|
||||
self.mask = mask.view(img_dims).bool().to(device)
|
||||
self.device = device
|
||||
self.tau = tau
|
||||
self.concept_process = concept_process
|
||||
self.img_dims = img_dims
|
||||
|
||||
if shift_mask is None:
|
||||
self.shift_mask = self.mask
|
||||
else:
|
||||
self.shift_mask = shift_mask.view(img_dims).bool().to(device)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn: Attention,
|
||||
hidden_states: torch.FloatTensor,
|
||||
encoder_hidden_states: torch.FloatTensor = None,
|
||||
attention_mask: Optional[torch.FloatTensor] = None,
|
||||
image_rotary_emb: Optional[torch.Tensor] = None,
|
||||
timestep = None,
|
||||
) -> torch.FloatTensor:
|
||||
batch_size, _, _ = hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
|
||||
|
||||
###################################################################################
|
||||
if timestep is not None:
|
||||
timestep = timestep
|
||||
else: #timestep is None
|
||||
timestep = 0.0
|
||||
concept_process = self.concept_process # token concatenation
|
||||
c_q = concept_process and True # if token concatenation is applied to q
|
||||
c_kv = concept_process and True # if token concatenation is applied to kv
|
||||
|
||||
t_flag = timestep > self.tau # token replacement
|
||||
r_q = True and t_flag # if token concatenation is applied to q
|
||||
r_k = True and t_flag # if token concatenation is applied to k
|
||||
r_v = True and t_flag # if token concatenation is applied to v
|
||||
|
||||
if encoder_hidden_states is not None:
|
||||
concept_feature_ = hidden_states[0, self.mask, :]
|
||||
else:
|
||||
concept_feature_ = hidden_states[0, self.token_len:, :][self.mask, :]
|
||||
|
||||
if r_k or r_q or r_v:
|
||||
r_hidden_states = hidden_states
|
||||
if encoder_hidden_states is not None:
|
||||
r_hidden_states[1, self.shift_mask, :] = concept_feature_
|
||||
else:
|
||||
text_hidden_states = hidden_states[1, :self.token_len, :]
|
||||
image_hidden_states = hidden_states[1, self.token_len:, :]
|
||||
image_hidden_states[self.shift_mask, :] = concept_feature_
|
||||
|
||||
r_hidden_states[1] = torch.cat([text_hidden_states, image_hidden_states], dim=0)
|
||||
###################################################################################
|
||||
|
||||
key = attn.to_k(hidden_states)
|
||||
value = attn.to_v(hidden_states)
|
||||
query = attn.to_q(hidden_states)
|
||||
|
||||
###################################################################################
|
||||
if r_k:
|
||||
key = attn.to_k(r_hidden_states)
|
||||
if r_q:
|
||||
query = attn.to_q(r_hidden_states)
|
||||
if r_v:
|
||||
value = attn.to_v(r_hidden_states)
|
||||
|
||||
if concept_process:
|
||||
if c_q:
|
||||
c_query = attn.to_q(concept_feature_)
|
||||
c_query = c_query.repeat(query.shape[0], 1, 1)
|
||||
query = torch.cat([query, c_query], dim=1)
|
||||
if c_kv:
|
||||
c_key = attn.to_k(concept_feature_)
|
||||
c_key = c_key.repeat(key.shape[0], 1, 1)
|
||||
c_value = attn.to_v(concept_feature_)
|
||||
c_value = c_value.repeat(value.shape[0], 1, 1)
|
||||
key = torch.cat([key, c_key], dim=1)
|
||||
value = torch.cat([value, c_value], dim=1)
|
||||
###################################################################################
|
||||
|
||||
inner_dim = key.shape[-1]
|
||||
head_dim = inner_dim // attn.heads
|
||||
|
||||
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
|
||||
if attn.norm_q is not None:
|
||||
query = attn.norm_q(query)
|
||||
if attn.norm_k is not None:
|
||||
key = attn.norm_k(key)
|
||||
|
||||
# the attention in FluxSingleTransformerBlock does not use `encoder_hidden_states`
|
||||
if encoder_hidden_states is not None:
|
||||
# `context` projections.
|
||||
encoder_hidden_states_query_proj = attn.add_q_proj(encoder_hidden_states)
|
||||
encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states)
|
||||
encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states)
|
||||
|
||||
encoder_hidden_states_query_proj = encoder_hidden_states_query_proj.view(
|
||||
batch_size, -1, attn.heads, head_dim
|
||||
).transpose(1, 2)
|
||||
encoder_hidden_states_key_proj = encoder_hidden_states_key_proj.view(
|
||||
batch_size, -1, attn.heads, head_dim
|
||||
).transpose(1, 2)
|
||||
encoder_hidden_states_value_proj = encoder_hidden_states_value_proj.view(
|
||||
batch_size, -1, attn.heads, head_dim
|
||||
).transpose(1, 2)
|
||||
|
||||
if attn.norm_added_q is not None:
|
||||
encoder_hidden_states_query_proj = attn.norm_added_q(encoder_hidden_states_query_proj)
|
||||
if attn.norm_added_k is not None:
|
||||
encoder_hidden_states_key_proj = attn.norm_added_k(encoder_hidden_states_key_proj)
|
||||
|
||||
# attention
|
||||
#print(encoder_hidden_states_query_proj.shape,query.shape)
|
||||
query = torch.cat([encoder_hidden_states_query_proj, query], dim=2)
|
||||
key = torch.cat([encoder_hidden_states_key_proj, key], dim=2)
|
||||
value = torch.cat([encoder_hidden_states_value_proj, value], dim=2)
|
||||
|
||||
|
||||
if image_rotary_emb is not None:
|
||||
from diffusers.models.embeddings import apply_rotary_emb
|
||||
|
||||
# use original position emb or text emb
|
||||
if not c_q:
|
||||
#print(query.shape,image_rotary_emb[0].shape,image_rotary_emb[1].shape) # comfy torch.Size([2, 24, 1280, 128]) torch.Size([1280, 128]) torch.Size([1280, 128]) diffuser #torch.Size([2, 24, 1536, 128]) torch.Size([1536, 128]) torch.Size([1536, 128])
|
||||
query = apply_rotary_emb(query, image_rotary_emb)
|
||||
if not c_kv:
|
||||
key = apply_rotary_emb(key, image_rotary_emb)
|
||||
|
||||
###################################################################################
|
||||
# 获取原位置的 embedding
|
||||
def get_concept_rotary_emb(ori_rotary_emb, mask,):
|
||||
enc_emb = ori_rotary_emb[:self.token_len, :]
|
||||
hid_emb = ori_rotary_emb[self.token_len:, :]
|
||||
concept_emb = hid_emb[mask, :]
|
||||
|
||||
image_rotary_emb = torch.cat([enc_emb, hid_emb, concept_emb], dim=0)
|
||||
return image_rotary_emb
|
||||
|
||||
if concept_process:
|
||||
# 1. use original position emb
|
||||
image_rotary_emb_0 = get_concept_rotary_emb(image_rotary_emb[0], self.shift_mask,)
|
||||
image_rotary_emb_1 = get_concept_rotary_emb(image_rotary_emb[1], self.shift_mask,)
|
||||
image_rotary_emb = (image_rotary_emb_0, image_rotary_emb_1)
|
||||
|
||||
# 2. use text emb
|
||||
# dims = (self.mask == 1).sum().item()
|
||||
# concept_rotary_emb_0 = torch.ones((dims, 128)).to(self.device)
|
||||
# concept_rotary_emb_1 = torch.zeros((dims, 128)).to(self.device)
|
||||
# image_rotary_emb = (
|
||||
# torch.cat([image_rotary_emb[0], concept_rotary_emb_0], dim=0),
|
||||
# torch.cat([image_rotary_emb[1], concept_rotary_emb_1], dim=0))
|
||||
|
||||
if c_q:
|
||||
query = apply_rotary_emb(query, image_rotary_emb)
|
||||
if c_kv:
|
||||
key = apply_rotary_emb(key, image_rotary_emb)
|
||||
###################################################################################
|
||||
|
||||
hidden_states = F.scaled_dot_product_attention(query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False)
|
||||
|
||||
hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
|
||||
hidden_states = hidden_states.to(query.dtype)
|
||||
|
||||
if encoder_hidden_states is not None:
|
||||
encoder_hidden_states, hidden_states = (
|
||||
hidden_states[:, : encoder_hidden_states.shape[1]],
|
||||
hidden_states[:, encoder_hidden_states.shape[1] :],
|
||||
)
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
|
||||
|
||||
################################################################
|
||||
# restore after token concatenation
|
||||
hidden_states = hidden_states[:, :self.img_dims, :]
|
||||
################################################################
|
||||
|
||||
return hidden_states, encoder_hidden_states
|
||||
else:
|
||||
################################################################
|
||||
dims = self.img_dims + self.token_len
|
||||
hidden_states = hidden_states[:, :dims, :]
|
||||
################################################################
|
||||
|
||||
return hidden_states
|
||||
|
||||
class MultiPersonalizeAnythingAttnProcessor:
|
||||
|
||||
def __init__(self, name, masks, device, tau=0.98, concept_process=False, shift_masks = None, img_dims=4096):
|
||||
if not hasattr(F, "scaled_dot_product_attention"):
|
||||
raise ImportError("FluxAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
|
||||
|
||||
self.name = name
|
||||
self.device = device
|
||||
self.tau = tau
|
||||
self.concept_process = concept_process
|
||||
self.img_dims = img_dims
|
||||
|
||||
for i in range(len(masks)):
|
||||
masks[i] = masks[i].view(img_dims).bool().to(device)
|
||||
self.masks = masks
|
||||
|
||||
if shift_masks is None:
|
||||
self.shift_masks = self.masks
|
||||
else:
|
||||
for i in range(len(shift_masks)):
|
||||
shift_masks[i] = shift_masks[i].view(img_dims).bool().to(device)
|
||||
self.shift_masks = shift_masks
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
attn: Attention,
|
||||
hidden_states: torch.FloatTensor,
|
||||
encoder_hidden_states: torch.FloatTensor = None,
|
||||
attention_mask: Optional[torch.FloatTensor] = None,
|
||||
image_rotary_emb: Optional[torch.Tensor] = None,
|
||||
timestep = None,
|
||||
) -> torch.FloatTensor:
|
||||
batch_size, _, _ = hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape
|
||||
|
||||
###################################################################################
|
||||
if timestep is not None:
|
||||
timestep = timestep
|
||||
|
||||
concept_process = self.concept_process # token concatenation
|
||||
c_q = concept_process and True # if token concatenation is applied to q
|
||||
c_kv = concept_process and True # if token concatenation is applied to kv
|
||||
|
||||
t_flag = timestep > self.tau # token replacement
|
||||
r_q = True and t_flag # if token concatenation is applied to q
|
||||
r_k = True and t_flag # if token concatenation is applied to k
|
||||
r_v = True and t_flag # if token concatenation is applied to v
|
||||
|
||||
concept_features = []
|
||||
r_hidden_states = hidden_states
|
||||
for id, mask in enumerate(self.masks):
|
||||
if encoder_hidden_states is not None:
|
||||
concept_feature_ = hidden_states[id, mask, :]
|
||||
else:
|
||||
concept_feature_ = hidden_states[id, 512:, :][mask, :]
|
||||
|
||||
shift_mask = self.shift_masks[id]
|
||||
concept_features.append(concept_feature_)
|
||||
|
||||
if r_k or r_q or r_v:
|
||||
if encoder_hidden_states is not None:
|
||||
r_hidden_states[-1, shift_mask, :] = concept_feature_
|
||||
else:
|
||||
text_hidden_states = r_hidden_states[-1, :512, :]
|
||||
image_hidden_states = r_hidden_states[-1, 512:, :]
|
||||
image_hidden_states[shift_mask, :] = concept_feature_
|
||||
r_hidden_states[-1] = torch.cat([text_hidden_states, image_hidden_states], dim=0)
|
||||
###################################################################################
|
||||
|
||||
key = attn.to_k(hidden_states)
|
||||
value = attn.to_v(hidden_states)
|
||||
query = attn.to_q(hidden_states)
|
||||
|
||||
###################################################################################
|
||||
if r_k:
|
||||
key = attn.to_k(r_hidden_states)
|
||||
if r_q:
|
||||
query = attn.to_q(r_hidden_states)
|
||||
if r_v:
|
||||
value = attn.to_v(r_hidden_states)
|
||||
|
||||
if concept_process:
|
||||
for concept_feature_ in concept_features:
|
||||
if c_q:
|
||||
c_query = attn.to_q(concept_feature_)
|
||||
c_query = c_query.repeat(query.shape[0], 1, 1)
|
||||
query = torch.cat([query, c_query], dim=1)
|
||||
if c_kv:
|
||||
c_key = attn.to_k(concept_feature_)
|
||||
c_key = c_key.repeat(key.shape[0], 1, 1)
|
||||
|
||||
c_value = attn.to_v(concept_feature_)
|
||||
c_value = c_value.repeat(value.shape[0], 1, 1)
|
||||
|
||||
key = torch.cat([key, c_key], dim=1)
|
||||
value = torch.cat([value, c_value], dim=1)
|
||||
###################################################################################
|
||||
|
||||
inner_dim = key.shape[-1]
|
||||
head_dim = inner_dim // attn.heads
|
||||
|
||||
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
||||
|
||||
if attn.norm_q is not None:
|
||||
query = attn.norm_q(query)
|
||||
if attn.norm_k is not None:
|
||||
key = attn.norm_k(key)
|
||||
|
||||
# the attention in FluxSingleTransformerBlock does not use `encoder_hidden_states`
|
||||
if encoder_hidden_states is not None:
|
||||
# `context` projections.
|
||||
encoder_hidden_states_query_proj = attn.add_q_proj(encoder_hidden_states)
|
||||
encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states)
|
||||
encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states)
|
||||
|
||||
encoder_hidden_states_query_proj = encoder_hidden_states_query_proj.view(
|
||||
batch_size, -1, attn.heads, head_dim
|
||||
).transpose(1, 2)
|
||||
encoder_hidden_states_key_proj = encoder_hidden_states_key_proj.view(
|
||||
batch_size, -1, attn.heads, head_dim
|
||||
).transpose(1, 2)
|
||||
encoder_hidden_states_value_proj = encoder_hidden_states_value_proj.view(
|
||||
batch_size, -1, attn.heads, head_dim
|
||||
).transpose(1, 2)
|
||||
|
||||
if attn.norm_added_q is not None:
|
||||
encoder_hidden_states_query_proj = attn.norm_added_q(encoder_hidden_states_query_proj)
|
||||
if attn.norm_added_k is not None:
|
||||
encoder_hidden_states_key_proj = attn.norm_added_k(encoder_hidden_states_key_proj)
|
||||
|
||||
# attention
|
||||
query = torch.cat([encoder_hidden_states_query_proj, query], dim=2)
|
||||
key = torch.cat([encoder_hidden_states_key_proj, key], dim=2)
|
||||
value = torch.cat([encoder_hidden_states_value_proj, value], dim=2)
|
||||
|
||||
if image_rotary_emb is not None:
|
||||
from diffusers.models.embeddings import apply_rotary_emb
|
||||
|
||||
# use original position emb or text emb
|
||||
if not c_q:
|
||||
query = apply_rotary_emb(query, image_rotary_emb)
|
||||
if not c_kv:
|
||||
key = apply_rotary_emb(key, image_rotary_emb)
|
||||
|
||||
###################################################################################
|
||||
def get_concept_rotary_emb(ori_rotary_emb, shift_masks):
|
||||
enc_emb = ori_rotary_emb[:512, :]
|
||||
hid_emb = ori_rotary_emb[512:, :]
|
||||
|
||||
concept_embs = []
|
||||
for mask in shift_masks:
|
||||
concept_embs.append(hid_emb[mask, :])
|
||||
concept_emb = torch.cat(concept_embs, dim=0) if len(concept_embs) > 0 else torch.zeros(0, hid_emb.shape[1], device=hid_emb.device)
|
||||
image_rotary_emb = torch.cat([enc_emb, hid_emb, concept_emb], dim=0)
|
||||
return image_rotary_emb
|
||||
|
||||
if concept_process:
|
||||
# 选项 1: 使用原始位置嵌入 + 多个 shift_masks
|
||||
image_rotary_emb_0 = get_concept_rotary_emb(image_rotary_emb[0], self.shift_masks)
|
||||
image_rotary_emb_1 = get_concept_rotary_emb(image_rotary_emb[1], self.shift_masks)
|
||||
image_rotary_emb = (image_rotary_emb_0, image_rotary_emb_1)
|
||||
|
||||
# 选项 2: 使用文本嵌入 + 多个 masks
|
||||
# total_dims = sum((mask == 1).sum().item() for mask in self.masks)
|
||||
# concept_rotary_emb_0 = torch.ones((total_dims, 128)).to(self.device)
|
||||
# concept_rotary_emb_1 = torch.zeros((total_dims, 128)).to(self.device)
|
||||
# image_rotary_emb = (
|
||||
# torch.cat([image_rotary_emb[0], concept_rotary_emb_0], dim=0),
|
||||
# torch.cat([image_rotary_emb[1], concept_rotary_emb_1], dim=0)
|
||||
# )
|
||||
|
||||
if c_q:
|
||||
query = apply_rotary_emb(query, image_rotary_emb)
|
||||
if c_kv:
|
||||
key = apply_rotary_emb(key, image_rotary_emb)
|
||||
###################################################################################
|
||||
|
||||
hidden_states = F.scaled_dot_product_attention(query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False)
|
||||
|
||||
hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
|
||||
hidden_states = hidden_states.to(query.dtype)
|
||||
|
||||
if encoder_hidden_states is not None:
|
||||
encoder_hidden_states, hidden_states = (
|
||||
hidden_states[:, : encoder_hidden_states.shape[1]],
|
||||
hidden_states[:, encoder_hidden_states.shape[1] :],
|
||||
)
|
||||
|
||||
# linear proj
|
||||
hidden_states = attn.to_out[0](hidden_states)
|
||||
# dropout
|
||||
hidden_states = attn.to_out[1](hidden_states)
|
||||
|
||||
encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
|
||||
|
||||
################################################################
|
||||
# restore after token concatenation
|
||||
hidden_states = hidden_states[:, :self.img_dims, :]
|
||||
################################################################
|
||||
|
||||
return hidden_states, encoder_hidden_states
|
||||
else:
|
||||
################################################################
|
||||
dims = self.img_dims + 512
|
||||
hidden_states = hidden_states[:, :dims, :]
|
||||
################################################################
|
||||
|
||||
return hidden_states
|
||||
-1227
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,95 +0,0 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import cv2
|
||||
from typing import Optional
|
||||
|
||||
def shift_tensor(tensor, x):
|
||||
shifted_tensor = torch.zeros_like(tensor)
|
||||
|
||||
if x > 0:
|
||||
shifted_tensor[:, x:] = tensor[:, :-x]
|
||||
elif x < 0:
|
||||
shifted_tensor[:, :x] = tensor[:, -x:]
|
||||
else:
|
||||
shifted_tensor = tensor # No shift for x == 0
|
||||
|
||||
return shifted_tensor
|
||||
|
||||
def create_mask(img_pil, w=64, h=64):
|
||||
|
||||
img = img_pil.resize((w, h), Image.Resampling.NEAREST).convert("L")
|
||||
img_array = np.array(img)
|
||||
mask = np.where(img_array == 255, 1, 0)
|
||||
mask_tensor = torch.tensor(mask).int()
|
||||
|
||||
return mask_tensor
|
||||
|
||||
def save_array_as_png(array, path):
|
||||
if array.dtype != np.uint8:
|
||||
array = (array * 255).clip(0, 255).astype(np.uint8)
|
||||
image = Image.fromarray(array, "RGBA")
|
||||
image.save(path)
|
||||
|
||||
def convert_to_mask_inpainting(image_array, mask_path):
|
||||
if image_array.shape[2] != 4:
|
||||
raise ValueError("输入数组必须是 RGBA 格式")
|
||||
mask = np.ones(image_array.shape[:2], dtype=np.uint8) * 255
|
||||
alpha_channel = image_array[:, :, 3]
|
||||
mask[alpha_channel != 0] = 0
|
||||
mask_image = Image.fromarray(mask, mode="L")
|
||||
mask_image.save(mask_path)
|
||||
|
||||
return mask_image
|
||||
|
||||
# mask for Subject Customiztion
|
||||
def composite_images(background_path: str, mask_path: str) -> Image.Image:
|
||||
background = Image.open(background_path).convert("RGBA")
|
||||
mask = Image.open(mask_path).convert("L")
|
||||
|
||||
if background.size != mask.size:
|
||||
mask = mask.resize(background.size)
|
||||
|
||||
mask_array = np.array(mask) > 128
|
||||
|
||||
if background.mode == "RGBA":
|
||||
white_canvas = Image.new("RGBA", background.size, (255, 255, 255, 255))
|
||||
else:
|
||||
white_canvas = Image.new("RGB", background.size, (255, 255, 255))
|
||||
|
||||
composite = Image.composite(background, white_canvas, Image.fromarray(mask_array))
|
||||
|
||||
return composite.convert("RGB")
|
||||
|
||||
def process_mask_array(mask_array: np.ndarray) -> Image.Image:
|
||||
alpha = mask_array[..., 3]
|
||||
gray_array = np.where(alpha > 0, 0, 255).astype(np.uint8)
|
||||
mask_image = Image.fromarray(gray_array, mode="L")
|
||||
return mask_image.convert("1")
|
||||
|
||||
def process_mask(mask: Image.Image) -> Image.Image:
|
||||
if mask.mode != "L":
|
||||
mask = mask.convert("L")
|
||||
return mask.point(lambda x: 1 if x > 128 else 0, mode="1")
|
||||
|
||||
def merge_masks(mask1: Image.Image, mask2: Image.Image) -> Image.Image:
|
||||
arr1 = np.array(mask1, dtype=bool)
|
||||
arr2 = np.array(mask2, dtype=bool)
|
||||
merged = np.logical_and(arr1, arr2)
|
||||
return Image.fromarray(merged).convert("1")
|
||||
|
||||
def save_merged_mask(
|
||||
mask_array: np.ndarray, mask: Optional[Image.Image], output_path: str
|
||||
) -> None:
|
||||
mask1 = process_mask_array(mask_array)
|
||||
|
||||
if mask is not None:
|
||||
mask2 = process_mask(mask)
|
||||
if mask1.size != mask2.size:
|
||||
mask2 = mask2.resize(mask1.size, Image.NEAREST)
|
||||
merged = merge_masks(mask1, mask2)
|
||||
else:
|
||||
merged = mask1
|
||||
|
||||
merged.save(output_path)
|
||||
|
||||
Reference in New Issue
Block a user