3063 lines
142 KiB
Python
3063 lines
142 KiB
Python
# !/usr/bin/env python
|
|
# -*- coding: UTF-8 -*-
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import random
|
|
import torch
|
|
import gc
|
|
from omegaconf import OmegaConf
|
|
from PIL import Image
|
|
import numpy as np
|
|
import cv2
|
|
from diffusers import ( DDIMScheduler,
|
|
KDPM2AncestralDiscreteScheduler, LMSDiscreteScheduler,
|
|
DPMSolverMultistepScheduler, DPMSolverSinglestepScheduler,
|
|
EulerDiscreteScheduler, HeunDiscreteScheduler,
|
|
KDPM2DiscreteScheduler,
|
|
EulerAncestralDiscreteScheduler, UniPCMultistepScheduler,
|
|
DDPMScheduler, LCMScheduler)
|
|
|
|
from .msdiffusion.models.projection import Resampler
|
|
from .msdiffusion.models.modelWrapper import MSAdapter as MSAdapterWarpper
|
|
from .utils.style_template import styles
|
|
from .utils.load_models_utils import get_lora_dict
|
|
from .PuLID.pulid.utils import resize_numpy_image_long
|
|
from transformers import AutoModel, AutoTokenizer
|
|
from comfy.utils import common_upscale,ProgressBar
|
|
import folder_paths
|
|
from comfy.clip_vision import load as clip_load
|
|
from .utils.gradio_utils import process_original_text,character_to_dict
|
|
import json
|
|
|
|
cur_path = os.path.dirname(os.path.abspath(__file__))
|
|
photomaker_dir=os.path.join(folder_paths.models_dir, "photomaker")
|
|
base_pt = os.path.join(photomaker_dir,"pt")
|
|
device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
|
|
|
|
lora_get = get_lora_dict()
|
|
lora_lightning_list = lora_get["lightning_xl_lora"]
|
|
|
|
|
|
SAMPLER_NAMES = ["euler", "euler_cfg_pp", "euler_ancestral", "euler_ancestral_cfg_pp", "heun", "heunpp2","dpm_2", "dpm_2_ancestral",
|
|
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_2s_ancestral_cfg_pp", "dpmpp_sde", "dpmpp_sde_gpu",
|
|
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm",
|
|
"ipndm", "ipndm_v", "deis","ddim", "uni_pc", "uni_pc_bh2"]
|
|
|
|
SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "beta"]
|
|
|
|
|
|
def gc_cleanup():
|
|
gc.collect()
|
|
torch.cuda.empty_cache()
|
|
|
|
|
|
def get_scheduler(name,scheduler_):
|
|
scheduler = False
|
|
if name == "euler" or name =="euler_cfg_pp":
|
|
scheduler = EulerDiscreteScheduler()
|
|
elif name == "euler_ancestral" or name =="euler_ancestral_cfg_pp":
|
|
scheduler = EulerAncestralDiscreteScheduler()
|
|
elif name == "ddim":
|
|
scheduler = DDIMScheduler()
|
|
elif name == "ddpm":
|
|
scheduler = DDPMScheduler()
|
|
elif name == "dpmpp_2m":
|
|
scheduler = DPMSolverMultistepScheduler()
|
|
elif name == "dpmpp_2m" and scheduler_=="karras":
|
|
scheduler = DPMSolverMultistepScheduler(use_karras_sigmas=True)
|
|
elif name == "dpmpp_2m_sde":
|
|
scheduler = DPMSolverMultistepScheduler(algorithm_type="sde-dpmsolver++")
|
|
elif name == "dpmpp_2m" and scheduler_=="karras":
|
|
scheduler = DPMSolverMultistepScheduler(use_karras_sigmas=True, algorithm_type="sde-dpmsolver++")
|
|
elif name == "dpmpp_sde" or name == "dpmpp_sde_gpu":
|
|
scheduler = DPMSolverSinglestepScheduler()
|
|
elif (name == "dpmpp_sde" or name == "dpmpp_sde_gpu") and scheduler_=="karras":
|
|
scheduler = DPMSolverSinglestepScheduler(use_karras_sigmas=True)
|
|
elif name == "dpm_2":
|
|
scheduler = KDPM2DiscreteScheduler()
|
|
elif name == "dpm_2" and scheduler_=="karras":
|
|
scheduler = KDPM2DiscreteScheduler(use_karras_sigmas=True)
|
|
elif name == "dpm_2_ancestral":
|
|
scheduler = KDPM2AncestralDiscreteScheduler()
|
|
elif name == "dpm_2_ancestral" and scheduler_=="karras":
|
|
scheduler = KDPM2AncestralDiscreteScheduler(use_karras_sigmas=True)
|
|
elif name == "heun":
|
|
scheduler = HeunDiscreteScheduler()
|
|
elif name == "lcm":
|
|
scheduler = LCMScheduler()
|
|
elif name == "lms":
|
|
scheduler = LMSDiscreteScheduler()
|
|
elif name == "lms" and scheduler_=="karras":
|
|
scheduler = LMSDiscreteScheduler(use_karras_sigmas=True)
|
|
elif name == "uni_pc":
|
|
scheduler = UniPCMultistepScheduler()
|
|
else:
|
|
scheduler = EulerDiscreteScheduler()
|
|
return scheduler
|
|
|
|
|
|
|
|
def get_extra_function(extra_function,extra_param,photomake_ckpt_path,image,infer_mode):
|
|
auraface=False
|
|
use_photov2=False
|
|
cached=False
|
|
inject=False
|
|
onnx_provider="gpu"
|
|
img2img_mode = True if isinstance(image, torch.Tensor) else False
|
|
trigger_words_dual="best"
|
|
dual_lora_scale=1.0
|
|
dreamo_mode="ip"
|
|
|
|
if extra_function:
|
|
extra_function = extra_function.strip().lower()
|
|
if "auraface" in extra_function:
|
|
auraface=True
|
|
|
|
|
|
if extra_param:
|
|
extra_param = extra_param.strip().lower()
|
|
if "cache" in extra_param:
|
|
cached=True
|
|
if "inject" in extra_param:
|
|
inject=True
|
|
if "cpu" in extra_param:
|
|
onnx_provider="cpu"
|
|
if "id" in extra_param:
|
|
dreamo_mode="id"
|
|
elif "style" in extra_param:
|
|
dreamo_mode="style"
|
|
if "[" in extra_param:
|
|
trigger_words_param=extra_param.split("[")[1].split("]")[0]
|
|
trigger_words_dual=trigger_words_param.split(",")[0]
|
|
dual_lora_scale=float(trigger_words_param.split(",")[1])
|
|
|
|
|
|
if isinstance(photomake_ckpt_path, str) and img2img_mode:
|
|
use_photov2 = True if "v2" in photomake_ckpt_path else False
|
|
|
|
return auraface,use_photov2,img2img_mode,cached,inject,onnx_provider,dreamo_mode,trigger_words_dual,dual_lora_scale
|
|
|
|
def extract_content_from_brackets(text):
|
|
# 正则表达式匹配多对方括号内的内容
|
|
return re.findall(r'\[(.*?)\]', text)
|
|
|
|
def phi2narry(img):
|
|
img = torch.from_numpy(np.array(img).astype(np.float32) / 255.0).unsqueeze(0)
|
|
return img
|
|
|
|
def tensor_to_image(tensor):
|
|
image_np = tensor.squeeze().mul(255).clamp(0, 255).byte().numpy()
|
|
image = Image.fromarray(image_np, mode='RGB')
|
|
return image
|
|
|
|
def tensortopil_list(tensor_in):
|
|
d1, _, _, _ = tensor_in.size()
|
|
if d1 == 1:
|
|
img_list = [tensor_to_image(tensor_in)]
|
|
else:
|
|
tensor_list = torch.chunk(tensor_in, chunks=d1)
|
|
img_list=[tensor_to_image(i) for i in tensor_list]
|
|
return img_list
|
|
|
|
def tensortopil_list_upscale(tensor_in,width,height):
|
|
d1, _, _, _ = tensor_in.size()
|
|
if d1 == 1:
|
|
img_list = [nomarl_upscale(tensor_in,width,height)]
|
|
else:
|
|
tensor_list = torch.chunk(tensor_in, chunks=d1)
|
|
img_list=[nomarl_upscale(i,width,height) for i in tensor_list]
|
|
return img_list
|
|
|
|
def tensortolist(tensor_in,width,height):
|
|
d1, _, _, _ = tensor_in.size()
|
|
if d1 == 1:
|
|
tensor_list = [nomarl_tensor_upscale(tensor_in,width,height)]
|
|
else:
|
|
tensor_list_ = torch.chunk(tensor_in, chunks=d1)
|
|
tensor_list=[nomarl_tensor_upscale(i,width,height) for i in tensor_list_]
|
|
return tensor_list
|
|
|
|
|
|
|
|
def nomarl_tensor_upscale(tensor, width, height):
|
|
samples = tensor.movedim(-1, 1)
|
|
samples = common_upscale(samples, width, height, "nearest-exact", "center")
|
|
samples = samples.movedim(1, -1)
|
|
return samples
|
|
def nomarl_upscale(img, width, height):
|
|
samples = img.movedim(-1, 1)
|
|
img = common_upscale(samples, width, height, "nearest-exact", "center")
|
|
samples = img.movedim(1, -1)
|
|
img = tensor_to_image(samples)
|
|
return img
|
|
def nomarl_upscale_tensor(img, width, height):
|
|
samples = img.movedim(-1, 1)
|
|
img = common_upscale(samples, width, height, "nearest-exact", "center")
|
|
samples = img.movedim(1, -1)
|
|
return samples
|
|
|
|
def center_crop(img):
|
|
width, height = img.size
|
|
square = min(width, height)
|
|
left = (width - square) / 2
|
|
top = (height - square) / 2
|
|
right = (width + square) / 2
|
|
bottom = (height + square) / 2
|
|
return img.crop((left, top, right, bottom))
|
|
|
|
def center_crop_s(img, new_width, new_height):
|
|
width, height = img.size
|
|
left = (width - new_width) / 2
|
|
top = (height - new_height) / 2
|
|
right = (width + new_width) / 2
|
|
bottom = (height + new_height) / 2
|
|
return img.crop((left, top, right, bottom))
|
|
|
|
|
|
def contains_brackets(s):
|
|
return '[' in s or ']' in s
|
|
|
|
def has_parentheses(s):
|
|
return bool(re.search(r'\(.*?\)', s))
|
|
def extract_content_from_brackets_(text):
|
|
# 正则表达式匹配多对圆括号内的内容
|
|
return re.findall(r'\((.*?)\)', text)
|
|
|
|
def narry_list(list_in):
|
|
for i in range(len(list_in)):
|
|
value = list_in[i]
|
|
modified_value = phi2narry(value)
|
|
list_in[i] = modified_value
|
|
return list_in
|
|
def remove_punctuation_from_strings(lst):
|
|
pattern = r"[\W]+$" # 匹配字符串末尾的所有非单词字符
|
|
return [re.sub(pattern, '', s) for s in lst]
|
|
|
|
def phi_list(list_in):
|
|
for i in range(len(list_in)):
|
|
value = list_in[i]
|
|
list_in[i] = value
|
|
return list_in
|
|
|
|
def narry_list_pil(list_in):
|
|
for i in range(len(list_in)):
|
|
value = list_in[i]
|
|
modified_value = tensor_to_image(value)
|
|
list_in[i] = modified_value
|
|
return list_in
|
|
|
|
|
|
def setup_seed(seed):
|
|
torch.manual_seed(seed)
|
|
if device == "cuda":
|
|
torch.cuda.manual_seed_all(seed)
|
|
np.random.seed(seed)
|
|
random.seed(seed)
|
|
torch.backends.cudnn.deterministic = True
|
|
|
|
def apply_style_positive(style_name: str, positive: str):
|
|
p, n = styles.get(style_name, styles[style_name])
|
|
#print(p, "test0", n)
|
|
return p.replace("{prompt}", positive),n
|
|
|
|
def apply_style(style_name: str, positives: list, negative: str = ""):
|
|
p, n = styles.get(style_name, styles[style_name])
|
|
#print(p,"test1",n)
|
|
return [
|
|
p.replace("{prompt}", positive) for positive in positives
|
|
], n + " " + negative
|
|
|
|
|
|
|
|
|
|
|
|
def face_bbox_to_square(bbox):
|
|
## l, t, r, b to square l, t, r, b
|
|
l,t,r,b = bbox
|
|
cent_x = (l + r) / 2
|
|
cent_y = (t + b) / 2
|
|
w, h = r - l, b - t
|
|
r = max(w, h) / 2
|
|
|
|
l0 = cent_x - r
|
|
r0 = cent_x + r
|
|
t0 = cent_y - r
|
|
b0 = cent_y + r
|
|
|
|
return [l0, t0, r0, b0]
|
|
|
|
|
|
|
|
def insight_face_loader(infer_mode,use_photov2,auraface,onnx_provider="cpu",mask_repo="briaai/RMBG-1.4"):
|
|
insightface_root_path= folder_paths.base_path
|
|
if infer_mode=="story" and use_photov2:
|
|
from .utils.insightface_package import FaceAnalysis2
|
|
if auraface:
|
|
from huggingface_hub import snapshot_download
|
|
snapshot_download(
|
|
"fal/AuraFace-v1",
|
|
local_dir="models/auraface",
|
|
)
|
|
app_face = FaceAnalysis2(name="auraface",
|
|
providers=["CUDAExecutionProvider", "CPUExecutionProvider"], root=insightface_root_path,
|
|
allowed_modules=['detection', 'recognition'])
|
|
else:
|
|
app_face = FaceAnalysis2(providers=['CUDAExecutionProvider'],
|
|
allowed_modules=['detection', 'recognition'])
|
|
app_face.prepare(ctx_id=0, det_size=(640, 640))
|
|
pipeline_mask = None
|
|
app_face_ = None
|
|
elif infer_mode=="kolor_face" :
|
|
from .kolors.models.sample_ipadapter_faceid_plus import FaceInfoGenerator
|
|
from huggingface_hub import snapshot_download
|
|
snapshot_download(
|
|
'DIAMONIK7777/antelopev2',
|
|
local_dir='models/antelopev2',
|
|
)
|
|
app_face = FaceInfoGenerator(root_dir=insightface_root_path)
|
|
pipeline_mask = None
|
|
app_face_ = None
|
|
elif infer_mode=="story_maker":
|
|
from insightface.app import FaceAnalysis
|
|
from transformers import pipeline
|
|
pipeline_mask = pipeline("image-segmentation", model=mask_repo,
|
|
trust_remote_code=True)
|
|
|
|
app_face = FaceAnalysis(name='buffalo_l', root=insightface_root_path,
|
|
providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
|
|
app_face.prepare(ctx_id=0, det_size=(640, 640))
|
|
app_face_ = None
|
|
|
|
elif infer_mode=="story_and_maker":
|
|
from insightface.app import FaceAnalysis
|
|
from transformers import pipeline
|
|
pipeline_mask = pipeline("image-segmentation", model=mask_repo,
|
|
trust_remote_code=True)
|
|
if use_photov2:
|
|
from .utils.insightface_package import FaceAnalysis2
|
|
if auraface:
|
|
from huggingface_hub import snapshot_download
|
|
snapshot_download(
|
|
"fal/AuraFace-v1",
|
|
local_dir="models/auraface",
|
|
)
|
|
app_face = FaceAnalysis2(name="auraface",
|
|
providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
|
|
root=insightface_root_path,
|
|
allowed_modules=['detection', 'recognition'])
|
|
else:
|
|
app_face = FaceAnalysis2(providers=['CUDAExecutionProvider'],
|
|
allowed_modules=['detection', 'recognition'])
|
|
app_face.prepare(ctx_id=0, det_size=(640, 640))
|
|
app_face_ = FaceAnalysis(name='buffalo_l', root=insightface_root_path,
|
|
providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
|
|
app_face_.prepare(ctx_id=0, det_size=(640, 640))
|
|
else:
|
|
app_face = FaceAnalysis(name='buffalo_l', root=insightface_root_path,
|
|
providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
|
|
app_face.prepare(ctx_id=0, det_size=(640, 640))
|
|
app_face_ = None
|
|
elif infer_mode=="flux_pulid":
|
|
import insightface
|
|
from facexlib.parsing import init_parsing_model
|
|
from facexlib.utils.face_restoration_helper import FaceRestoreHelper
|
|
from insightface.app import FaceAnalysis
|
|
from huggingface_hub import snapshot_download
|
|
# antelopev2
|
|
snapshot_download('DIAMONIK7777/antelopev2', local_dir='models/antelopev2')
|
|
|
|
providers = ['CPUExecutionProvider'] if onnx_provider == 'cpu' else ['CUDAExecutionProvider', 'CPUExecutionProvider']
|
|
app_face = FaceAnalysis(name='antelopev2', root=insightface_root_path, providers=providers)
|
|
app_face.prepare(ctx_id=0, det_size=(640, 640))
|
|
# face_helper
|
|
face_helper = FaceRestoreHelper(
|
|
upscale_factor=1,
|
|
face_size=512,
|
|
crop_ratio=(1, 1),
|
|
det_model='retinaface_resnet50',
|
|
save_ext='png',
|
|
device=device,)
|
|
face_helper.face_parse = None
|
|
face_helper.face_parse = init_parsing_model(model_name='bisenet', device=device)
|
|
app_face_ = face_helper
|
|
handler_ante = insightface.model_zoo.get_model('models/antelopev2/glintr100.onnx',providers=providers)
|
|
handler_ante.prepare(ctx_id=0)
|
|
pipeline_mask = handler_ante
|
|
elif infer_mode=="infiniteyou":
|
|
from facexlib.recognition import init_recognition_model
|
|
from insightface.app import FaceAnalysis
|
|
# Load face encoder
|
|
|
|
app_face = FaceAnalysis(name='antelopev2',
|
|
root=insightface_root_path, providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
|
|
app_face.prepare(ctx_id=0, det_size=(640, 640))
|
|
app_face_ = init_recognition_model('arcface', device='cuda')
|
|
pipeline_mask = None
|
|
else:
|
|
app_face = None
|
|
pipeline_mask = None
|
|
app_face_ = None
|
|
return app_face,pipeline_mask,app_face_
|
|
|
|
|
|
def get_float(str_in):
|
|
list_str=str_in.split(",")
|
|
float_box=[float(x) for x in list_str]
|
|
return float_box
|
|
|
|
|
|
def adjust_indices(original_indices, deleted_indices):
|
|
"""根据删除的索引列表调整原索引"""
|
|
# 处理删除列表为空的特殊情况
|
|
if not deleted_indices:
|
|
return original_indices.copy() # 直接返回原索引，无需调整
|
|
|
|
deleted_sorted = sorted(deleted_indices)
|
|
adjusted = []
|
|
for idx in original_indices:
|
|
# 计算偏移量：有多少个删除索引 < 当前索引
|
|
offset = sum(1 for d in deleted_sorted if d < idx)
|
|
new_idx = idx - offset
|
|
# 如果原索引未被删除，则保留
|
|
if idx not in deleted_sorted:
|
|
adjusted.append(new_idx)
|
|
else:
|
|
adjusted.append(-1) # 标记为无效
|
|
return adjusted
|
|
|
|
|
|
def get_insight_dict(app_face,app_face_,pipeline_mask,infer_mode,use_photov2,image_list,
|
|
character_list_,condition_image,width, height,model=None,image_proj_model=None):
|
|
input_id_emb_s_dict = {}
|
|
input_id_img_s_dict = {}
|
|
input_id_emb_un_dict = {}
|
|
|
|
for ind, img in enumerate(image_list): # 最大只有2个ID
|
|
if infer_mode == "story" and use_photov2:
|
|
from .utils.insightface_package import analyze_faces
|
|
img_ = np.array(img)
|
|
img_ = cv2.cvtColor(img_, cv2.COLOR_RGB2BGR)
|
|
faces = analyze_faces(app_face, img_, )
|
|
id_embed_list = torch.from_numpy((faces[0]['embedding']))
|
|
crop_image = img_
|
|
uncond_id_embeddings = None
|
|
elif infer_mode == "kolor_face":
|
|
device = (
|
|
"cuda"
|
|
if torch.cuda.is_available()
|
|
else "mps" if torch.backends.mps.is_available() else "cpu"
|
|
)
|
|
face_info = app_face.get_faceinfo_one_img(img)
|
|
face_bbox_square = face_bbox_to_square(face_info["bbox"])
|
|
crop_image = img.crop(face_bbox_square)
|
|
crop_image = crop_image.resize((336, 336))
|
|
face_embeds = torch.from_numpy(np.array([face_info["embedding"]]))
|
|
id_embed_list = face_embeds.to(device, dtype=torch.float16)
|
|
uncond_id_embeddings = None
|
|
elif infer_mode=="story_maker":
|
|
crop_image = pipeline_mask(img, return_mask=True).convert('RGB') # outputs a pillow mask
|
|
# timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
# crop_image.copy().save(os.path.join(folder_paths.get_output_directory(),f"{timestamp}_mask.png"))
|
|
face_info = app_face.get(cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR))
|
|
id_embed_list = sorted(face_info,key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[-1] # only use the maximum face
|
|
|
|
uncond_id_embeddings = img
|
|
elif infer_mode=="story_and_maker":
|
|
if use_photov2:
|
|
from .utils.insightface_package import analyze_faces
|
|
img = np.array(img)
|
|
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
|
|
faces = analyze_faces(app_face, img, )
|
|
id_embed_list = torch.from_numpy((faces[0]['embedding']))
|
|
crop_image = pipeline_mask(img, return_mask=True).convert(
|
|
'RGB') # outputs a pillow mask
|
|
face_info = app_face_.get(cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR))
|
|
uncond_id_embeddings = sorted(face_info,key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[-1] # only use the maximum face
|
|
#photomake_mode = "v2"
|
|
# make+v2模式下，emb存v2的向量，corp 和 unemb 存make的向量
|
|
else: # V1不需要调用emb
|
|
crop_image = pipeline_mask(img, return_mask=True).convert(
|
|
'RGB') # outputs a pillow mask
|
|
face_info = app_face.get(cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR))
|
|
id_embed_list = sorted(face_info,key=lambda x: (x['bbox'][2] - x['bbox'][0]) * (x['bbox'][3] - x['bbox'][1]))[-1] # only use the maximum face
|
|
uncond_id_embeddings = img
|
|
|
|
elif infer_mode=="flux_pulid":
|
|
id_image = resize_numpy_image_long(img, 1024)
|
|
use_true_cfg = abs(1.0 - 1.0) > 1e-2
|
|
|
|
id_embed_list, uncond_id_embeddings=model.pulid_model.get_id_embedding_( id_image,app_face_,app_face,pipeline_mask,cal_uncond=use_true_cfg)
|
|
|
|
|
|
#id_embed_list, uncond_id_embeddings = pipe.pulid_model.get_id_embedding(id_image,cal_uncond=use_true_cfg)
|
|
crop_image = img
|
|
|
|
elif infer_mode=="infiniteyou":
|
|
|
|
def _detect_face(app_face, id_image_cv2):
|
|
face_info = app_face.get(id_image_cv2)
|
|
if face_info:
|
|
return face_info
|
|
else:
|
|
print("No face detected in the input ID image")
|
|
return []
|
|
from .pipelines.pipeline_infu_flux import extract_arcface_bgr_embedding,resize_and_pad_image,draw_kps
|
|
# Extract ID embeddings
|
|
print('Preparing ID embeddings')
|
|
id_image_cv2 = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
|
|
face_info = _detect_face(app_face,id_image_cv2)
|
|
if len(face_info) == 0:
|
|
raise ValueError('No face detected in the input ID image')
|
|
|
|
face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1] # only use the maximum face
|
|
landmark = face_info['kps']
|
|
id_embed = extract_arcface_bgr_embedding(id_image_cv2, landmark, app_face_)
|
|
id_embed = id_embed.clone().unsqueeze(0).float().cuda()
|
|
id_embed = id_embed.reshape([1, -1, 512])
|
|
id_embed = id_embed.to(device='cuda', dtype=torch.bfloat16)
|
|
with torch.no_grad():
|
|
id_embed = image_proj_model(id_embed)
|
|
bs_embed, seq_len, _ = id_embed.shape
|
|
id_embed = id_embed.repeat(1, 1, 1)
|
|
id_embed = id_embed.view(bs_embed * 1, seq_len, -1)
|
|
id_embed = id_embed.to(device='cuda', dtype=torch.bfloat16)
|
|
|
|
# Load control image
|
|
print('Preparing the control image')
|
|
if isinstance(condition_image, torch.Tensor):
|
|
e1, _, _, _ = condition_image.size()
|
|
if e1 == 1:
|
|
cn_image_load = [nomarl_upscale(condition_image, width, height)]
|
|
else:
|
|
img_list = list(torch.chunk(condition_image, chunks=e1))
|
|
cn_image_load = [nomarl_upscale(img, width, height) for img in img_list]
|
|
# control_image = control_image.convert("RGB")
|
|
# control_image = resize_and_pad_image(control_image, (width, height))
|
|
control_image=[] #如果是单人多张控制图，可能导致失序，所以统一改成列表，并在采样的时候依次调用
|
|
for img in cn_image_load:
|
|
face_info = _detect_face(app_face,cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR))
|
|
if len(face_info) == 0:
|
|
print('No face detected in the control image,use empty image,无法识别到面部,加载全黑图片替代.') #卡通人物很难识别，所以避免反复加载， 直接用黑图
|
|
face_cn=Image.fromarray(np.zeros([height, width, 3]).astype(np.uint8))
|
|
else:
|
|
face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1] # only use the maximum face
|
|
face_cn = draw_kps(img, face_info['kps'])
|
|
control_image.append(face_cn)
|
|
else:
|
|
out_img = np.zeros([height, width, 3])
|
|
control_image = Image.fromarray(out_img.astype(np.uint8))
|
|
id_embed_list=id_embed
|
|
crop_image = control_image # inf use crop to control img
|
|
uncond_id_embeddings = None
|
|
else:
|
|
id_embed_list = None
|
|
uncond_id_embeddings = None
|
|
crop_image = None
|
|
input_id_img_s_dict[character_list_[ind]] = [crop_image]
|
|
input_id_emb_s_dict[character_list_[ind]] = [id_embed_list]
|
|
input_id_emb_un_dict[character_list_[ind]] = [uncond_id_embeddings]
|
|
|
|
|
|
app_face = None
|
|
app_face_= None
|
|
pipeline_mask =None
|
|
gc_cleanup()
|
|
|
|
if isinstance(condition_image, torch.Tensor) and infer_mode=="story_maker":
|
|
e1, _, _, _ = condition_image.size()
|
|
if e1 == 1:
|
|
cn_image_load = [nomarl_upscale(condition_image, width, height)]
|
|
else:
|
|
img_list = list(torch.chunk(condition_image, chunks=e1))
|
|
cn_image_load = [nomarl_upscale(img, width, height) for img in img_list]
|
|
input_id_cloth_dict = {}
|
|
if len(cn_image_load)>2:
|
|
cn_image_load_role=cn_image_load[0:2]
|
|
else:
|
|
cn_image_load_role=cn_image_load
|
|
for ind, img in enumerate(cn_image_load_role):
|
|
input_id_cloth_dict[character_list_[ind]] = [img]
|
|
if len(cn_image_load)>2: #处理多张control img
|
|
my_list=cn_image_load[2:]
|
|
for ind,img in enumerate(my_list):
|
|
input_id_cloth_dict[f"dual{ind}"] = [img]
|
|
else:
|
|
input_id_cloth_dict = {}
|
|
return input_id_emb_s_dict,input_id_img_s_dict,input_id_emb_un_dict,input_id_cloth_dict
|
|
|
|
|
|
def load_model_tag(repo,device,select_method):
|
|
if "flor" in select_method.lower():#"thwri/CogFlorence-2-Large-Freeze"
|
|
#pip install flash_attn
|
|
from transformers import AutoModelForCausalLM, AutoProcessor, AutoConfig
|
|
model = AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True).to(
|
|
device)
|
|
processor = AutoProcessor.from_pretrained(repo, trust_remote_code=True)
|
|
else:
|
|
model = AutoModel.from_pretrained(repo, trust_remote_code=True)
|
|
processor = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)#tokenizer
|
|
model.eval()
|
|
return model,processor
|
|
|
|
class StoryLiteTag:
|
|
def __init__(self, device,temperature,select_method,repo="pzc163/MiniCPMv2_6-prompt-generator",):
|
|
self.device = device
|
|
self.repo = repo
|
|
self.select_method=select_method
|
|
self.model, self.processor=load_model_tag(self.repo, self.device,self.select_method)
|
|
self.temperature=temperature
|
|
def run_tag(self,image):
|
|
if "flor" in self.select_method.lower():
|
|
inputs = self.processor(text="<MORE_DETAILED_CAPTION>" , images=image, return_tensors="pt").to(device)
|
|
generated_ids = self.model.generate(
|
|
input_ids=inputs["input_ids"],
|
|
pixel_values=inputs["pixel_values"],
|
|
max_new_tokens=1024,
|
|
num_beams=3,
|
|
do_sample=True
|
|
)
|
|
generated_text = self.processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
|
|
parsed_answer = self.processor.post_process_generation(generated_text, task="<MORE_DETAILED_CAPTION>" ,
|
|
image_size=(image.width, image.height))
|
|
res=parsed_answer["<MORE_DETAILED_CAPTION>"]
|
|
else:
|
|
question = 'Provide a detailed description of the details and content contained in the image, and generate a short prompt that can be used for image generation tasks in Stable Diffusion,remind you only need respons prompt itself and no other information.'
|
|
msgs = [{'role': 'user', 'content': [image, question]}]
|
|
res = self.model.chat(
|
|
image=None,
|
|
msgs=msgs,
|
|
tokenizer=self.processor,# tokenizer
|
|
temperature=self.temperature
|
|
)
|
|
res=res.split(":",1)[1].strip('"')
|
|
s=res.strip()
|
|
res=re.sub(r'^\n+|\n+$', '', s)
|
|
res.strip("'")
|
|
logging.info(f"{res}")
|
|
return res
|
|
|
|
|
|
def images_generator(img_list: list, ):
|
|
# get img size
|
|
sizes = {}
|
|
for image_ in img_list:
|
|
if isinstance(image_, Image.Image):
|
|
count = sizes.get(image_.size, 0)
|
|
sizes[image_.size] = count + 1
|
|
elif isinstance(image_, np.ndarray):
|
|
count = sizes.get(image_.shape[:2][::-1], 0)
|
|
sizes[image_.shape[:2][::-1]] = count + 1
|
|
else:
|
|
raise "unsupport image list,must be pil or cv2!!!"
|
|
size = max(sizes.items(), key=lambda x: x[1])[0]
|
|
yield size[0], size[1]
|
|
|
|
# any to tensor
|
|
def load_image(img_in):
|
|
if isinstance(img_in, Image.Image):
|
|
img_in = img_in.convert("RGB")
|
|
i = np.array(img_in, dtype=np.float32)
|
|
i = torch.from_numpy(i).div_(255)
|
|
if i.shape[0] != size[1] or i.shape[1] != size[0]:
|
|
i = torch.from_numpy(i).movedim(-1, 0).unsqueeze(0)
|
|
i = common_upscale(i, size[0], size[1], "lanczos", "center")
|
|
i = i.squeeze(0).movedim(0, -1).numpy()
|
|
return i
|
|
elif isinstance(img_in, np.ndarray):
|
|
i = cv2.cvtColor(img_in, cv2.COLOR_BGR2RGB).astype(np.float32)
|
|
i = torch.from_numpy(i).div_(255)
|
|
print(i.shape)
|
|
return i
|
|
else:
|
|
raise "unsupport image list,must be pil,cv2 or tensor!!!"
|
|
|
|
total_images = len(img_list)
|
|
processed_images = 0
|
|
pbar = ProgressBar(total_images)
|
|
images = map(load_image, img_list)
|
|
try:
|
|
prev_image = next(images)
|
|
while True:
|
|
next_image = next(images)
|
|
yield prev_image
|
|
processed_images += 1
|
|
pbar.update_absolute(processed_images, total_images)
|
|
prev_image = next_image
|
|
except StopIteration:
|
|
pass
|
|
if prev_image is not None:
|
|
yield prev_image
|
|
|
|
|
|
def load_images_list(img_list: list, ):
|
|
gen = images_generator(img_list)
|
|
(width, height) = next(gen)
|
|
images = torch.from_numpy(np.fromiter(gen, np.dtype((np.float32, (height, width, 3)))))
|
|
if len(images) == 0:
|
|
raise FileNotFoundError(f"No images could be loaded .")
|
|
return images
|
|
|
|
|
|
def fitter_cf_model_type(model):
|
|
VALUE_=model.model.model_type.value
|
|
if VALUE_ == 8:
|
|
cf_model_type = "FLUX"
|
|
elif VALUE_ == 4:
|
|
cf_model_type = "CASCADE"
|
|
elif VALUE_ == 6:
|
|
cf_model_type = "FLOW"
|
|
elif VALUE_ == 5:
|
|
cf_model_type = "EDM"
|
|
elif VALUE_ == 1:
|
|
cf_model_type = "EPS"
|
|
elif VALUE_ == 2:
|
|
cf_model_type = "V_PREDICTION"
|
|
elif VALUE_ == 3:
|
|
cf_model_type = "V_PREDICTION_EDM"
|
|
elif VALUE_ == 7:
|
|
cf_model_type = "PREDICTION_CONTINUOUS"
|
|
else:
|
|
raise "unsupport checkpoints"
|
|
return cf_model_type
|
|
|
|
def pre_text2infer(role_text,scene_text,lora_trigger_words,use_lora,tag_list):
|
|
'''
|
|
|
|
Args:
|
|
role_index_dict: {'[Taylor]': [0, 1], '[Lecun]': [2, 3]}
|
|
invert_role_index_dict{0: ['[Taylor]'], 1: ['[Taylor]'], 2: ['[Lecun]'], 3: ['[Lecun]']}
|
|
ref_role_index_dict {'[Taylor]': [0, 1], '[Lecun]': [2, 3]}
|
|
ref_role_totals[0, 1, 2, 3]
|
|
role_list ['[Taylor]', '[Lecun]']
|
|
role_dict {'[Taylor]': ' a woman img, wearing a white T-shirt, blue loose hair.', '[Lecun]': ' a man img,wearing a suit,black hair.'}
|
|
nc_txt_list [' a panda']
|
|
nc_indexs[4]
|
|
positions_index_dual[]
|
|
positions_index_char_1 0 2
|
|
positions_index_char_2 []
|
|
prompts_dual[] #['[A] play whith [B] in the garden']
|
|
index_char_1_list [0, 1]
|
|
index_char_2_list[2, 3]
|
|
|
|
Returns:
|
|
character_index_dict:{'[Taylor]': [0, 3], '[sam]': [1, 2]},if 1 role {'[Taylor]': [0, 1, 2]}
|
|
|
|
'''
|
|
add_trigger_words = " " + lora_trigger_words + " style "
|
|
# pre role_text
|
|
role_dict, role_list = character_to_dict(role_text, use_lora, add_trigger_words)
|
|
|
|
id_len=len(role_dict)
|
|
#pre scene_text
|
|
scene_text_origin = [i.strip() for i in scene_text.splitlines()]
|
|
scene_text_origin = [i for i in scene_text_origin if '[' in i] # 删除空行
|
|
prompt_sence = [i for i in scene_text_origin if not len(extract_content_from_brackets(i)) >= 2] # 剔除双角色的场景词
|
|
|
|
positions_index_dual = [index for index, prompt in enumerate(scene_text_origin) if
|
|
len(extract_content_from_brackets(prompt)) >= 2] #获取双角色出现的位置
|
|
prompts_dual = [prompt for prompt in scene_text_origin if len(extract_content_from_brackets(prompt)) >= 2] # 改成单句中双方括号方法，利于MS组句，[A]... [B]...[C]
|
|
|
|
#print(id_len,role_list)
|
|
if id_len == 2:
|
|
index_char_1_list=[index for index, prompt in enumerate(scene_text_origin) if role_list[0] in prompt]
|
|
positions_index_char_1 = index_char_1_list[0] # 获取角色出现的索引列表，并获取首次出现的位置
|
|
index_char_2_list=[index for index, prompt in enumerate(scene_text_origin) if role_list[1] in prompt]
|
|
positions_index_char_2 = index_char_2_list[0] # 获取角色出现的索引列表，并获取首次出现的位置
|
|
|
|
# print(positions_index_char_1, positions_index_char_2) #0,2
|
|
else:
|
|
index_char_1_list=[index for index, prompt in enumerate(scene_text_origin) if role_list[0] in prompt]
|
|
index_char_2_list=[]
|
|
positions_index_char_1=[]
|
|
positions_index_char_2=[]
|
|
|
|
if index_char_1_list and index_char_2_list: #dual 情况下，该列表有误，须排除重复元素
|
|
common_ = set(index_char_1_list) & set(index_char_2_list)
|
|
if common_:
|
|
index_char_1_list=[x for x in index_char_1_list if x not in common_]
|
|
index_char_2_list = [x for x in index_char_2_list if x not in common_]
|
|
|
|
|
|
clipped_prompts = prompt_sence[:] # copy
|
|
|
|
nc_indexs = []
|
|
for ind, prompt in enumerate(clipped_prompts): #获取NC的index
|
|
if "[NC]" in prompt:
|
|
nc_indexs.append(ind)
|
|
if ind < id_len:
|
|
raise f"The first [role] row need be a id prompts, cannot use [NC]!"
|
|
prompts = [
|
|
i if "[NC]" not in i else i.replace("[NC]", "") for i in clipped_prompts]
|
|
#去除#
|
|
prompts = [prompt.rpartition("#")[0] if "#" in prompt else prompt for prompt in prompts]
|
|
|
|
# character_dict:{'[Taylor]': ' a woman img, wearing a white T-shirt, blue loose hair.'},character_list:['[Taylor]'] 1role
|
|
#character_dict:{'[Taylor]': ' a woman img, wearing a white T-shirt, blue loose hair.', '[Lecun]': ' a man img,wearing a suit,black hair.'},character_list:['[Taylor]', '[Lecun]'] 2 role
|
|
#获取字典，实际用词等
|
|
role_index_dict, invert_role_index_dict, replace_prompts, ref_role_index_dict, ref_role_totals= process_original_text(role_dict, prompts)
|
|
|
|
if tag_list: #tag方法
|
|
if len(tag_list) < len(replace_prompts):
|
|
raise "The number of input condition images is less than the number of scene prompts!"
|
|
replace_prompts = [prompt + " " + tag_list[i] for i, prompt in enumerate(replace_prompts)]
|
|
|
|
if nc_indexs:
|
|
for x in nc_indexs: # 获取NC列表
|
|
nc_txt_list = [item for i, item in enumerate(replace_prompts) if i == x]
|
|
|
|
for x in nc_indexs: # 去除NC列表
|
|
replace_prompts = [item for i, item in enumerate(replace_prompts) if i != x]
|
|
else:
|
|
nc_txt_list = []
|
|
|
|
|
|
replace_prompts=[i for i in replace_prompts]
|
|
|
|
return replace_prompts,role_index_dict,invert_role_index_dict,ref_role_index_dict,ref_role_totals,role_list,role_dict,nc_txt_list,nc_indexs,positions_index_char_1,positions_index_char_2,positions_index_dual,prompts_dual,index_char_1_list,index_char_2_list
|
|
|
|
|
|
def convert_cf2diffuser(model):
|
|
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_unet_checkpoint
|
|
from diffusers import UNet2DConditionModel
|
|
config_file = os.path.join(cur_path,"local_repo/unet/config.json")
|
|
cf_state_dict = model.diffusion_model.state_dict()
|
|
unet_state_dict = model.model_config.process_unet_state_dict_for_saving(cf_state_dict)
|
|
unet_config = UNet2DConditionModel.load_config(config_file)
|
|
Unet = UNet2DConditionModel.from_config(unet_config).to(device, torch.float16)
|
|
cf_state_dict = convert_ldm_unet_checkpoint(unet_state_dict, Unet.config)
|
|
Unet.load_state_dict(cf_state_dict, strict=False)
|
|
del cf_state_dict,unet_state_dict
|
|
gc.collect()
|
|
torch.cuda.empty_cache()
|
|
return Unet
|
|
|
|
|
|
def cf_clip(txt_list, clip, infer_mode,role_list,input_split=True):
|
|
|
|
if infer_mode == "classic":
|
|
input_split= False
|
|
if len(role_list)==1:
|
|
input_split= False
|
|
if input_split:
|
|
role_emb_dict={}
|
|
for role,role_text_list in zip(role_list,txt_list):
|
|
pos_cond_list=[]
|
|
for j in role_text_list:
|
|
tokens_p = clip.tokenize(j)
|
|
output_p = clip.encode_from_tokens(tokens_p, return_dict=True) # {"pooled_output":tensor}
|
|
cond_p = output_p.pop("cond")
|
|
if cond_p.shape[1] / 77 > 1 and infer_mode != "classic":
|
|
# logging.warning("prompt'tokens length is abvoe 77,split it")
|
|
cond_p = torch.chunk(cond_p, cond_p.shape[1] // 77, dim=1)[0]
|
|
positive = [cond_p, output_p]
|
|
pos_cond_list.append(positive)
|
|
role_emb_dict[role]=pos_cond_list
|
|
return role_emb_dict
|
|
|
|
pos_cond_list = []
|
|
for i in txt_list[0]:
|
|
tokens_p = clip.tokenize(i)
|
|
output_p = clip.encode_from_tokens(tokens_p, return_dict=True) # {"pooled_output":tensor}
|
|
cond_p = output_p.pop("cond")
|
|
if cond_p.shape[1] / 77 > 1 and infer_mode != "classic":
|
|
# logging.warning("prompt'tokens length is abvoe 77,split it")
|
|
cond_p = torch.chunk(cond_p, cond_p.shape[1] // 77, dim=1)[0]
|
|
if infer_mode == "classic":
|
|
positive = [[cond_p, output_p]]
|
|
else:
|
|
positive = [cond_p, output_p]
|
|
# logging.info(f"sampler text is {i}")
|
|
pos_cond_list.append(positive)
|
|
return pos_cond_list
|
|
|
|
def get_eot_idx_cf(tokenizer, prompt):
|
|
words = prompt.split()
|
|
start = 1
|
|
for w in words:
|
|
start += len(tokenizer.encode(w)) - 2
|
|
return start
|
|
|
|
def get_phrase_idx_cf(tokenizer, phrase, prompt, get_last_word=False, num=0):
|
|
def is_equal_words(pr_words, ph_words):
|
|
if len(pr_words) != len(ph_words):
|
|
return False
|
|
for pr_word, ph_word in zip(pr_words, ph_words):
|
|
if "-"+ph_word not in pr_word and ph_word != re.sub(r'[.!?,:]$', '', pr_word):
|
|
return False
|
|
return True
|
|
|
|
phrase_words = phrase.split()
|
|
if len(phrase_words) == 0:
|
|
return [0, 0], None
|
|
if get_last_word:
|
|
phrase_words = phrase_words[-1:]
|
|
# prompt_words = re.findall(r'\b[\w\'-]+\b', prompt)
|
|
prompt_words = prompt.split()
|
|
start = 1
|
|
end = 0
|
|
res_words = phrase_words
|
|
for i in range(len(prompt_words)):
|
|
if is_equal_words(prompt_words[i:i+len(phrase_words)], phrase_words):
|
|
if num != 0:
|
|
# skip this one
|
|
num -= 1
|
|
continue
|
|
end = start
|
|
res_words = prompt_words[i:i+len(phrase_words)]
|
|
res_words = [re.sub(r'[.!?,:]$', '', w) for w in res_words]
|
|
prompt_words[i+len(phrase_words)-1] = res_words[-1] # remove the last punctuation
|
|
for j in range(i, i+len(phrase_words)):
|
|
end += len(tokenizer.encode(prompt_words[j])) - 2
|
|
break
|
|
else:
|
|
start += len(tokenizer.encode(prompt_words[i])) - 2
|
|
|
|
if end == 0:
|
|
return [0, 0], None
|
|
|
|
return [start, end], res_words
|
|
|
|
def get_phrases_idx_cf(tokenizer, phrases, prompt):
|
|
res = []
|
|
phrase_cnt = {}
|
|
for phrase in phrases:
|
|
if phrase in phrase_cnt:
|
|
cur_cnt = phrase_cnt[phrase]
|
|
phrase_cnt[phrase] += 1
|
|
else:
|
|
cur_cnt = 0
|
|
phrase_cnt[phrase] = 1
|
|
res.append(get_phrase_idx_cf(tokenizer, phrase, prompt, num=cur_cnt)[0])
|
|
return res
|
|
|
|
def replicate_data_by_indices(data_list, index_list1, index_list2):
|
|
# 确定新列表的长度
|
|
max_index = max(max(index_list1), max(index_list2)) if (index_list1 and index_list2) else 0
|
|
new_list = [None] * (max_index + 1)
|
|
|
|
# 根据索引填充数据
|
|
for idx in index_list1:
|
|
new_list[idx] = data_list[0]
|
|
for idx in index_list2:
|
|
new_list[idx] = data_list[1]
|
|
|
|
return new_list
|
|
|
|
|
|
|
|
def get_ms_phrase_emb(boxes, device, weight_dtype, drop_grounding_tokens, bsz, phrase_idxes,
|
|
num_samples, eot_idxes,phrases,clip,tokenizer):
|
|
cross_attention_kwargs = None
|
|
grounding_kwargs = None
|
|
if boxes is not None:
|
|
boxes = torch.tensor(boxes).to(device, weight_dtype) #torch.Size([1, 2, 4])
|
|
if phrases is not None:
|
|
drop_grounding_tokens = drop_grounding_tokens if drop_grounding_tokens is not None else [0] * bsz
|
|
batch_boxes = boxes.view(bsz * boxes.shape[1], -1).to(device)
|
|
phrase_input_ids=[]
|
|
for phrase in phrases:
|
|
phrase_input_id = tokenizer(phrase, max_length=tokenizer.model_max_length,
|
|
padding="max_length", truncation=True,
|
|
return_tensors="pt").input_ids
|
|
int_list = phrase_input_id.tolist()[0]
|
|
clean_input_ids_1=[(i,1.0) for i in int_list]
|
|
clean_input_ids_2=[(i,1.0) for i in int_list if i==int_list[0] or i==49407 ]
|
|
clean_input_ids_2=clean_input_ids_2[:77] if len(clean_input_ids_2) >=77 else clean_input_ids_2 + [(0,1.0)]*(77 - len(clean_input_ids_2))
|
|
phrase_input_ids.append([clean_input_ids_1,clean_input_ids_2])
|
|
|
|
phrase_embeds_list = []
|
|
for i in phrase_input_ids:
|
|
_, pooled_prompt_embed_= clip.cond_stage_model.clip_l.encode_token_weights(i)
|
|
pooled_prompt_embed_= pooled_prompt_embed_.to(device,dtype=weight_dtype)
|
|
phrase_embeds_list.append(pooled_prompt_embed_)
|
|
#phrase_input_id = clip.tokenize(phrase, return_word_ids=False)["l"]
|
|
#output_=clip.cond_stage_model.clip_l.encode_token_weights(phrase_input_id)[1]
|
|
|
|
phrase_embeds=torch.cat(phrase_embeds_list,dim=0)
|
|
#print(phrase_embeds.shape,phrase_embeds.is_cuda,batch_boxes.shape,batch_boxes.is_cuda)#torch.Size([2, 768]) torch.Size([2 4])
|
|
grounding_kwargs = {"boxes": batch_boxes, "phrase_embeds": phrase_embeds,"drop_grounding_tokens": drop_grounding_tokens}
|
|
else:
|
|
grounding_kwargs = None
|
|
boxes = torch.repeat_interleave(boxes, repeats=num_samples, dim=0)
|
|
uncond_boxes = torch.zeros_like(boxes)
|
|
boxes = torch.cat([uncond_boxes, boxes], dim=0)
|
|
cross_attention_kwargs = {"boxes": boxes}
|
|
|
|
if phrase_idxes is not None:
|
|
# phrase_idxes = torch.tensor(phrase_idxes).to(device, torch.int)
|
|
# eot_idxes = torch.tensor(eot_idxes).to(device, torch.int)
|
|
phrase_idxes = torch.tensor(phrase_idxes).to(device, weight_dtype)
|
|
eot_idxes = torch.tensor(eot_idxes).to(device, weight_dtype)
|
|
|
|
phrase_idxes = torch.repeat_interleave(phrase_idxes, repeats=num_samples, dim=0)
|
|
eot_idxes = torch.repeat_interleave(eot_idxes, repeats=num_samples, dim=0)
|
|
uncond_phrase_idxes = torch.zeros_like(phrase_idxes)
|
|
uncond_eot_idxes = torch.zeros_like(eot_idxes)
|
|
phrase_idxes = torch.cat([uncond_phrase_idxes, phrase_idxes], dim=0)
|
|
eot_idxes = torch.cat([uncond_eot_idxes, eot_idxes], dim=0)
|
|
if cross_attention_kwargs is None:
|
|
cross_attention_kwargs = {"phrase_idxes": phrase_idxes, "eot_idxes": eot_idxes}
|
|
else:
|
|
cross_attention_kwargs["phrase_idxes"] = phrase_idxes
|
|
cross_attention_kwargs["eot_idxes"] = eot_idxes
|
|
|
|
return cross_attention_kwargs,grounding_kwargs
|
|
def Infer_MSdiffusion(model, photomake_or_ipadapter_path,image_embeds, prompt_embeds_, negative_prompt_embeds_,grounding_kwargs,cross_attention_kwargs, bsz,mask_threshold,height,width,steps,seed,cfg,pooled_prompt_embeds,negative_pooled_prompt_embeds,):
|
|
image_proj_model = Resampler(
|
|
dim=1280,
|
|
depth=4,
|
|
dim_head=64,
|
|
heads=20,
|
|
num_queries=16,
|
|
embedding_dim=1664,
|
|
output_dim=model.unet.config.cross_attention_dim,
|
|
ff_mult=4,
|
|
latent_init_mode="grounding",
|
|
phrase_embeddings_dim=768,
|
|
).to(device, dtype=torch.float16)
|
|
PIPE = MSAdapterWarpper(model, image_proj_model, ckpt_path=photomake_or_ipadapter_path, device=device, num_tokens=16)
|
|
samples=PIPE.generate(model, image_embeds, prompt_embeds_, negative_prompt_embeds_,grounding_kwargs,cross_attention_kwargs, bsz,height,width,steps,seed,cfg,pooled_prompt_embeds,negative_pooled_prompt_embeds,scale=1.0,
|
|
num_samples=bsz, weight_dtype=torch.float16, subject_scales=None, mask_threshold=mask_threshold, start_step=5)
|
|
return samples
|
|
|
|
def convert_clip2diffuser(clip):
|
|
|
|
from transformers import (
|
|
CLIPTextModel,CLIPTextModelWithProjection,
|
|
CLIPTextConfig,
|
|
)
|
|
from contextlib import nullcontext
|
|
|
|
try:
|
|
from accelerate import init_empty_weights
|
|
from accelerate.utils import set_module_tensor_to_device
|
|
is_accelerate_available = True
|
|
except:
|
|
pass
|
|
|
|
import comfy.model_management as mm
|
|
|
|
offload_device=mm.unet_offload_device()
|
|
|
|
text_encoder_config=CLIPTextConfig.from_pretrained(os.path.join(cur_path, 'local_repo/text_encoder'), local_files_only=True)
|
|
ctx = init_empty_weights if is_accelerate_available else nullcontext
|
|
with ctx():
|
|
text_encoder = CLIPTextModel(text_encoder_config)
|
|
text_encoder_sd = clip.get_sd()
|
|
text_encoder_2_sd = clip.get_sd()
|
|
|
|
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_open_clip_checkpoint,convert_ldm_clip_checkpoint
|
|
text_encoder = convert_ldm_clip_checkpoint(text_encoder_sd, text_encoder=text_encoder)
|
|
text_encoder=None
|
|
text_encoder_2 = convert_open_clip_checkpoint(text_encoder_2_sd,config_name=os.path.join(cur_path, 'local_repo/text_encoder_2'))
|
|
|
|
return text_encoder,text_encoder_2
|
|
|
|
|
|
|
|
def convert_cfvae2diffuser(VAE,use_flux=False):
|
|
from diffusers import AutoencoderKL
|
|
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_vae_checkpoint
|
|
if use_flux:
|
|
vae_config = os.path.join(cur_path, "config/FLUX.1-dev/vae/config.json")
|
|
else:
|
|
vae_config = os.path.join(cur_path, "local_repo/vae/config.json")
|
|
vae_state_dict=VAE.get_sd()
|
|
ae_config = AutoencoderKL.load_config(vae_config)
|
|
|
|
if not use_flux:
|
|
AE = AutoencoderKL.from_config(ae_config).to(device, torch.float16)
|
|
cf_state_dict = convert_ldm_vae_checkpoint(vae_state_dict, AE.config)
|
|
AE.load_state_dict(cf_state_dict, strict=False)
|
|
del cf_state_dict,vae_state_dict
|
|
else:
|
|
AE = AutoencoderKL.from_config(ae_config).to(device, torch.bfloat16)
|
|
AE.load_state_dict(vae_state_dict, strict=False)
|
|
del vae_state_dict
|
|
torch.cuda.empty_cache()
|
|
return AE
|
|
|
|
|
|
def Loader_storydiffusion(cf_model,photomake_ckpt_path,VAE,UNET=None):
|
|
sdxl_repo = os.path.join(cur_path, "local_repo")
|
|
vae_config=OmegaConf.load(os.path.join(cur_path,"local_repo/vae/config.json"))
|
|
|
|
from .utils.pipeline_wrapper import StableDiffusionXLPipelineWrapper
|
|
from .utils.pipeline_img import PhotoMakerStableDiffusionXLPipeline as PhotoMakerStableDiffusionXLPipelineWrapper
|
|
from .utils.pipeline_v2_warrper import PhotoMakerStableDiffusionXLPipeline as PhotoMakerStableDiffusionXLPipelineV2
|
|
if cf_model is not None:
|
|
Unet=convert_cf2diffuser(cf_model.model)
|
|
if photomake_ckpt_path is not None: # guess img2img
|
|
if VAE is None:
|
|
raise "need link a SDXL vae when use img2img!"
|
|
vae = convert_cfvae2diffuser(VAE)
|
|
if "v1" in photomake_ckpt_path:
|
|
model = PhotoMakerStableDiffusionXLPipelineWrapper.from_pretrained(sdxl_repo, unet=Unet, vae=vae,text_encoder=None,text_encoder_2=None,
|
|
torch_dtype=torch.float16, use_safetensors=True)
|
|
model.load_photomaker_adapter(
|
|
photomake_ckpt_path,
|
|
subfolder="",
|
|
weight_name="photomaker-v1.bin",
|
|
trigger_word="img" # define the trigger word
|
|
)
|
|
|
|
else:
|
|
model = PhotoMakerStableDiffusionXLPipelineV2.from_pretrained(sdxl_repo, unet=Unet, vae=vae,text_encoder=None,text_encoder_2=None,
|
|
torch_dtype=torch.float16, use_safetensors=True)
|
|
model.load_photomaker_adapter(
|
|
photomake_ckpt_path,
|
|
subfolder="",
|
|
weight_name="photomaker-v2.bin",
|
|
trigger_word="img",
|
|
pm_version='v2',
|
|
)
|
|
else:
|
|
model = StableDiffusionXLPipelineWrapper.from_pretrained(sdxl_repo, unet=Unet, vae_config=vae_config,
|
|
torch_dtype=torch.float16, use_safetensors=True)
|
|
elif UNET is not None:
|
|
model = StableDiffusionXLPipelineWrapper.from_pretrained(sdxl_repo, unet=UNET.unet, vae_config=vae_config,
|
|
torch_dtype=torch.float16, use_safetensors=True)
|
|
else:
|
|
raise "need link a comfyUI checkpoint model!"
|
|
|
|
model.to(device)
|
|
return model
|
|
|
|
|
|
|
|
|
|
def Loader_story_maker(cf_model,ipadapter_ckpt_path,VAE,low_vram,lora_scale,controlnet=None,UNET=None):
|
|
sdxl_repo = os.path.join(cur_path, "local_repo")
|
|
#vae_config = OmegaConf.load(vae_config)
|
|
if cf_model is not None:
|
|
Unet = convert_cf2diffuser(cf_model.model)
|
|
AE=convert_cfvae2diffuser(VAE)
|
|
from .StoryMaker.pipeline_sdxl_storymaker_wrapper import StableDiffusionXLStoryMakerPipeline as StableDiffusionXLStoryMakerPipeline_wapper
|
|
model = StableDiffusionXLStoryMakerPipeline_wapper.from_pretrained(sdxl_repo, vae=AE,unet=Unet,text_encoder=None,text_encoder_2=None,
|
|
torch_dtype=torch.float16, use_safetensors=True)
|
|
elif UNET is not None:
|
|
AE=convert_cfvae2diffuser(VAE)
|
|
from .StoryMaker.pipeline_sdxl_storymaker_wrapper import StableDiffusionXLStoryMakerPipeline as StableDiffusionXLStoryMakerPipeline_wapper
|
|
model = StableDiffusionXLStoryMakerPipeline_wapper.from_pretrained(sdxl_repo, vae=AE,unet=UNET.unet,text_encoder=None,text_encoder_2=None,
|
|
torch_dtype=torch.float16, use_safetensors=True)
|
|
else:
|
|
raise "need link a comfyUI checkpoint model!"
|
|
|
|
|
|
if device != "mps":
|
|
if not low_vram:
|
|
model.cuda()
|
|
|
|
model.load_storymaker_adapter(ipadapter_ckpt_path, scale=0.8, lora_scale=lora_scale, controlnet=controlnet)
|
|
#model.scheduler = UniPCMultistepScheduler.from_config(model.scheduler.config)
|
|
return model
|
|
|
|
|
|
def Loader_Flux_Pulid(model,cf_model, ipadapter_ckpt_path,quantized_mode,aggressive_offload,offload,if_repo,clip_vision_path):
|
|
|
|
logging.info("start flux-pulid processing...")
|
|
from .PuLID.app_flux import FluxGenerator
|
|
|
|
pipe = FluxGenerator(model, "cuda:0", offload,aggressive_offload, pretrained_model=ipadapter_ckpt_path,
|
|
quantized_mode=quantized_mode, if_repo=if_repo,clip_vision_path=clip_vision_path)
|
|
|
|
return pipe
|
|
|
|
def Loader_InfiniteYou(extra_info,VAE,quantize_mode):
|
|
logging.info("start InfiniteYou mode processing...")
|
|
from .pipelines.pipeline_infu_flux import InfUFluxPipeline
|
|
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig,FluxTransformer2DModel
|
|
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig,T5EncoderModel
|
|
from .pipelines.resampler import Resampler
|
|
|
|
|
|
# quantize T5
|
|
# quant_config = TransformersBitsAndBytesConfig(
|
|
# load_in_4bit=True,
|
|
# bnb_4bit_quant_type="nf4",
|
|
# )
|
|
|
|
# text_encoder_2_4bit = T5EncoderModel.from_pretrained(
|
|
# repo_id,
|
|
# subfolder="text_encoder_2",
|
|
# quantization_config=quant_config,
|
|
# torch_dtype=torch.bfloat16,
|
|
# ) # init nf4
|
|
repo_list=[i for i in [extra_info.get("repo1_path") , extra_info.get("repo2_path")] if i is not None]
|
|
assert repo_list is not [] ,"repo_list is None"
|
|
find_infusenet=[i for i in repo_list if "sim_stage1" in i or "aes_stage2" in i]
|
|
|
|
assert len(find_infusenet)>=1 ,"no infusenet"
|
|
|
|
repo_id=os.path.join(cur_path, "config/FLUX.1-dev")
|
|
use_svdq=False
|
|
if extra_info.get("svdq_repo") is not None:
|
|
print("use svdq quantization")
|
|
from nunchaku import NunchakuFluxTransformer2dModel
|
|
use_svdq=True
|
|
transformer = NunchakuFluxTransformer2dModel.from_pretrained(extra_info.get("svdq_repo"),offload=True)
|
|
try:
|
|
transformer.set_attention_impl("nunchaku-fp16")
|
|
except:
|
|
pass
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
elif extra_info.get("gguf_path") is not None:
|
|
print("use gguf quantization")
|
|
from diffusers import GGUFQuantizationConfig
|
|
transformer = FluxTransformer2DModel.from_single_file(
|
|
extra_info.get("gguf_path"),
|
|
config=os.path.join(repo_id, "transformer"),
|
|
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
elif extra_info.get("unet_path") is not None:
|
|
|
|
if quantize_mode=="fp8":
|
|
transformer = FluxTransformer2DModel.from_single_file(extra_info.get("unet_path"), config=os.path.join(repo_id,"transformer/config.json"),
|
|
torch_dtype=torch.bfloat16)
|
|
else:
|
|
from safetensors.torch import load_file
|
|
t_state_dict=load_file(extra_info.get("unet_path"))
|
|
unet_config = FluxTransformer2DModel.load_config(os.path.join(repo_id,"transformer/config.json"))
|
|
transformer = FluxTransformer2DModel.from_config(unet_config).to(torch.bfloat16)
|
|
transformer.load_state_dict(t_state_dict, strict=False)
|
|
del t_state_dict
|
|
gc_cleanup()
|
|
else:
|
|
vae = None
|
|
find_flux=[i for i in repo_list if "flux" in i ]
|
|
if not find_flux:
|
|
raise "need fill flux repo in EasyFunction_Lite repo1 or repo2!"
|
|
print("use nf4 quantization")
|
|
if quantize_mode=="fp8":
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
find_flux[0],
|
|
subfolder="transformer",
|
|
quantization_config=DiffusersBitsAndBytesConfig(load_in_8bit=True,),
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
|
|
elif quantize_mode=="nf4": #nf4
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
find_flux[0],
|
|
subfolder="transformer",
|
|
quantization_config=DiffusersBitsAndBytesConfig(
|
|
load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
|
|
),
|
|
torch_dtype=torch.bfloat16,)
|
|
else:
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
find_flux[0],
|
|
subfolder="transformer",
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
|
|
|
|
infusenet_path = os.path.join(find_infusenet[0], 'InfuseNetModel')
|
|
|
|
|
|
pipe = InfUFluxPipeline(
|
|
repo_id,infusenet_path,
|
|
text_encoder_2=None,
|
|
text_encoder=None,
|
|
transformer=transformer,
|
|
vae=vae,
|
|
use_svdq=use_svdq,
|
|
)
|
|
# Load image proj model
|
|
num_tokens = 8 # image_proj_num_tokens
|
|
image_emb_dim = 512
|
|
image_proj_model = Resampler(
|
|
dim=1280,
|
|
depth=4,
|
|
dim_head=64,
|
|
heads=20,
|
|
num_queries=num_tokens,
|
|
embedding_dim=image_emb_dim,
|
|
output_dim=4096,
|
|
ff_mult=4,
|
|
)
|
|
image_proj_model_path = os.path.join(find_infusenet[0], 'image_proj_model.bin')
|
|
ipm_state_dict = torch.load(image_proj_model_path, map_location="cpu",weights_only=False)
|
|
image_proj_model.load_state_dict(ipm_state_dict['image_proj'])
|
|
del ipm_state_dict
|
|
torch.cuda.empty_cache()
|
|
image_proj_model.to('cuda', torch.bfloat16)
|
|
image_proj_model.eval()
|
|
|
|
pipe.pipe.enable_model_cpu_offload()
|
|
return pipe,image_proj_model
|
|
|
|
def load_pipeline_consistory(cf_model,VAE):
|
|
from .consistory.consistory_unet_sdxl import ConsistorySDXLUNet2DConditionModel
|
|
from .consistory.consistory_pipeline_wapper import ConsistoryExtendAttnSDXLPipeline as ConsistoryExtendAttnSDXLPipeline_wapper
|
|
float_type = torch.float16
|
|
|
|
device = torch.device(f'cuda:0') if torch.cuda.is_available() else torch.device('cpu')
|
|
|
|
if cf_model is not None:
|
|
config_file = os.path.join(cur_path,"local_repo/unet/config.json")
|
|
sdxl_repo = os.path.join(cur_path, "local_repo")
|
|
|
|
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_unet_checkpoint
|
|
cf_state_dict = cf_model.model.diffusion_model.state_dict()
|
|
unet_state_dict = cf_model.model.model_config.process_unet_state_dict_for_saving(cf_state_dict)
|
|
|
|
unet_config = ConsistorySDXLUNet2DConditionModel.load_config(config_file)
|
|
Unet = ConsistorySDXLUNet2DConditionModel.from_config(unet_config).to(torch.float16)
|
|
cf_state_dict = convert_ldm_unet_checkpoint(unet_state_dict, Unet.config)
|
|
Unet.load_state_dict(cf_state_dict, strict=False)
|
|
del cf_state_dict,unet_state_dict
|
|
gc_cleanup()
|
|
|
|
AE=convert_cfvae2diffuser(VAE)
|
|
|
|
story_pipeline=ConsistoryExtendAttnSDXLPipeline_wapper.from_pretrained(sdxl_repo,unet=Unet,vae=AE,text_encoder=None,text_encoder_2=None, torch_dtype=float_type, )
|
|
story_pipeline.to(device)
|
|
else:
|
|
raise "need link a sdxl checkpoints"
|
|
|
|
story_pipeline.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2)
|
|
return story_pipeline
|
|
|
|
def Loader_KOLOR(repo_id,clip_vision_path,face_ckpt,):
|
|
|
|
from .kolors.models.modeling_chatglm import ChatGLMModel
|
|
from .kolors.models.tokenization_chatglm import ChatGLMTokenizer
|
|
from transformers import CLIPVisionModelWithProjection,CLIPImageProcessor
|
|
from diffusers import EulerDiscreteScheduler, UNet2DConditionModel, AutoencoderKL
|
|
|
|
logging.info("loader kolor processing...")
|
|
|
|
# text_encoder = ChatGLMModel.from_pretrained(
|
|
# f'{repo_id}/text_encoder', torch_dtype=torch.float16).half()
|
|
vae = AutoencoderKL.from_pretrained(f"{repo_id}/vae", revision=None).half()
|
|
#tokenizer = ChatGLMTokenizer.from_pretrained(f'{repo_id}/text_encoder')
|
|
scheduler = EulerDiscreteScheduler.from_pretrained(f"{repo_id}/scheduler")
|
|
|
|
|
|
from .kolors.pipelines.pipeline_stable_diffusion_xl_chatglm_256_ipadapter_FaceID import \
|
|
StableDiffusionXLPipeline as StableDiffusionXLPipelineFaceID
|
|
unet = UNet2DConditionModel.from_pretrained(f'{repo_id}/unet', revision=None).half()
|
|
|
|
if clip_vision_path:
|
|
|
|
clip_image_encoder = clip_vision_path
|
|
clip_image_processor = CLIPImageProcessor(size=224, crop_size=224)
|
|
use_singel_clip = True
|
|
else:
|
|
clip_image_encoder = CLIPVisionModelWithProjection.from_pretrained(
|
|
f'{repo_id}/clip-vit-large-patch14-336', ignore_mismatched_sizes=True)
|
|
clip_image_encoder.to("cuda")
|
|
clip_image_processor = CLIPImageProcessor(size=336, crop_size=336)
|
|
use_singel_clip = False
|
|
|
|
pipe = StableDiffusionXLPipelineFaceID(
|
|
vae=vae,
|
|
text_encoder=None,
|
|
tokenizer=None,
|
|
unet=unet,
|
|
scheduler=scheduler,
|
|
face_clip_encoder=clip_image_encoder,
|
|
face_clip_processor=clip_image_processor,
|
|
force_zeros_for_empty_prompt=False,
|
|
use_single_clip=use_singel_clip,
|
|
)
|
|
pipe = pipe.to("cuda")
|
|
pipe.load_ip_adapter_faceid_plus(face_ckpt, device="cuda")
|
|
pipe.set_face_fidelity_scale(0.8)
|
|
return pipe
|
|
|
|
def GLM_clip(cur_path,chatglm3_path): #https://github.com/kijai/ComfyUI-KwaiKolorsWrapper/blob/main/nodes.py @kijai
|
|
from .kolors.models.modeling_chatglm import ChatGLMModel, ChatGLMConfig
|
|
from contextlib import nullcontext
|
|
from comfy.utils import ProgressBar, load_torch_file
|
|
try:
|
|
from accelerate import init_empty_weights
|
|
from accelerate.utils import set_module_tensor_to_device
|
|
is_accelerate_available = True
|
|
except:
|
|
pass
|
|
|
|
import comfy.model_management as mm
|
|
import json
|
|
offload_device=mm.unet_offload_device()
|
|
text_encoder_config = os.path.join(cur_path, 'config/glm/text_encoder_config.json')
|
|
with open(text_encoder_config, 'r') as file:
|
|
config = json.load(file)
|
|
|
|
text_encoder_config = ChatGLMConfig(**config)
|
|
with (init_empty_weights() if is_accelerate_available else nullcontext()):
|
|
text_encoder = ChatGLMModel(text_encoder_config)
|
|
if '4bit' in chatglm3_path:
|
|
text_encoder.quantize(4)
|
|
elif '8bit' in chatglm3_path:
|
|
text_encoder.quantize(8)
|
|
|
|
text_encoder_sd = load_torch_file(chatglm3_path)
|
|
if is_accelerate_available:
|
|
for key in text_encoder_sd:
|
|
set_module_tensor_to_device(text_encoder, key, device=offload_device, value=text_encoder_sd[key])
|
|
else:
|
|
text_encoder.load_state_dict()
|
|
|
|
|
|
return text_encoder
|
|
|
|
def glm_single_encode(chatglm3_model, prompt_list,role_list, negative_prompt_, num_images_per_prompt,nc_mode=False): #https://github.com/kijai/ComfyUI-KwaiKolorsWrapper/blob/main/nodes.py @kijai
|
|
import comfy.model_management as mm
|
|
device = mm.get_torch_device()
|
|
offload_device = mm.unet_offload_device()
|
|
mm.unload_all_models()
|
|
mm.soft_empty_cache()
|
|
# Function to randomly select an option from the brackets
|
|
# def choose_random_option(match):
|
|
# options = match.group(1).split('|')
|
|
# return random.choice(options)
|
|
|
|
# # Randomly choose between options in brackets for prompt and negative_prompt
|
|
# prompt = re.sub(r'\{([^{}]*)\}', choose_random_option, prompt)
|
|
# negative_prompt = re.sub(r'\{([^{}]*)\}', choose_random_option, negative_prompt)
|
|
|
|
# if "|" in prompt:
|
|
# prompt = prompt.split("|")
|
|
# negative_prompt = [negative_prompt] * len(prompt) # Replicate negative_prompt to match length of prompt list
|
|
|
|
do_classifier_free_guidance = True
|
|
#print(prompt)
|
|
if nc_mode:
|
|
only_role_emb,only_role_emb_ne = [],[]
|
|
for prompt,negative_prompt in zip(prompt_list,[negative_prompt_]*len(prompt_list)):
|
|
|
|
if prompt is not None and isinstance(prompt, str):
|
|
batch_size = 1
|
|
elif prompt is not None and isinstance(prompt, list):
|
|
batch_size = len(prompt)
|
|
|
|
# Define tokenizers and text encoders
|
|
tokenizer = chatglm3_model['tokenizer']
|
|
text_encoder = chatglm3_model['text_encoder']
|
|
|
|
text_encoder.to(device)
|
|
|
|
text_inputs = tokenizer(
|
|
prompt,
|
|
padding="max_length",
|
|
max_length=256,
|
|
truncation=True,
|
|
return_tensors="pt",
|
|
).to(device)
|
|
|
|
output = text_encoder(
|
|
input_ids=text_inputs['input_ids'] ,
|
|
attention_mask=text_inputs['attention_mask'],
|
|
position_ids=text_inputs['position_ids'],
|
|
output_hidden_states=True)
|
|
|
|
prompt_embeds = output.hidden_states[-2].permute(1, 0, 2).clone() # [batch_size, 77, 4096]
|
|
text_proj = output.hidden_states[-1][-1, :, :].clone() # [batch_size, 4096]
|
|
bs_embed, seq_len, _ = prompt_embeds.shape
|
|
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
|
prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
|
|
|
|
|
|
if do_classifier_free_guidance:
|
|
uncond_tokens = []
|
|
if negative_prompt is None:
|
|
uncond_tokens = [""] * batch_size
|
|
elif prompt is not None and type(prompt) is not type(negative_prompt):
|
|
raise TypeError(
|
|
f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
|
|
f" {type(prompt)}."
|
|
)
|
|
elif isinstance(negative_prompt, str):
|
|
uncond_tokens = [negative_prompt]
|
|
elif batch_size != len(negative_prompt):
|
|
raise ValueError(
|
|
f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
|
|
f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
|
|
" the batch size of `prompt`."
|
|
)
|
|
else:
|
|
uncond_tokens = negative_prompt
|
|
|
|
|
|
max_length = prompt_embeds.shape[1]
|
|
uncond_input = tokenizer(
|
|
uncond_tokens,
|
|
padding="max_length",
|
|
max_length=max_length,
|
|
truncation=True,
|
|
return_tensors="pt",
|
|
).to(device)
|
|
output = text_encoder(
|
|
input_ids=uncond_input['input_ids'] ,
|
|
attention_mask=uncond_input['attention_mask'],
|
|
position_ids=uncond_input['position_ids'],
|
|
output_hidden_states=True)
|
|
negative_prompt_embeds = output.hidden_states[-2].permute(1, 0, 2).clone() # [batch_size, 77, 4096]
|
|
negative_text_proj = output.hidden_states[-1][-1, :, :].clone() # [batch_size, 4096]
|
|
|
|
if do_classifier_free_guidance:
|
|
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
|
|
seq_len = negative_prompt_embeds.shape[1]
|
|
|
|
negative_prompt_embeds = negative_prompt_embeds.to(dtype=text_encoder.dtype, device=device)
|
|
|
|
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
|
negative_prompt_embeds = negative_prompt_embeds.view(
|
|
batch_size * num_images_per_prompt, seq_len, -1
|
|
)
|
|
|
|
bs_embed = text_proj.shape[0]
|
|
text_proj = text_proj.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
negative_text_proj = negative_text_proj.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
math_emb=[prompt_embeds,text_proj]
|
|
only_role_emb.append(math_emb)
|
|
math_emb_n=[negative_prompt_embeds,negative_text_proj]
|
|
only_role_emb_ne.append(math_emb_n)
|
|
|
|
text_encoder.to(offload_device)
|
|
mm.soft_empty_cache()
|
|
gc.collect()
|
|
return only_role_emb,only_role_emb_ne
|
|
else:
|
|
role_emb_dict={}
|
|
for key,prompts in zip(role_list,prompt_list):
|
|
only_role_emb,only_role_emb_ne = [],[]
|
|
for prompt,negative_prompt in zip(prompts,[negative_prompt_]*len(prompts)):
|
|
|
|
if prompt is not None and isinstance(prompt, str):
|
|
batch_size = 1
|
|
elif prompt is not None and isinstance(prompt, list):
|
|
batch_size = len(prompt)
|
|
|
|
# Define tokenizers and text encoders
|
|
tokenizer = chatglm3_model['tokenizer']
|
|
text_encoder = chatglm3_model['text_encoder']
|
|
|
|
text_encoder.to(device)
|
|
|
|
text_inputs = tokenizer(
|
|
prompt,
|
|
padding="max_length",
|
|
max_length=256,
|
|
truncation=True,
|
|
return_tensors="pt",
|
|
).to(device)
|
|
|
|
output = text_encoder(
|
|
input_ids=text_inputs['input_ids'] ,
|
|
attention_mask=text_inputs['attention_mask'],
|
|
position_ids=text_inputs['position_ids'],
|
|
output_hidden_states=True)
|
|
|
|
prompt_embeds = output.hidden_states[-2].permute(1, 0, 2).clone() # [batch_size, 77, 4096]
|
|
text_proj = output.hidden_states[-1][-1, :, :].clone() # [batch_size, 4096]
|
|
bs_embed, seq_len, _ = prompt_embeds.shape
|
|
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
|
prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
|
|
|
|
|
|
if do_classifier_free_guidance:
|
|
uncond_tokens = []
|
|
if negative_prompt is None:
|
|
uncond_tokens = [""] * batch_size
|
|
elif prompt is not None and type(prompt) is not type(negative_prompt):
|
|
raise TypeError(
|
|
f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
|
|
f" {type(prompt)}."
|
|
)
|
|
elif isinstance(negative_prompt, str):
|
|
uncond_tokens = [negative_prompt]
|
|
elif batch_size != len(negative_prompt):
|
|
raise ValueError(
|
|
f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
|
|
f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
|
|
" the batch size of `prompt`."
|
|
)
|
|
else:
|
|
uncond_tokens = negative_prompt
|
|
|
|
|
|
max_length = prompt_embeds.shape[1]
|
|
uncond_input = tokenizer(
|
|
uncond_tokens,
|
|
padding="max_length",
|
|
max_length=max_length,
|
|
truncation=True,
|
|
return_tensors="pt",
|
|
).to(device)
|
|
output = text_encoder(
|
|
input_ids=uncond_input['input_ids'] ,
|
|
attention_mask=uncond_input['attention_mask'],
|
|
position_ids=uncond_input['position_ids'],
|
|
output_hidden_states=True)
|
|
negative_prompt_embeds = output.hidden_states[-2].permute(1, 0, 2).clone() # [batch_size, 77, 4096]
|
|
negative_text_proj = output.hidden_states[-1][-1, :, :].clone() # [batch_size, 4096]
|
|
|
|
if do_classifier_free_guidance:
|
|
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
|
|
seq_len = negative_prompt_embeds.shape[1]
|
|
|
|
negative_prompt_embeds = negative_prompt_embeds.to(dtype=text_encoder.dtype, device=device)
|
|
|
|
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
|
negative_prompt_embeds = negative_prompt_embeds.view(
|
|
batch_size * num_images_per_prompt, seq_len, -1
|
|
)
|
|
|
|
bs_embed = text_proj.shape[0]
|
|
text_proj = text_proj.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
negative_text_proj = negative_text_proj.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
math_emb=[prompt_embeds,text_proj]
|
|
only_role_emb.append(math_emb)
|
|
math_emb_n=[negative_prompt_embeds,negative_text_proj]
|
|
only_role_emb_ne.append(math_emb_n)
|
|
role_emb_dict[key]=only_role_emb
|
|
|
|
text_encoder.to(offload_device)
|
|
mm.soft_empty_cache()
|
|
gc.collect()
|
|
|
|
return role_emb_dict,only_role_emb_ne
|
|
|
|
|
|
def encode_prompt_with_trigger_word(clip,model_,
|
|
prompt: str,
|
|
prompt_2 = None,
|
|
num_id_images= 1,
|
|
device = None,
|
|
prompt_embeds = None,
|
|
pooled_prompt_embeds= None,
|
|
class_tokens_mask = None,
|
|
nc_flag: bool = False,
|
|
trigger_word="img",
|
|
unet_dtype=torch.float16
|
|
):
|
|
device = device
|
|
tokenizer_2=model_.tokenizer_2
|
|
tokenizer=model_.tokenizer
|
|
#text_encoders=clip.encode_from_tokens
|
|
|
|
if prompt is not None and isinstance(prompt, str):
|
|
batch_size = 1
|
|
elif prompt is not None and isinstance(prompt, list):
|
|
batch_size = len(prompt)
|
|
else:
|
|
batch_size = prompt_embeds.shape[0]
|
|
|
|
# Find the token id of the trigger word
|
|
image_token_id = tokenizer_2.convert_tokens_to_ids(trigger_word)
|
|
|
|
# Define tokenizers and text encoders
|
|
tokenizers = [tokenizer, tokenizer_2] if tokenizer is not None else [tokenizer_2]
|
|
# text_encoders = (
|
|
# [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
|
|
# )
|
|
|
|
if prompt_embeds is None:
|
|
prompt_2 = prompt_2 or prompt
|
|
prompt_embeds_list = []
|
|
prompts = [prompt, prompt_2]
|
|
|
|
for prompt, tokenizer,text_en in zip(prompts, tokenizers,[clip.cond_stage_model.clip_l.encode_token_weights,clip.cond_stage_model.clip_g.encode_token_weights]):
|
|
input_ids = tokenizer.encode(prompt) # TODO: batch encode
|
|
clean_index = 0
|
|
clean_input_ids = []
|
|
class_token_index = []
|
|
# Find out the corresponding class word token based on the newly added trigger word token
|
|
for i, token_id in enumerate(input_ids):
|
|
if token_id == image_token_id:
|
|
class_token_index.append(clean_index - 1)
|
|
else:
|
|
clean_input_ids.append(token_id)
|
|
clean_index += 1
|
|
if nc_flag:
|
|
return None, None, None
|
|
if len(class_token_index) > 1:
|
|
raise ValueError(
|
|
f"PhotoMaker currently does not support multiple trigger words in a single prompt.\
|
|
Trigger word: {trigger_word}, Prompt: {prompt}."
|
|
)
|
|
elif len(class_token_index) == 0 and not nc_flag:
|
|
raise ValueError(
|
|
f"PhotoMaker currently does not support multiple trigger words in a single prompt.\
|
|
Trigger word: {trigger_word}, Prompt: {prompt}."
|
|
)
|
|
class_token_index = class_token_index[0]
|
|
|
|
# Expand the class word token and corresponding mask
|
|
class_token = clean_input_ids[class_token_index]
|
|
clean_input_ids = clean_input_ids[:class_token_index] + [class_token] * num_id_images + \
|
|
clean_input_ids[class_token_index+1:]
|
|
|
|
# Truncation or padding
|
|
max_len = tokenizer.model_max_length
|
|
if len(clean_input_ids) > max_len:
|
|
clean_input_ids = clean_input_ids[:max_len]
|
|
else:
|
|
clean_input_ids = clean_input_ids + [tokenizer.pad_token_id] * (
|
|
max_len - len(clean_input_ids)
|
|
)
|
|
|
|
class_tokens_mask = [True if class_token_index <= i < class_token_index+num_id_images else False \
|
|
for i in range(len(clean_input_ids))]
|
|
#print(clean_input_ids)
|
|
#print(max_len)
|
|
|
|
clean_input_ids=clean_input_ids[:77] if len(clean_input_ids) > 77 else clean_input_ids + [0]*(77 - len(clean_input_ids))
|
|
|
|
clean_input_ids_1=[(i,1.0) for i in clean_input_ids]
|
|
clean_input_ids_2=[(i,1.0) for i in clean_input_ids if i==clean_input_ids[0] or i==49407 ]
|
|
clean_input_ids_2=clean_input_ids_2[:77] if len(clean_input_ids_2) > 77 else clean_input_ids_2 + [(0,1.0)]*(77 - len(clean_input_ids_2))
|
|
clean_input_ids_list=[clean_input_ids_1,clean_input_ids_2]
|
|
#print(clean_input_ids)
|
|
#clean_input_ids_ = torch.tensor(clean_input_ids, dtype=torch.long).unsqueeze(0)
|
|
class_tokens_mask = torch.tensor(class_tokens_mask, dtype=torch.bool).unsqueeze(0)
|
|
|
|
|
|
prompt_embeds, pooled_prompt_embeds = text_en(clean_input_ids_list) # print(prompt_embeds.shape,pooled_prompt_embeds.shape)torch.Size([1, 154, 768]) torch.Size([1, 768]) torch.Size([1, 154, 1280]) torch.Size([1, 1280])
|
|
|
|
|
|
prompt_embeds_list.append((prompt_embeds,pooled_prompt_embeds))
|
|
|
|
#prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
|
|
#prompt_embeds=prompt_embeds_list[1]
|
|
pooled_prompt_embeds=prompt_embeds_list[1][1]
|
|
# cut_to = min(prompt_embeds_list[0][0].shape[1], prompt_embeds_list[1][0].shape[1])
|
|
# print(cut_to)
|
|
cut_to=77
|
|
prompt_embeds= torch.cat([prompt_embeds_list[0][0][:,:cut_to], prompt_embeds_list[1][0][:,:cut_to]], dim=-1)
|
|
prompt_embeds = prompt_embeds.to(device=device,dtype=unet_dtype, )
|
|
class_tokens_mask = class_tokens_mask.to(device=device) # TODO: ignoring two-prompt case
|
|
|
|
return prompt_embeds, pooled_prompt_embeds, class_tokens_mask
|
|
|
|
def cf_clip_single(clip,prompt,
|
|
prompt_2,
|
|
device,
|
|
num_images_per_prompt,
|
|
do_classifier_free_guidance,
|
|
negative_prompt,
|
|
negative_prompt_2,
|
|
prompt_embeds,
|
|
negative_prompt_embeds,
|
|
pooled_prompt_embeds,
|
|
negative_pooled_prompt_embeds,):
|
|
|
|
tokens_p = clip.tokenize(prompt)
|
|
output_p = clip.encode_from_tokens(tokens_p, return_dict=True) # {"pooled_output":tensor}
|
|
cond_p = output_p.pop("cond")
|
|
if cond_p.shape[1] / 77 > 1 :
|
|
logging.warning("prompt'tokens length is abvoe 77,split it")
|
|
cond_p = torch.chunk(cond_p, cond_p.shape[1] // 77, dim=1)[0]
|
|
pool_p=output_p.pop("pooled_output")
|
|
|
|
tokens_n = clip.tokenize(negative_prompt)
|
|
output_n = clip.encode_from_tokens(tokens_n, return_dict=True) # {"pooled_output":tensor}
|
|
cond_n = output_n.pop("cond")
|
|
if cond_n.shape[1] / 77 > 1 :
|
|
logging.warning("prompt'tokens length is abvoe 77,split it")
|
|
cond_n = torch.chunk(cond_n, cond_n.shape[1] // 77, dim=1)[0]
|
|
pool_n = output_n.pop("pooled_output")
|
|
|
|
return cond_p,cond_n,pool_p,pool_n # TODO: replace the pooled_prompt_embeds with text only prompt
|
|
|
|
|
|
|
|
def photomaker_clip(clip,model_,prompt_list,negative_prompt,input_id_images,trigger_word="img",num_images_per_prompt=1,nc_flag=False,prompt_2=None,prompt_embeds=None,pooled_prompt_embeds=None,class_tokens_mask=None,prompt_embeds_text_only=None,negative_pooled_prompt_embeds=None):
|
|
id_encoder=model_.id_encoder
|
|
id_image_processor=model_.id_image_processor
|
|
|
|
tokenizer=model_.tokenizer_2
|
|
batch_size=len(prompt_list)
|
|
|
|
num_id_images = len(input_id_images) #双角色时，须确保输入的图片是列表，且跟prompt对应上
|
|
prompt_arr = prompt_list
|
|
negative_prompt_embeds_arr = []
|
|
prompt_embeds_text_only_arr = []
|
|
prompt_embeds_arr = []
|
|
latents_arr = []
|
|
add_time_ids_arr = []
|
|
negative_pooled_prompt_embeds_arr = []
|
|
pooled_prompt_embeds_text_only_arr = []
|
|
pooled_prompt_embeds_arr = []
|
|
for prompt in prompt_arr:
|
|
(
|
|
prompt_embeds,
|
|
pooled_prompt_embeds,
|
|
class_tokens_mask,
|
|
) = encode_prompt_with_trigger_word(clip,model_,
|
|
prompt=prompt,
|
|
prompt_2=prompt_2,
|
|
device=device,
|
|
num_id_images=num_id_images,
|
|
prompt_embeds=prompt_embeds,
|
|
pooled_prompt_embeds=pooled_prompt_embeds,
|
|
class_tokens_mask=class_tokens_mask,
|
|
nc_flag = nc_flag,
|
|
)
|
|
|
|
# 4. Encode input prompt without the trigger word for delayed conditioning
|
|
# encode, remove trigger word token, then decode
|
|
tokens_text_only = tokenizer.encode(prompt, add_special_tokens=False)
|
|
trigger_word_token = tokenizer.convert_tokens_to_ids(trigger_word)
|
|
if not nc_flag:
|
|
tokens_text_only.remove(trigger_word_token)
|
|
prompt_text_only = tokenizer.decode(tokens_text_only, add_special_tokens=False)
|
|
#print(prompt_text_only)
|
|
(
|
|
prompt_embeds_text_only,
|
|
negative_prompt_embeds,
|
|
pooled_prompt_embeds_text_only, # TODO: replace the pooled_prompt_embeds with text only prompt
|
|
negative_pooled_prompt_embeds,
|
|
) = cf_clip_single(clip,
|
|
prompt=prompt_text_only,
|
|
prompt_2=prompt_2,
|
|
device=device,
|
|
num_images_per_prompt=num_images_per_prompt,
|
|
do_classifier_free_guidance=True,
|
|
negative_prompt=negative_prompt,
|
|
negative_prompt_2=None,
|
|
prompt_embeds=prompt_embeds_text_only,
|
|
negative_prompt_embeds=None,
|
|
pooled_prompt_embeds=None,
|
|
negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
|
|
)
|
|
|
|
# 5. Prepare the input ID images
|
|
dtype = next(id_encoder.parameters()).dtype
|
|
if not isinstance(input_id_images[0], torch.Tensor):
|
|
id_pixel_values = id_image_processor(input_id_images, return_tensors="pt").pixel_values
|
|
|
|
id_pixel_values = id_pixel_values.unsqueeze(0).to(device=device, dtype=dtype) # TODO: multiple prompts
|
|
|
|
# duplicate text embeddings for each generation per prompt, using mps friendly method
|
|
if not nc_flag:
|
|
# 6. Get the update text embedding with the stacked ID embedding
|
|
prompt_embeds = id_encoder(id_pixel_values, prompt_embeds, class_tokens_mask)
|
|
|
|
bs_embed, seq_len, _ = prompt_embeds.shape
|
|
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
|
prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
|
|
pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
pooled_prompt_embeds_arr.append(pooled_prompt_embeds)
|
|
pooled_prompt_embeds = None
|
|
|
|
negative_prompt_embeds_arr.append(negative_prompt_embeds)
|
|
negative_prompt_embeds = None
|
|
negative_pooled_prompt_embeds_arr.append(negative_pooled_prompt_embeds)
|
|
negative_pooled_prompt_embeds = None
|
|
prompt_embeds_text_only_arr.append(prompt_embeds_text_only)
|
|
prompt_embeds_text_only = None
|
|
prompt_embeds_arr.append(prompt_embeds)
|
|
prompt_embeds = None
|
|
pooled_prompt_embeds_text_only_arr.append(pooled_prompt_embeds_text_only)
|
|
pooled_prompt_embeds_text_only = None
|
|
|
|
emb_dict={"negative_prompt_embeds_arr":negative_prompt_embeds_arr,
|
|
"negative_prompt_embeds":negative_prompt_embeds,
|
|
"negative_pooled_prompt_embeds_arr":negative_pooled_prompt_embeds_arr,
|
|
"negative_pooled_prompt_embeds":negative_pooled_prompt_embeds,
|
|
"prompt_embeds_text_only_arr":prompt_embeds_text_only_arr,
|
|
"prompt_embeds_text_only":prompt_embeds_text_only,
|
|
"prompt_embeds_arr":prompt_embeds_arr,
|
|
"prompt_embeds":prompt_embeds,
|
|
"pooled_prompt_embeds":pooled_prompt_embeds,
|
|
"pooled_prompt_embeds_text_only_arr":pooled_prompt_embeds_text_only_arr,
|
|
"pooled_prompt_embeds_text_only":pooled_prompt_embeds_text_only,
|
|
"pooled_prompt_embeds_arr":pooled_prompt_embeds_arr,
|
|
"batch_size":batch_size,
|
|
}
|
|
return emb_dict
|
|
|
|
|
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
def encode_prompt_with_trigger_word_v2(clip,model_,
|
|
prompt: str,
|
|
prompt_2 = None,
|
|
num_id_images= 1,
|
|
device = None,
|
|
prompt_embeds = None,
|
|
negative_prompt = None,
|
|
negative_prompt_2 = None,
|
|
negative_prompt_embeds= None,
|
|
pooled_prompt_embeds= None,
|
|
negative_pooled_prompt_embeds= None,
|
|
num_images_per_prompt=1,
|
|
class_tokens_mask = None,
|
|
nc_flag: bool = False,
|
|
trigger_word="img",
|
|
unet_dtype=torch.float16,
|
|
clip_skip=None,
|
|
lora_scale=None,
|
|
do_classifier_free_guidance=True,
|
|
force_zeros_for_empty_prompts=False,
|
|
num_tokens=2,
|
|
|
|
):
|
|
|
|
device = device
|
|
tokenizer_2=model_.tokenizer_2
|
|
tokenizer=model_.tokenizer
|
|
#text_encoders=clip.encode_from_tokens
|
|
|
|
if prompt is not None and isinstance(prompt, str):
|
|
batch_size = 1
|
|
elif prompt is not None and isinstance(prompt, list):
|
|
batch_size = len(prompt)
|
|
else:
|
|
batch_size = prompt_embeds.shape[0]
|
|
|
|
# Find the token id of the trigger word
|
|
image_token_id = tokenizer_2.convert_tokens_to_ids(trigger_word)
|
|
|
|
# Define tokenizers and text encoders
|
|
tokenizers = [tokenizer, tokenizer_2] if tokenizer is not None else [tokenizer_2]
|
|
# text_encoders = (
|
|
# [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
|
|
# )
|
|
|
|
if prompt_embeds is None:
|
|
prompt_2 = prompt_2 or prompt
|
|
prompt_embeds_list = []
|
|
prompts = [prompt, prompt_2]
|
|
|
|
for prompt, tokenizer,text_en in zip(prompts, tokenizers,[clip.cond_stage_model.clip_l.encode_token_weights,clip.cond_stage_model.clip_g.encode_token_weights]):
|
|
# if isinstance(self, TextualInversionLoaderMixin):
|
|
# prompt = self.maybe_convert_prompt(prompt, tokenizer)
|
|
|
|
input_ids = tokenizer.encode(prompt) # TODO: batch encode
|
|
clean_index = 0
|
|
clean_input_ids = []
|
|
class_token_index = []
|
|
# Find out the corresponding class word token based on the newly added trigger word token
|
|
for i, token_id in enumerate(input_ids):
|
|
if token_id == image_token_id:
|
|
class_token_index.append(clean_index - 1)
|
|
else:
|
|
clean_input_ids.append(token_id)
|
|
clean_index += 1
|
|
if nc_flag:
|
|
return None, None, None,None, None
|
|
if len(class_token_index) > 1:
|
|
raise ValueError(
|
|
f"PhotoMaker currently does not support multiple trigger words in a single prompt.\
|
|
Trigger word: {trigger_word}, Prompt: {prompt}."
|
|
)
|
|
elif len(class_token_index) == 0 and not nc_flag:
|
|
raise ValueError(
|
|
f"PhotoMaker currently does not support multiple trigger words in a single prompt.\
|
|
Trigger word: {trigger_word}, Prompt: {prompt}."
|
|
)
|
|
class_token_index = class_token_index[0]
|
|
|
|
# Expand the class word token and corresponding mask
|
|
class_token = clean_input_ids[class_token_index]
|
|
clean_input_ids = clean_input_ids[:class_token_index] + [class_token] * num_id_images * num_tokens + clean_input_ids[class_token_index + 1:]
|
|
# Truncation or padding
|
|
max_len = tokenizer.model_max_length
|
|
if len(clean_input_ids) > max_len:
|
|
clean_input_ids = clean_input_ids[:max_len]
|
|
else:
|
|
clean_input_ids = clean_input_ids + [tokenizer.pad_token_id] * (
|
|
max_len - len(clean_input_ids)
|
|
)
|
|
|
|
#class_tokens_mask = [True if class_token_index <= i < class_token_index+num_id_images else False for i in range(len(clean_input_ids))]
|
|
class_tokens_mask = [True if class_token_index <= i < class_token_index + (num_id_images * num_tokens) else False for i in range(len(clean_input_ids))]
|
|
#print(clean_input_ids)
|
|
#print(max_len)
|
|
|
|
clean_input_ids=clean_input_ids[:77] if len(clean_input_ids) > 77 else clean_input_ids + [0]*(77 - len(clean_input_ids))
|
|
|
|
clean_input_ids_1=[(i,1.0) for i in clean_input_ids]
|
|
clean_input_ids_2=[(i,1.0) for i in clean_input_ids if i==clean_input_ids[0] or i==49407 ]
|
|
clean_input_ids_2=clean_input_ids_2[:77] if len(clean_input_ids_2) > 77 else clean_input_ids_2 + [(0,1.0)]*(77 - len(clean_input_ids_2))
|
|
clean_input_ids_list=[clean_input_ids_1,clean_input_ids_2]
|
|
#print(clean_input_ids)
|
|
#clean_input_ids_ = torch.tensor(clean_input_ids, dtype=torch.long).unsqueeze(0)
|
|
class_tokens_mask = torch.tensor(class_tokens_mask, dtype=torch.bool).unsqueeze(0)
|
|
|
|
|
|
prompt_embeds, pooled_prompt_embeds = text_en(clean_input_ids_list) # print(prompt_embeds.shape,pooled_prompt_embeds.shape)torch.Size([1, 154, 768]) torch.Size([1, 768]) torch.Size([1, 154, 1280]) torch.Size([1, 1280])
|
|
|
|
|
|
prompt_embeds_list.append((prompt_embeds,pooled_prompt_embeds))
|
|
|
|
#prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)
|
|
#prompt_embeds=prompt_embeds_list[1]
|
|
pooled_prompt_embeds=prompt_embeds_list[1][1]
|
|
# cut_to = min(prompt_embeds_list[0][0].shape[1], prompt_embeds_list[1][0].shape[1])
|
|
# print(cut_to)
|
|
cut_to=77
|
|
prompt_embeds= torch.cat([prompt_embeds_list[0][0][:,:cut_to], prompt_embeds_list[1][0][:,:cut_to]], dim=-1)
|
|
prompt_embeds = prompt_embeds.to(device=device,dtype=unet_dtype, )
|
|
class_tokens_mask = class_tokens_mask.to(device=device) # TODO: ignoring two-prompt case
|
|
# get unconditional embeddings for classifier free guidance
|
|
zero_out_negative_prompt = negative_prompt is None and force_zeros_for_empty_prompts
|
|
if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
|
|
negative_prompt_embeds = torch.zeros_like(prompt_embeds)
|
|
negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
|
|
elif do_classifier_free_guidance and negative_prompt_embeds is None:
|
|
negative_prompt = negative_prompt or ""
|
|
negative_prompt_2 = negative_prompt_2 or negative_prompt
|
|
|
|
# normalize str to list
|
|
#negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
|
|
negative_prompt = str(negative_prompt) if isinstance(negative_prompt, list) else negative_prompt
|
|
negative_prompt_2 = (
|
|
batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
|
|
)
|
|
|
|
uncond_tokens: List[str]
|
|
if prompt is not None and type(prompt) is not type(negative_prompt):
|
|
raise TypeError(
|
|
f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
|
|
f" {type(prompt)}."
|
|
)
|
|
# elif batch_size != len(negative_prompt):
|
|
# raise ValueError(
|
|
# f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
|
|
# f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
|
|
# " the batch size of `prompt`."
|
|
# )
|
|
else:
|
|
uncond_tokens = [negative_prompt, negative_prompt_2]
|
|
# ng 没用上
|
|
negative_prompt_embeds_list = []
|
|
for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, [clip.cond_stage_model.clip_l.encode_token_weights,clip.cond_stage_model.clip_g.encode_token_weights]):
|
|
# if isinstance(self, TextualInversionLoaderMixin):
|
|
# negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)
|
|
|
|
max_length = prompt_embeds.shape[1]
|
|
uncond_input=tokenizer.encode(negative_prompt)
|
|
# uncond_input = tokenizer(
|
|
# negative_prompt,
|
|
# padding="max_length",
|
|
# max_length=max_length,
|
|
# truncation=True,
|
|
# return_tensors="pt",
|
|
# )
|
|
# negative_prompt_embeds = text_encoder(
|
|
# uncond_input.input_ids.to(device),
|
|
# output_hidden_states=True,
|
|
# )
|
|
clean_input_ids_n=uncond_input[:77] if len(uncond_input) > 77 else uncond_input + [0]*(77 - len(uncond_input))
|
|
|
|
clean_input_ids_1_=[(i,1.0) for i in clean_input_ids_n]
|
|
clean_input_ids_2_=[(i,1.0) for i in clean_input_ids_n if i==clean_input_ids_n[0] or i==49407 ]
|
|
clean_input_ids_2_=clean_input_ids_2_[:77] if len(clean_input_ids_2_) > 77 else clean_input_ids_2_ + [(0,1.0)]*(77 - len(clean_input_ids_2_))
|
|
clean_input_ids_list_n=[clean_input_ids_1_,clean_input_ids_2_]
|
|
prompt_embeds_n, pooled_prompt_embeds_n = text_en(clean_input_ids_list_n)
|
|
negative_prompt_embeds_list.append((prompt_embeds_n,pooled_prompt_embeds_n))
|
|
|
|
|
|
|
|
# We are only ALWAYS interested in the pooled output of the final text encoder
|
|
# negative_pooled_prompt_embeds = negative_prompt_embeds[0]
|
|
# negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
|
|
|
|
# negative_prompt_embeds_list.append(negative_prompt_embeds)
|
|
|
|
negative_pooled_prompt_embeds=negative_prompt_embeds_list[1][1]
|
|
#negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)
|
|
cut_to=77
|
|
negative_prompt_embeds= torch.cat([negative_prompt_embeds_list[0][0][:,:cut_to], negative_prompt_embeds_list[1][0][:,:cut_to]], dim=-1)
|
|
negative_prompt_embeds = negative_prompt_embeds.to(device=device,dtype=unet_dtype, )
|
|
|
|
|
|
bs_embed, seq_len, _ = prompt_embeds.shape
|
|
|
|
if do_classifier_free_guidance:
|
|
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
|
|
seq_len = negative_prompt_embeds.shape[1]
|
|
|
|
|
|
negative_prompt_embeds = negative_prompt_embeds.to(dtype=unet_dtype, device=device)
|
|
|
|
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
|
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
|
|
|
|
pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
if do_classifier_free_guidance:
|
|
negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
|
|
|
|
return prompt_embeds, negative_prompt_embeds,pooled_prompt_embeds, negative_pooled_prompt_embeds, class_tokens_mask
|
|
|
|
|
|
def photomaker_clip_v2(clip,model_,prompt_list,negative_prompt,input_id_images,id_embeds,trigger_word="img",num_images_per_prompt=1,nc_flag=False,prompt_2=None,prompt_embeds=None,pooled_prompt_embeds=None,class_tokens_mask=None,prompt_embeds_text_only=None,negative_pooled_prompt_embeds=None):
|
|
id_encoder=model_.id_encoder
|
|
id_image_processor=model_.id_image_processor
|
|
|
|
tokenizer=model_.tokenizer_2
|
|
batch_size=len(prompt_list)
|
|
|
|
num_id_images = len(input_id_images) #双角色时，须确保输入的图片是列表，且跟prompt对应上
|
|
prompt_arr = prompt_list
|
|
negative_prompt_embeds_arr = []
|
|
prompt_embeds_text_only_arr = []
|
|
prompt_embeds_arr = []
|
|
latents_arr = []
|
|
add_time_ids_arr = []
|
|
negative_pooled_prompt_embeds_arr = []
|
|
pooled_prompt_embeds_text_only_arr = []
|
|
pooled_prompt_embeds_arr = []
|
|
for prompt in prompt_arr:
|
|
(
|
|
prompt_embeds,
|
|
_,
|
|
pooled_prompt_embeds,
|
|
_,
|
|
class_tokens_mask
|
|
) = encode_prompt_with_trigger_word_v2(clip,model_,
|
|
prompt=prompt,
|
|
prompt_2=prompt_2,
|
|
device=device,
|
|
num_id_images=num_id_images,
|
|
class_tokens_mask=class_tokens_mask,
|
|
num_images_per_prompt=num_images_per_prompt,
|
|
do_classifier_free_guidance=True,
|
|
negative_prompt=negative_prompt,
|
|
negative_prompt_2=None,
|
|
prompt_embeds=None,
|
|
negative_prompt_embeds=None,
|
|
pooled_prompt_embeds=None,
|
|
negative_pooled_prompt_embeds=None,
|
|
nc_flag=nc_flag,
|
|
)
|
|
|
|
# 4. Encode input prompt without the trigger word for delayed conditioning
|
|
# encode, remove trigger word token, then decode
|
|
tokens_text_only = tokenizer.encode(prompt, add_special_tokens=False)
|
|
trigger_word_token = tokenizer.convert_tokens_to_ids(trigger_word)
|
|
if not nc_flag:
|
|
tokens_text_only.remove(trigger_word_token)
|
|
prompt_text_only = tokenizer.decode(tokens_text_only, add_special_tokens=False)
|
|
#print(prompt_text_only)
|
|
(
|
|
prompt_embeds_text_only,
|
|
negative_prompt_embeds,
|
|
pooled_prompt_embeds_text_only, # TODO: replace the pooled_prompt_embeds with text only prompt
|
|
negative_pooled_prompt_embeds,
|
|
) = cf_clip_single(clip,
|
|
prompt=prompt_text_only,
|
|
prompt_2=prompt_2,
|
|
device=device,
|
|
num_images_per_prompt=num_images_per_prompt,
|
|
do_classifier_free_guidance=True,
|
|
negative_prompt=negative_prompt,
|
|
negative_prompt_2=None,
|
|
prompt_embeds=prompt_embeds_text_only,
|
|
negative_prompt_embeds=None,
|
|
pooled_prompt_embeds=None,
|
|
negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
|
|
)
|
|
|
|
# 5. Prepare the input ID images
|
|
dtype = next(id_encoder.parameters()).dtype
|
|
if not isinstance(input_id_images[0], torch.Tensor):
|
|
id_pixel_values = id_image_processor(input_id_images, return_tensors="pt").pixel_values
|
|
|
|
id_pixel_values = id_pixel_values.unsqueeze(0).to(device=device, dtype=dtype) # TODO: multiple prompts
|
|
|
|
# duplicate text embeddings for each generation per prompt, using mps friendly method
|
|
if not nc_flag:
|
|
if id_embeds is not None:
|
|
id_embeds = id_embeds.unsqueeze(0).to(device=device, dtype=dtype)
|
|
|
|
prompt_embeds = id_encoder(id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds) #torch.Size([1, 1, 3, 224, 224]) torch.Size([1, 77, 2048]) torch.Size([1, 77]) torch.Size([1, 512])
|
|
else:
|
|
prompt_embeds = id_encoder(id_pixel_values, prompt_embeds, class_tokens_mask)
|
|
# 6. Get the update text embedding with the stacked ID embedding
|
|
|
|
bs_embed, seq_len, _ = prompt_embeds.shape
|
|
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
|
prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
|
|
pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
|
|
bs_embed * num_images_per_prompt, -1
|
|
)
|
|
pooled_prompt_embeds_arr.append(pooled_prompt_embeds)
|
|
pooled_prompt_embeds = None
|
|
|
|
negative_prompt_embeds_arr.append(negative_prompt_embeds)
|
|
negative_prompt_embeds = None
|
|
negative_pooled_prompt_embeds_arr.append(negative_pooled_prompt_embeds)
|
|
negative_pooled_prompt_embeds = None
|
|
prompt_embeds_text_only_arr.append(prompt_embeds_text_only)
|
|
prompt_embeds_text_only = None
|
|
prompt_embeds_arr.append(prompt_embeds)
|
|
prompt_embeds = None
|
|
pooled_prompt_embeds_text_only_arr.append(pooled_prompt_embeds_text_only)
|
|
pooled_prompt_embeds_text_only = None
|
|
|
|
emb_dict={"negative_prompt_embeds_arr":negative_prompt_embeds_arr,
|
|
"negative_prompt_embeds":negative_prompt_embeds,
|
|
"negative_pooled_prompt_embeds_arr":negative_pooled_prompt_embeds_arr,
|
|
"negative_pooled_prompt_embeds":negative_pooled_prompt_embeds,
|
|
"prompt_embeds_text_only_arr":prompt_embeds_text_only_arr,
|
|
"prompt_embeds_text_only":prompt_embeds_text_only,
|
|
"prompt_embeds_arr":prompt_embeds_arr,
|
|
"prompt_embeds":prompt_embeds,
|
|
"pooled_prompt_embeds":pooled_prompt_embeds,
|
|
"pooled_prompt_embeds_text_only_arr":pooled_prompt_embeds_text_only_arr,
|
|
"pooled_prompt_embeds_text_only":pooled_prompt_embeds_text_only,
|
|
"pooled_prompt_embeds_arr":pooled_prompt_embeds_arr,
|
|
"batch_size":batch_size,
|
|
}
|
|
return emb_dict
|
|
|
|
def Loader_UNO(extra_info,offload,quantize_mode,save_quantezed,lora_rank):
|
|
from .UNO.uno.flux.pipeline import UNOPipeline
|
|
from accelerate import Accelerator
|
|
accelerator = Accelerator()
|
|
device= accelerator.device
|
|
if extra_info is not None:
|
|
model_path = extra_info["unet_path"]
|
|
lora_list=[i for i in [extra_info.get("lora1_path"),extra_info.get("lora2_path")] if i is not None]
|
|
if lora_list:
|
|
lora_ckpt_path=lora_list[0]
|
|
if "dit_lora" not in lora_ckpt_path.lower():
|
|
raise ValueError("lora_ckpt_path must be a dit_lora checkpoint")
|
|
only_lora=True
|
|
else:
|
|
only_lora=False
|
|
|
|
|
|
use_fp8=True if quantize_mode=="fp8" else False
|
|
|
|
if "schnell" in model_path:
|
|
model_type="flux-schnell"
|
|
else:
|
|
model_type="flux-dev"
|
|
|
|
|
|
if "schnell" in model_path:
|
|
model_type="flux-schnell"
|
|
else:
|
|
model_type="flux-dev"
|
|
|
|
if only_lora :
|
|
from .UNO.uno.flux.util import load_flow_model_only_lora
|
|
model_ = load_flow_model_only_lora(
|
|
model_type,model_path,lora_ckpt_path,use_fp8, device="cpu" if offload else device,save_quantezed=save_quantezed, lora_rank=lora_rank)
|
|
else:
|
|
from .UNO.uno.flux.util import load_flow_model
|
|
model_ = load_flow_model(model_type,model_path,use_fp8, device="cpu" if offload else device,save_quantezed=save_quantezed)
|
|
model = UNOPipeline(
|
|
model_,
|
|
device,
|
|
offload,
|
|
use_fp8,
|
|
)
|
|
else:
|
|
raise ValueError("must use EasyFunction_Lite node")
|
|
return model
|
|
|
|
def load_pipeline_realcustom(cf_model,realcustom_checkpoint):
|
|
from .RealCustom.models.unet_2d_condition_custom import UNet2DConditionModel as UNet2DConditionModelDiffusers
|
|
if "shallow" in realcustom_checkpoint.lower():
|
|
unet_config_path=os.path.join(cur_path, "RealCustom/configs/realcustom_sigdino_highres_shallow.json")
|
|
else:
|
|
unet_config_path=os.path.join(cur_path, "RealCustom/configs/realcustom_sigdino_highres.json")
|
|
with open(unet_config_path, 'r') as f:
|
|
unet_config = json.load(f)
|
|
|
|
# Settings for image encoder
|
|
vision_model_config = unet_config.pop("vision_model_config", None)
|
|
vision_model_config_ar = vision_model_config.pop("vision_model_config", None)
|
|
|
|
|
|
|
|
unet_type = unet_config.pop("type", None)
|
|
unet_model = UNet2DConditionModelDiffusers(**unet_config)
|
|
unet_model.eval()
|
|
|
|
#cf_state_dict=torch.load(unet_checkpoint,weights_only=False, map_location="cpu")
|
|
cf_state_dict = cf_model.model.diffusion_model.state_dict()
|
|
unet_state_dict = cf_model.model.model_config.process_unet_state_dict_for_saving(cf_state_dict)
|
|
|
|
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_unet_checkpoint
|
|
cf_state_dict = convert_ldm_unet_checkpoint(unet_state_dict, unet_model.config)
|
|
unet_model.load_state_dict(cf_state_dict, strict=False)
|
|
|
|
realcustom_DICT=torch.load(realcustom_checkpoint,weights_only=False, map_location="cpu")
|
|
unet_model.load_state_dict(realcustom_DICT, strict=False)
|
|
unet_model = torch.compile(unet_model, disable=True)# CHECK THIS
|
|
|
|
del cf_state_dict,realcustom_DICT
|
|
gc.collect()
|
|
torch.cuda.empty_cache()
|
|
print("loading unet base model finished.")
|
|
|
|
return unet_model,vision_model_config_ar,unet_type
|
|
|
|
|
|
|
|
|
|
def load_clip_clipvsion(clip_paths,token_paths,dino_path,siglip_path,vision_model_config):
|
|
from .RealCustom.models.text import TextModel
|
|
from transformers import CLIPTextModel,CLIPTextConfig
|
|
from .RealCustom.utils import instantiate_from_config
|
|
|
|
# text_encoder=CLIPTextModel.from_pretrained(clip_paths[0], local_files_only=True)
|
|
# text_encoder_2=CLIPTextModel.from_pretrained(clip_paths[1], local_files_only=True)
|
|
|
|
from safetensors.torch import load_file
|
|
text_encoder_config=CLIPTextConfig.from_pretrained(token_paths[0], local_files_only=True)
|
|
text_encoder = CLIPTextModel(text_encoder_config)
|
|
text_encoder_sd_1 = load_file(clip_paths[0])
|
|
text_encoder.load_state_dict(text_encoder_sd_1, strict=False)
|
|
|
|
|
|
text_encoder_config_2=CLIPTextConfig.from_pretrained(token_paths[1], local_files_only=True)
|
|
text_encoder_2 = CLIPTextModel(text_encoder_config_2)
|
|
text_encoder_sd_2 = load_file(clip_paths[1])
|
|
text_encoder_2.load_state_dict(text_encoder_sd_2, strict=False)
|
|
del text_encoder_sd_2,text_encoder_sd_1
|
|
gc.collect()
|
|
|
|
|
|
text_model = TextModel([text_encoder,text_encoder_2],token_paths,["penultimate_nonorm"])
|
|
text_model.eval()
|
|
|
|
siglip_path = os.path.normpath(siglip_path).replace("\\", "/")
|
|
dino_path = os.path.normpath(dino_path).replace("\\", "/")
|
|
|
|
vision_model_config["params"]["siglip_path"] = siglip_path
|
|
vision_model_config["params"]["dino_path"] = dino_path
|
|
|
|
vision_model = instantiate_from_config(vision_model_config)
|
|
vision_model = vision_model.eval().to(device)
|
|
|
|
print("loading image model and text_model finished.")
|
|
return text_model,vision_model
|
|
|
|
|
|
|
|
|
|
def realcustom_clip_emb(text_model,vision_model,vae_config,vae_downsample_factor,positive_prompt,negative_prompt,positive_image,target_phrase,width,height,device,samples_per_prompt,guidance_weight=7.5):
|
|
from .RealCustom.inference.inference_utils import find_phrase_positions_in_text
|
|
import torchvision
|
|
with torch.no_grad():
|
|
image_metadata_validate = torch.tensor(
|
|
data=[
|
|
width, # original_height
|
|
height, # original_width
|
|
0, # coordinate top
|
|
0, # coordinate left
|
|
width, # target_height
|
|
height, # target_width
|
|
],
|
|
device=device,
|
|
dtype=torch.float32
|
|
).view(1, -1).repeat(samples_per_prompt, 1)
|
|
|
|
if guidance_weight != 1:
|
|
text_negative_output = text_model(negative_prompt)
|
|
# Compute target phrases
|
|
target_token = torch.zeros(1, 77).to(device)
|
|
|
|
positions = find_phrase_positions_in_text(positive_prompt, target_phrase)
|
|
for position in positions:
|
|
prompt_before = positive_prompt[:position] # NOTE We do not need -1 here because the SDXL text encoder does not encode the trailing space.
|
|
prompt_include = positive_prompt[:position+len(target_phrase)]
|
|
#print("prompt before: ", prompt_before, ", prompt_include: ", prompt_include)
|
|
prompt_before_length = text_model.get_vaild_token_length(prompt_before) + 1
|
|
prompt_include_length = text_model.get_vaild_token_length(prompt_include) + 1
|
|
# print("prompt_before_length: ", prompt_before_length, ", prompt_include_length: ", prompt_include_length)
|
|
target_token[:, prompt_before_length:prompt_include_length] = 1
|
|
|
|
# Text used for progress bar
|
|
pbar_text = positive_prompt[:40]
|
|
|
|
# Compute text embeddings
|
|
text_positive_output = text_model(positive_prompt)
|
|
text_positive_embeddings = text_positive_output.embeddings.repeat_interleave(samples_per_prompt, dim=0)
|
|
text_positive_pooled = text_positive_output.pooled[-1].repeat_interleave(samples_per_prompt, dim=0)
|
|
if guidance_weight != 1:
|
|
text_negative_embeddings = text_negative_output.embeddings.repeat_interleave(samples_per_prompt, dim=0)
|
|
text_negative_pooled = text_negative_output.pooled[-1].repeat_interleave(samples_per_prompt, dim=0)
|
|
|
|
# Compute image embeddings
|
|
# positive_image = Image.open(positive_promt_image_path).convert("RGB")
|
|
positive_image = torchvision.transforms.ToTensor()(positive_image)
|
|
|
|
positive_image = positive_image.unsqueeze(0).repeat_interleave(samples_per_prompt, dim=0)
|
|
positive_image = torch.nn.functional.interpolate(
|
|
positive_image,
|
|
size=(768, 768),
|
|
mode="bilinear",
|
|
align_corners=False
|
|
)
|
|
negative_image = torch.zeros_like(positive_image)
|
|
#print("positive_image:",positive_image.size(), negative_image.size()) #torch.Size([1, 3, 768, 768]) torch.Size([1, 3, 768, 768])
|
|
positive_image = positive_image.to(device)
|
|
negative_image = negative_image.to(device)
|
|
|
|
positive_image_dict = {"image_ref": positive_image}
|
|
positive_image_output = vision_model(positive_image_dict, device=device)
|
|
#positive_image_output = daul_encoder(vision_model,positive_image_dict)
|
|
|
|
negative_image_dict = {"image_ref": negative_image}
|
|
negative_image_output = vision_model(negative_image_dict, device=device)
|
|
#negative_image_output=daul_encoder(vision_model,negative_image_dict)
|
|
# Initialize latent with input latent + noise (i2i) / pure noise (t2i)
|
|
# latent = torch.randn(
|
|
# size=[
|
|
# samples_per_prompt,
|
|
# vae_config["latent_channels"],
|
|
# height // vae_downsample_factor,
|
|
# width // vae_downsample_factor
|
|
# ],
|
|
# device=device,
|
|
# generator=torch.Generator(device).manual_seed(seed))
|
|
target_h = (height // vae_downsample_factor) // 2
|
|
target_w = (width // vae_downsample_factor) // 2
|
|
|
|
emb_dict={
|
|
"text_positive_embeddings":text_positive_embeddings.to(device),
|
|
"text_positive_pooled":text_positive_pooled.to(device),
|
|
"image_metadata_validate":image_metadata_validate.to(device),
|
|
"positive_image_output":positive_image_output,
|
|
"text_negative_embeddings":text_negative_embeddings.to(device),
|
|
"text_negative_pooled":text_negative_pooled.to(device),
|
|
"negative_image_output":negative_image_output,
|
|
}
|
|
|
|
latent_dict={
|
|
"vae_config":vae_config,
|
|
"vae_downsample_factor":vae_downsample_factor,
|
|
"target_h":target_h,
|
|
"target_w":target_w,
|
|
"pbar_text":pbar_text,
|
|
"guidance_weight":guidance_weight,
|
|
"target_token":target_token,
|
|
|
|
}
|
|
|
|
return emb_dict,latent_dict
|
|
|
|
|
|
def realcustom_infer(unet_model,sample_steps,mask_reused_step,emb_dict,latent_dict,mask_scope,mask_strategy,guidance_weight,height,width,seed,
|
|
device,schedule_type="squared_linear",schedule_shift_snr=1.0,scheduler_type="ddim",unet_prediction="epsilon"):
|
|
from .RealCustom.schedulers.ddim import DDIMScheduler
|
|
from .RealCustom.schedulers.dpm_s import DPMSolverSingleStepScheduler
|
|
from .RealCustom.schedulers.utils import get_betas
|
|
from .RealCustom.inference.mask_generation import mask_generation
|
|
from .RealCustom.inference.inference_utils import classifier_free_guidance_image_prompt_cascade
|
|
from torchvision.transforms.functional import to_pil_image
|
|
from tqdm import tqdm
|
|
# Initialize ddim scheduler
|
|
ddim_train_steps = 1000
|
|
ddim_betas = get_betas(name=schedule_type, num_steps=ddim_train_steps, shift_snr=schedule_shift_snr, terminal_pure_noise=False)
|
|
scheduler_class = DPMSolverSingleStepScheduler if scheduler_type == 'dpm' else DDIMScheduler
|
|
scheduler = scheduler_class(betas=ddim_betas, num_train_timesteps=ddim_train_steps, num_inference_timesteps=sample_steps, device=device)
|
|
infer_timesteps = scheduler.timesteps
|
|
#print(infer_timesteps)
|
|
# Real Reverse diffusion process.
|
|
text2image_crossmap_2d_all_timesteps_list = []
|
|
current_step = 0
|
|
pbar_text = latent_dict.get("pbar_text")
|
|
#guidance_weight = latent_dict.get("guidance_weight")
|
|
unet_model.to(device)
|
|
latent=latent_dict.get("latent")
|
|
latent = torch.randn(
|
|
size=[
|
|
1,
|
|
latent_dict.get("vae_config")["latent_channels"],
|
|
height // latent_dict.get("vae_downsample_factor"),
|
|
width // latent_dict.get("vae_downsample_factor")
|
|
],
|
|
device=device,
|
|
generator=torch.Generator(device).manual_seed(seed))
|
|
|
|
|
|
with torch.no_grad():
|
|
for timestep in tqdm(iterable=infer_timesteps, desc=f"[{pbar_text}]", dynamic_ncols=True):
|
|
|
|
if current_step < mask_reused_step:
|
|
pred_cond, pred_cond_dict = unet_model(
|
|
sample=latent,
|
|
timestep=timestep,
|
|
encoder_hidden_states=emb_dict.get("text_positive_embeddings"),
|
|
encoder_attention_mask=None,
|
|
added_cond_kwargs=dict(
|
|
text_embeds=emb_dict.get("text_positive_pooled"),
|
|
time_ids=emb_dict.get("image_metadata_validate")
|
|
),
|
|
vision_input_dict=None,
|
|
vision_guided_mask=None,
|
|
return_as_origin=False,
|
|
return_text2image_mask=True,
|
|
)
|
|
|
|
crossmap_2d_avg = mask_generation(
|
|
crossmap_2d_list=pred_cond_dict.get("text2image_crossmap_2d",[]), selfmap_2d_list=pred_cond_dict.get("self_attention_map", []),
|
|
target_token=latent_dict.get("target_token"), mask_scope=mask_scope,
|
|
mask_target_h=latent_dict.get("target_h"), mask_target_w=latent_dict.get("target_w"), mask_mode=mask_strategy,
|
|
)
|
|
else:
|
|
# using previous step's mask
|
|
crossmap_2d_avg = text2image_crossmap_2d_all_timesteps_list[-1].squeeze(1)
|
|
if crossmap_2d_avg.dim() == 5: # Means that each layer uses a separate mask weight.
|
|
text2image_crossmap_2d_all_timesteps_list.append(crossmap_2d_avg.mean(dim=2).unsqueeze(1))
|
|
else:
|
|
text2image_crossmap_2d_all_timesteps_list.append(crossmap_2d_avg.unsqueeze(1))
|
|
|
|
pred_cond, pred_cond_dict = unet_model(
|
|
sample=latent,
|
|
timestep=timestep,
|
|
encoder_hidden_states=emb_dict.get("text_positive_embeddings"),
|
|
encoder_attention_mask=None,
|
|
added_cond_kwargs=dict(
|
|
text_embeds=emb_dict.get("text_positive_pooled"),
|
|
time_ids=emb_dict.get("image_metadata_validate")
|
|
),
|
|
vision_input_dict=emb_dict.get("positive_image_output"),
|
|
vision_guided_mask=crossmap_2d_avg,
|
|
return_as_origin=False,
|
|
return_text2image_mask=True,
|
|
multiple_reference_image=False
|
|
)
|
|
|
|
crossmap_2d_avg_neg = crossmap_2d_avg.mean(dim=1, keepdim=True)
|
|
pred_negative, pred_negative_dict = unet_model(
|
|
sample=latent,
|
|
timestep=timestep,
|
|
encoder_hidden_states=emb_dict.get("text_negative_embeddings"),
|
|
encoder_attention_mask=None,
|
|
added_cond_kwargs=dict(
|
|
text_embeds=emb_dict.get("text_negative_pooled"),
|
|
time_ids=emb_dict.get("image_metadata_validate")
|
|
),
|
|
vision_input_dict=emb_dict.get("negative_image_output"),
|
|
vision_guided_mask=crossmap_2d_avg,
|
|
return_as_origin=False,
|
|
return_text2image_mask=True,
|
|
multiple_reference_image=False
|
|
)
|
|
|
|
pred = classifier_free_guidance_image_prompt_cascade(
|
|
pred_t_cond=None, pred_ti_cond=pred_cond, pred_uncond=pred_negative,
|
|
guidance_weight_t=guidance_weight, guidance_weight_i=guidance_weight,
|
|
guidance_stdev_rescale_factor=0, cfg_rescale_mode="naive_global_direct"
|
|
)
|
|
step = scheduler.step(
|
|
model_output=pred,
|
|
model_output_type=unet_prediction,
|
|
timestep=timestep,
|
|
sample=latent,
|
|
)
|
|
|
|
latent = step.prev_sample
|
|
|
|
current_step += 1
|
|
|
|
sample=step.pred_original_sample/0.13025
|
|
|
|
return sample
|
|
|
|
|
|
def load_realcustom_vae(vae,device):
|
|
from .RealCustom.models.vae import AutoencoderKL
|
|
from diffusers.pipelines.stable_diffusion.convert_from_ckpt import convert_ldm_vae_checkpoint
|
|
|
|
vae_config_path = os.path.join(cur_path, "local_repo/vae/config.json")
|
|
# Initialize vae model
|
|
with open(vae_config_path, 'r') as vae_config_file:
|
|
vae_config = json.load(vae_config_file)
|
|
vae_downsample_factor = 2 ** (len(vae_config["block_out_channels"]) - 1) # 2 ** 3 = 8
|
|
vae_model = AutoencoderKL(**vae_config)
|
|
vae_model.eval().to(device)
|
|
|
|
vae_state_dict=vae.get_sd()
|
|
#AE = AutoencoderKL.from_config(ae_config).to(device, torch.float16)
|
|
cf_state_dict = convert_ldm_vae_checkpoint(vae_state_dict, vae_model.config)
|
|
vae_model.load_state_dict(cf_state_dict, strict=False)
|
|
del cf_state_dict,vae_state_dict
|
|
|
|
|
|
#vae_decoder = torch.compile(lambda x: vae_model.decode(x / vae_model.scaling_factor).sample.clip(-1, 1), disable=True)
|
|
vae_encoder = torch.compile(lambda x: vae_model.encode(x).latent_dist.mode().mul_(vae_model.scaling_factor), disable=True)
|
|
print("loading vae finished.")
|
|
return vae_encoder,vae_downsample_factor,vae_config
|
|
|
|
|
|
|
|
def load_pipeline_instant_character(extra_info,ip_adapter_path,VAE,quantize_mode):
|
|
from .InstantCharacter.pipeline_wrapper import InstantCharacterFluxPipeline
|
|
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig,FluxTransformer2DModel
|
|
if extra_info.get("svdq_repo") is not None:
|
|
print("use svdq")
|
|
from nunchaku import NunchakuFluxTransformer2dModel
|
|
transformer = NunchakuFluxTransformer2dModel.from_pretrained(extra_info.get("svdq_repo"),offload=True)
|
|
try:
|
|
transformer.set_attention_impl("nunchaku-fp16")
|
|
except:
|
|
pass
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
pipe = InstantCharacterFluxPipeline.from_pretrained(os.path.join(cur_path,"config/FLUX.1-dev"),vae=vae,transformer=transformer,text_encoder=None,text_encoder_2=None, torch_dtype=torch.bfloat16)
|
|
elif extra_info.get("gguf_path") is not None:
|
|
print("use gguf quantization")
|
|
from diffusers import GGUFQuantizationConfig
|
|
transformer = FluxTransformer2DModel.from_single_file(
|
|
extra_info.get("gguf_path") ,
|
|
config=os.path.join(cur_path, "config/FLUX.1-dev/transformer"),
|
|
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
pipe = InstantCharacterFluxPipeline.from_pretrained(os.path.join(cur_path,"config/FLUX.1-dev"),vae=vae,transformer=transformer,text_encoder=None,text_encoder_2=None, torch_dtype=torch.bfloat16)
|
|
elif extra_info.get("unet_path"):
|
|
|
|
if quantize_mode=="fp8":
|
|
transformer = FluxTransformer2DModel.from_single_file(extra_info.get("unet_path"), config=os.path.join(cur_path,"config/FLUX.1-dev/transformer/config.json"),
|
|
torch_dtype=torch.bfloat16)
|
|
else:
|
|
from safetensors.torch import load_file
|
|
t_state_dict=load_file(extra_info.get("unet_path"))
|
|
unet_config = FluxTransformer2DModel.load_config(os.path.join(cur_path,"config/FLUX.1-dev/transformer/config.json"))
|
|
transformer = FluxTransformer2DModel.from_config(unet_config).to(torch.bfloat16)
|
|
transformer.load_state_dict(t_state_dict, strict=False)
|
|
del t_state_dict
|
|
gc_cleanup()
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
pipe = InstantCharacterFluxPipeline.from_pretrained(os.path.join(cur_path,"config/FLUX.1-dev"),transformer=transformer,vae=vae,text_encoder=None,text_encoder_2=None, torch_dtype=torch.bfloat16)
|
|
else:
|
|
print(f"use {quantize_mode} quantization")
|
|
find_flux=[i for i in [extra_info.get("repo1_path") , extra_info.get("repo2_path")] if "flux" in i ]
|
|
if not find_flux:
|
|
raise ValueError("can not find flux repo")
|
|
if quantize_mode=="fp8":
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
find_flux[0],
|
|
subfolder="transformer",
|
|
quantization_config=DiffusersBitsAndBytesConfig(load_in_8bit=True,),
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
|
|
elif quantize_mode=="nf4": #nf4
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
find_flux[0],
|
|
subfolder="transformer",
|
|
quantization_config=DiffusersBitsAndBytesConfig(
|
|
load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
|
|
),
|
|
torch_dtype=torch.bfloat16,)
|
|
else:
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
find_flux[0],
|
|
subfolder="transformer",
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
|
|
pipe = InstantCharacterFluxPipeline.from_pretrained(find_flux[0],transformer=transformer,text_encoder=None,text_encoder_2=None, torch_dtype=torch.bfloat16)
|
|
# if not cf_model.get("use_svdq"):
|
|
# pipe.enable_model_cpu_offload()
|
|
pipe.to(device)
|
|
pipe.init_adapter(
|
|
subject_ipadapter_cfg=dict(subject_ip_adapter_path=ip_adapter_path, nb_token=1024),
|
|
)
|
|
|
|
return pipe
|
|
|
|
def cf_flux_prompt_clip(cf_clip,prompt):
|
|
tokens = cf_clip.tokenize(prompt)
|
|
tokens["t5xxl"] = cf_clip.tokenize(prompt)["t5xxl"]
|
|
prompt_embeds = cf_clip.encode_from_tokens(tokens, return_pooled=True, return_dict=True).pop("cond")
|
|
tokens["l"] = cf_clip.tokenize(prompt)["l"]
|
|
pooled_prompt_embeds = cf_clip.encode_from_tokens(tokens, return_dict=True).pop("pooled_output")
|
|
prompt_embeds=prompt_embeds.to(device,torch.bfloat16)
|
|
pooled_prompt_embeds=pooled_prompt_embeds.to(device,torch.bfloat16)
|
|
text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=torch.bfloat16)
|
|
return prompt_embeds,pooled_prompt_embeds,text_ids
|
|
|
|
|
|
def load_dual_clip(image_encoder_path,image_encoder_2_path,device,dtype):
|
|
from transformers import SiglipVisionModel, SiglipImageProcessor, AutoModel, AutoImageProcessor
|
|
# image encoder
|
|
print(f"=> loading image_encoder_1: {image_encoder_path}")
|
|
image_encoder = SiglipVisionModel.from_pretrained(image_encoder_path)
|
|
image_processor = SiglipImageProcessor.from_pretrained(image_encoder_path)
|
|
image_encoder.eval()
|
|
image_encoder.to(device, dtype=dtype)
|
|
|
|
|
|
# image encoder 2
|
|
print(f"=> loading image_encoder_2: {image_encoder_2_path}")
|
|
image_encoder_2 = AutoModel.from_pretrained(image_encoder_2_path)
|
|
image_processor_2 = AutoImageProcessor.from_pretrained(image_encoder_2_path)
|
|
image_encoder_2.eval()
|
|
image_encoder_2.to(device, dtype=dtype)
|
|
image_processor_2.crop_size = dict(height=384, width=384)
|
|
image_processor_2.size = dict(shortest_edge=384)
|
|
|
|
|
|
return image_encoder,image_processor,image_encoder_2,image_processor_2
|
|
|
|
|
|
|
|
@torch.inference_mode()
|
|
def instant_character_id_clip(subject_image,siglip_image_encoder,siglip_image_processor,dino_image_encoder_2,dino_image_processor_2,device,dtype):
|
|
|
|
|
|
from einops import rearrange
|
|
|
|
def encode_siglip_image_emb(siglip_image, device, dtype):
|
|
siglip_image = siglip_image.to(device, dtype=dtype)
|
|
res = siglip_image_encoder(siglip_image, output_hidden_states=True)
|
|
|
|
siglip_image_embeds = res.last_hidden_state
|
|
|
|
siglip_image_shallow_embeds = torch.cat([res.hidden_states[i] for i in [7, 13, 26]], dim=1)
|
|
|
|
return siglip_image_embeds, siglip_image_shallow_embeds
|
|
|
|
|
|
def encode_dinov2_image_emb(dinov2_image, device, dtype):
|
|
dinov2_image = dinov2_image.to(device, dtype=dtype)
|
|
res = dino_image_encoder_2(dinov2_image, output_hidden_states=True)
|
|
|
|
dinov2_image_embeds = res.last_hidden_state[:, 1:]
|
|
|
|
dinov2_image_shallow_embeds = torch.cat([res.hidden_states[i][:, 1:] for i in [9, 19, 29]], dim=1)
|
|
|
|
return dinov2_image_embeds, dinov2_image_shallow_embeds
|
|
|
|
|
|
|
|
def encode_image_emb(siglip_image, device, dtype):
|
|
object_image_pil = siglip_image
|
|
object_image_pil_low_res = [object_image_pil.resize((384, 384))]
|
|
object_image_pil_high_res = object_image_pil.resize((768, 768))
|
|
object_image_pil_high_res = [
|
|
object_image_pil_high_res.crop((0, 0, 384, 384)),
|
|
object_image_pil_high_res.crop((384, 0, 768, 384)),
|
|
object_image_pil_high_res.crop((0, 384, 384, 768)),
|
|
object_image_pil_high_res.crop((384, 384, 768, 768)),
|
|
]
|
|
nb_split_image = len(object_image_pil_high_res)
|
|
|
|
siglip_image_embeds = encode_siglip_image_emb(
|
|
siglip_image_processor(images=object_image_pil_low_res, return_tensors="pt").pixel_values,
|
|
device,
|
|
dtype
|
|
)
|
|
dinov2_image_embeds = encode_dinov2_image_emb(
|
|
dino_image_processor_2(images=object_image_pil_low_res, return_tensors="pt").pixel_values,
|
|
device,
|
|
dtype
|
|
)
|
|
|
|
image_embeds_low_res_deep = torch.cat([siglip_image_embeds[0], dinov2_image_embeds[0]], dim=2)
|
|
image_embeds_low_res_shallow = torch.cat([siglip_image_embeds[1], dinov2_image_embeds[1]], dim=2)
|
|
|
|
siglip_image_high_res = siglip_image_processor(images=object_image_pil_high_res, return_tensors="pt").pixel_values
|
|
siglip_image_high_res = siglip_image_high_res[None]
|
|
siglip_image_high_res = rearrange(siglip_image_high_res, 'b n c h w -> (b n) c h w')
|
|
siglip_image_high_res_embeds = encode_siglip_image_emb(siglip_image_high_res, device, dtype)
|
|
siglip_image_high_res_deep = rearrange(siglip_image_high_res_embeds[0], '(b n) l c -> b (n l) c', n=nb_split_image)
|
|
dinov2_image_high_res = dino_image_processor_2(images=object_image_pil_high_res, return_tensors="pt").pixel_values
|
|
dinov2_image_high_res = dinov2_image_high_res[None]
|
|
dinov2_image_high_res = rearrange(dinov2_image_high_res, 'b n c h w -> (b n) c h w')
|
|
dinov2_image_high_res_embeds = encode_dinov2_image_emb(dinov2_image_high_res, device, dtype)
|
|
dinov2_image_high_res_deep = rearrange(dinov2_image_high_res_embeds[0], '(b n) l c -> b (n l) c', n=nb_split_image)
|
|
image_embeds_high_res_deep = torch.cat([siglip_image_high_res_deep, dinov2_image_high_res_deep], dim=2)
|
|
|
|
image_embeds_dict = dict(
|
|
image_embeds_low_res_shallow=image_embeds_low_res_shallow.to(device),
|
|
image_embeds_low_res_deep=image_embeds_low_res_deep.to(device),
|
|
image_embeds_high_res_deep=image_embeds_high_res_deep.to(device),
|
|
)
|
|
|
|
return image_embeds_dict
|
|
|
|
subject_image = subject_image.resize((max(subject_image.size), max(subject_image.size)))
|
|
subject_image_embeds_dict = encode_image_emb(subject_image, device, dtype)
|
|
|
|
return subject_image_embeds_dict
|
|
|
|
def Loader_Dreamo(extra_info,VAE,quantize_mode,dreamo_lora_path,cfg_distill_path,Turbo_path,device,dreamo_version):
|
|
from .DreamO.dreamo.dreamo_pipeline import DreamOPipeline
|
|
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig,FluxTransformer2DModel
|
|
#vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
repo_list=[i for i in [extra_info.get("repo1_path"),extra_info.get("repo2_path")] if i is not None]
|
|
if repo_list:
|
|
flux_repo=[i for i in repo_list if "flux" in i.lower()]
|
|
else:
|
|
raise "you must fill a flux repo"
|
|
if extra_info.get("gguf_path") is not None: #get error
|
|
print("use gguf quantization")
|
|
from diffusers import GGUFQuantizationConfig
|
|
transformer = FluxTransformer2DModel.from_single_file(
|
|
extra_info.get("gguf_path"),
|
|
config=os.path.join(cur_path, "config/FLUX.1-dev/transformer"),
|
|
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
dreamo_pipeline = DreamOPipeline.from_pretrained(flux_repo, transformer=transformer,torch_dtype=torch.bfloat16)
|
|
dreamo_pipeline.load_dreamo_model(device,dreamo_lora_path,cfg_distill_path,Turbo_path, use_turbo=True,dreamo_version=dreamo_version)
|
|
elif extra_info.get("svdq_repo") is not None:#get error
|
|
print("use svdq")
|
|
from nunchaku import NunchakuFluxTransformer2dModel
|
|
transformer = NunchakuFluxTransformer2dModel.from_pretrained(extra_info.get("svdq_repo"),offload=True)
|
|
try:
|
|
transformer.set_attention_impl("nunchaku-fp16")
|
|
except:
|
|
pass
|
|
dreamo_pipeline = DreamOPipeline.from_pretrained(flux_repo, transformer=transformer,torch_dtype=torch.bfloat16)
|
|
dreamo_pipeline.load_dreamo_model(device,dreamo_lora_path,cfg_distill_path,Turbo_path, use_turbo=True,use_svdq=True,dreamo_version=dreamo_version)
|
|
elif extra_info.get("unet_path") is not None:
|
|
print("use single unet")
|
|
if quantize_mode=="fp8":
|
|
transformer = FluxTransformer2DModel.from_single_file(extra_info.get("unet_path"), config=os.path.join(cur_path,"config/FLUX.1-dev/transformer/config.json"),
|
|
torch_dtype=torch.bfloat16)
|
|
else:
|
|
from safetensors.torch import load_file
|
|
t_state_dict=load_file(extra_info.get("unet_path"))
|
|
unet_config = FluxTransformer2DModel.load_config(os.path.join(cur_path,"config/FLUX.1-dev/transformer/config.json"))
|
|
transformer = FluxTransformer2DModel.from_config(unet_config).to(torch.bfloat16)
|
|
transformer.load_state_dict(t_state_dict, strict=False)
|
|
del t_state_dict
|
|
gc_cleanup()
|
|
dreamo_pipeline = DreamOPipeline.from_pretrained(flux_repo, transformer=transformer,torch_dtype=torch.bfloat16)
|
|
dreamo_pipeline.load_dreamo_model(device,dreamo_lora_path,cfg_distill_path,Turbo_path, use_turbo=True,dreamo_version=dreamo_version)
|
|
else:
|
|
print(f"use {quantize_mode} quantization")
|
|
if quantize_mode=="fp8":
|
|
# transformer = FluxTransformer2DModel.from_pretrained(
|
|
# cf_model.get("extra_repo"),
|
|
# subfolder="transformer",
|
|
# quantization_config=DiffusersBitsAndBytesConfig(load_in_8bit=True,),
|
|
# torch_dtype=torch.bfloat16,
|
|
# )
|
|
dreamo_pipeline = DreamOPipeline.from_pretrained(flux_repo, torch_dtype=torch.bfloat16)
|
|
dreamo_pipeline.load_dreamo_model(device,dreamo_lora_path,cfg_distill_path,Turbo_path, use_turbo=True,dreamo_version=dreamo_version)
|
|
from optimum.quanto import freeze, qint8, quantize
|
|
quantize(dreamo_pipeline.transformer, qint8)
|
|
freeze(dreamo_pipeline.transformer)
|
|
quantize(dreamo_pipeline.text_encoder_2, qint8)
|
|
freeze(dreamo_pipeline.text_encoder_2)
|
|
|
|
else:
|
|
if quantize_mode=="nf4": #nf4
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
flux_repo,
|
|
subfolder="transformer",
|
|
quantization_config=DiffusersBitsAndBytesConfig(
|
|
load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
|
|
),
|
|
torch_dtype=torch.bfloat16,)
|
|
else:
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
flux_repo,
|
|
subfolder="transformer",
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
dreamo_pipeline = DreamOPipeline.from_pretrained(flux_repo, transformer=transformer,torch_dtype=torch.bfloat16)
|
|
dreamo_pipeline.load_dreamo_model(device,dreamo_lora_path,cfg_distill_path,Turbo_path, use_turbo=True,dreamo_version=dreamo_version)
|
|
dreamo_pipeline.enable_model_cpu_offload()
|
|
return dreamo_pipeline
|
|
|
|
@torch.no_grad()
|
|
def get_align_face(img,face_helper):
|
|
from .DreamO.dreamo.utils import img2tensor,tensor2img
|
|
from torchvision.transforms.functional import normalize
|
|
# the face preprocessing code is same as PuLID
|
|
face_helper.clean_all()
|
|
image_bgr = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
|
|
face_helper.read_image(image_bgr)
|
|
face_helper.get_face_landmarks_5(only_center_face=True)
|
|
face_helper.align_warp_face()
|
|
if len(face_helper.cropped_faces) == 0:
|
|
return None
|
|
align_face = face_helper.cropped_faces[0]
|
|
|
|
input = img2tensor(align_face, bgr2rgb=True).unsqueeze(0) / 255.0
|
|
input = input.to(torch.device("cuda"))
|
|
parsing_out = face_helper.face_parse(normalize(input, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]))[0]
|
|
parsing_out = parsing_out.argmax(dim=1, keepdim=True)
|
|
bg_label = [0, 16, 18, 7, 8, 9, 14, 15]
|
|
bg = sum(parsing_out == i for i in bg_label).bool()
|
|
white_image = torch.ones_like(input)
|
|
# only keep the face features
|
|
face_features_image = torch.where(bg, white_image, input)
|
|
face_features_image = tensor2img(face_features_image, rgb2bgr=False)
|
|
|
|
return face_features_image
|
|
|
|
def Dreamo_image_encoder(BEN2_path,ref_image1,ref_image2,ref_task1,ref_task2,ref_res):
|
|
from .DreamO.dreamo.utils import img2tensor, resize_numpy_image_area
|
|
ref_conds = []
|
|
#debug_images = []
|
|
|
|
ref_images = [ref_image1, ref_image2]
|
|
ref_tasks = [ref_task1, ref_task2]
|
|
|
|
for idx, (ref_image, ref_task) in enumerate(zip(ref_images, ref_tasks)):
|
|
if ref_image is not None:
|
|
if ref_task == "id":
|
|
from facexlib.utils.face_restoration_helper import FaceRestoreHelper
|
|
face_helper = FaceRestoreHelper(
|
|
upscale_factor=1,
|
|
face_size=512,
|
|
crop_ratio=(1, 1),
|
|
det_model='retinaface_resnet50',
|
|
save_ext='png',
|
|
device=device,
|
|
)
|
|
ref_image = resize_numpy_image_long(ref_image, 1024)
|
|
ref_image = get_align_face(ref_image,face_helper)
|
|
elif ref_task != "style": # ip
|
|
from .DreamO.tools import BEN2
|
|
bg_rm_model = BEN2.BEN_Base().to(device).eval() #TODO 反复加载 需要修复
|
|
#hf_hub_download(repo_id='PramaLLC/BEN2', filename='BEN2_Base.pth', local_dir='models')
|
|
bg_rm_model.loadcheckpoints(BEN2_path)
|
|
ref_image = bg_rm_model.inference(ref_image) #NEED TO CHECK
|
|
bg_rm_model.to(torch.device('cpu'))
|
|
if ref_task != "id":
|
|
ref_image = resize_numpy_image_area(np.array(ref_image), ref_res * ref_res)
|
|
|
|
#debug_images.append(ref_image)
|
|
ref_image = img2tensor(ref_image, bgr2rgb=False).unsqueeze(0) / 255.0
|
|
ref_image = 2 * ref_image - 1.0
|
|
ref_conds.append(
|
|
{
|
|
'img': ref_image,
|
|
'task': ref_task,
|
|
'idx': idx + 1,
|
|
}
|
|
)
|
|
return ref_conds
|
|
|
|
def Loader_Flux_Diffuser(extra_info,omi_lora_path,VAE,quantize_mode):
|
|
from .OmniConsistency.src_inference.pipeline import FluxPipeline as FluxPipeline_dif
|
|
from .OmniConsistency.src_inference.pipeline_ import FluxPipeline as FluxPipeline_dif_original
|
|
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig,FluxTransformer2DModel
|
|
from .OmniConsistency.src_inference.lora_helper import set_single_lora
|
|
flux_dif_repo=os.path.join(cur_path,"config/FLUX.1-dev")
|
|
if extra_info.get("gguf_path") is not None:
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
print("use gguf quantization")
|
|
|
|
from diffusers import GGUFQuantizationConfig
|
|
transformer = FluxTransformer2DModel.from_single_file(
|
|
extra_info.get("gguf_path"),
|
|
config=os.path.join(cur_path, "config/FLUX.1-dev/transformer"),
|
|
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
pipeline = FluxPipeline_dif.from_pretrained(flux_dif_repo, vae=vae,transformer=transformer,text_encoder=None,text_encoder_2=None,torch_dtype=torch.bfloat16)
|
|
|
|
elif extra_info.get("svdq_repo") is not None:
|
|
print("use svdq")
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
from nunchaku import NunchakuFluxTransformer2dModel
|
|
transformer = NunchakuFluxTransformer2dModel.from_pretrained(extra_info.get("svdq_repo"),offload=True)
|
|
try:
|
|
transformer.set_attention_impl("nunchaku-fp16")
|
|
except:
|
|
pass
|
|
pipeline = FluxPipeline_dif.from_pretrained(flux_dif_repo, vae=vae,transformer=transformer,text_encoder=None,text_encoder_2=None,torch_dtype=torch.bfloat16)
|
|
|
|
elif extra_info.get("unet_path") is not None:
|
|
print("use single unet")
|
|
vae=convert_cfvae2diffuser(VAE,use_flux=True)
|
|
if quantize_mode=="fp8":
|
|
transformer = FluxTransformer2DModel.from_single_file(extra_info.get("unet_path"), config=os.path.join(cur_path,"config/FLUX.1-dev/transformer/config.json"),
|
|
torch_dtype=torch.bfloat16)
|
|
else:
|
|
from safetensors.torch import load_file
|
|
t_state_dict=load_file(extra_info.get("unet_path"))
|
|
unet_config = FluxTransformer2DModel.load_config(os.path.join(cur_path,"config/FLUX.1-dev/transformer/config.json"))
|
|
transformer = FluxTransformer2DModel.from_config(unet_config).to(torch.bfloat16)
|
|
transformer.load_state_dict(t_state_dict, strict=False)
|
|
del t_state_dict
|
|
gc_cleanup()
|
|
pipeline = FluxPipeline_dif.from_pretrained(flux_dif_repo, vae=vae,transformer=transformer,text_encoder=None,text_encoder_2=None,torch_dtype=torch.bfloat16)
|
|
|
|
else:
|
|
print(f"use {quantize_mode} quantization")
|
|
repo_list=[i for i in [extra_info.get("repo1_path"),extra_info.get("repo2_path")] if i is not None]
|
|
if repo_list:
|
|
flux_repo=[i for i in repo_list if "flux" in i.lower()]
|
|
else:
|
|
raise "you must fill a flux repo"
|
|
if quantize_mode=="nf4": #nf4
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
flux_repo[0],
|
|
subfolder="transformer",
|
|
quantization_config=DiffusersBitsAndBytesConfig(
|
|
load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
|
|
),
|
|
torch_dtype=torch.bfloat16,)
|
|
elif quantize_mode=="fp8":
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
flux_repo[0],
|
|
subfolder="transformer",
|
|
quantization_config=DiffusersBitsAndBytesConfig(load_in_8bit=True,),
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
else:
|
|
transformer = FluxTransformer2DModel.from_pretrained(
|
|
flux_repo[0],
|
|
subfolder="transformer",
|
|
torch_dtype=torch.bfloat16,
|
|
)
|
|
pipeline = FluxPipeline_dif_original.from_pretrained(flux_repo[0],transformer=transformer,torch_dtype=torch.bfloat16)
|
|
|
|
multi_lora=[]
|
|
if extra_info.get("lora1_path") is not None:
|
|
multi_lora.append(extra_info.get("lora1_path"))
|
|
if extra_info.get("lora2_path") is not None:
|
|
multi_lora.append(extra_info.get("lora2_path"))
|
|
if multi_lora:
|
|
if len(multi_lora)==1:
|
|
set_single_lora(pipeline.transformer, omi_lora_path, lora_weights=[1], cond_size=512)
|
|
pipeline.unload_lora_weights()
|
|
pipeline.load_lora_weights(os.path.dirname(multi_lora[0]),weight_name=os.path.basename(multi_lora[0]))
|
|
|
|
else:
|
|
from .OmniConsistency.src_inference.lora_helper import set_multi_lora
|
|
lora_weights = [1]*len(multi_lora)
|
|
set_multi_lora(pipeline.transformer, multi_lora, lora_weights=[[lora_weights]], cond_size=512)
|
|
pipeline.unload_lora_weights()
|
|
else:
|
|
raise "need chocie a lora model in EasyFunction_Lite!"
|
|
pipeline.enable_model_cpu_offload()
|
|
return pipeline
|
|
|
|
|
|
|
|
def load_lora_for_unet_only(pipeline, lora_path, adapter_name="default", lora_scale=1.0):
|
|
|
|
try:
|
|
|
|
pipeline.load_lora_weights(lora_path, adapter_name=adapter_name)
|
|
pipeline.set_adapters(adapter_name, adapter_weights=lora_scale)
|
|
print(f"成功加载LoRA权重: {adapter_name} (scale: {lora_scale})")
|
|
except Exception as e:
|
|
print(f"加载LoRA权重失败: {e}")
|
|
|
|
return pipeline
|