Files
2024-01-09 17:32:43 +09:00

613 lines
22 KiB
Python

import copy
import os
import gc
import torch
import traceback
from safetensors.torch import load_file
from .utils import SCHEDULERS, token_auto_concat_embeds, vae_pt_to_vae_diffuser, convert_images_to_tensors, convert_tensors_to_images, resize_images
from comfy.model_management import get_torch_device
import folder_paths
from streamdiffusion import StreamDiffusion
from streamdiffusion.image_utils import postprocess_image
from diffusers import StableDiffusionPipeline, AutoencoderKL, AutoencoderTiny
from pathlib import Path
class DiffusersPipelineLoader:
def __init__(self):
self.tmp_dir = folder_paths.get_temp_directory()
self.dtype = torch.float32
@classmethod
def INPUT_TYPES(s):
return {"required": { "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ), }}
RETURN_TYPES = ("PIPELINE", "AUTOENCODER", "SCHEDULER",)
FUNCTION = "create_pipeline"
CATEGORY = "Diffusers"
def create_pipeline(self, ckpt_name):
ckpt_cache_path = os.path.join(self.tmp_dir, ckpt_name)
StableDiffusionPipeline.from_single_file(
pretrained_model_link_or_path=folder_paths.get_full_path("checkpoints", ckpt_name),
torch_dtype=self.dtype,
cache_dir=self.tmp_dir,
).save_pretrained(ckpt_cache_path, safe_serialization=True)
pipe = StableDiffusionPipeline.from_pretrained(
pretrained_model_name_or_path=ckpt_cache_path,
torch_dtype=self.dtype,
cache_dir=self.tmp_dir,
)
return ((pipe, ckpt_cache_path), pipe.vae, pipe.scheduler)
class DiffusersVaeLoader:
def __init__(self):
self.tmp_dir = folder_paths.get_temp_directory()
self.dtype = torch.float32
@classmethod
def INPUT_TYPES(s):
return {"required": { "vae_name": (folder_paths.get_filename_list("vae"), ), }}
RETURN_TYPES = ("AUTOENCODER",)
FUNCTION = "create_pipeline"
CATEGORY = "Diffusers"
def create_pipeline(self, vae_name):
ckpt_cache_path = os.path.join(self.tmp_dir, vae_name)
vae_pt_to_vae_diffuser(folder_paths.get_full_path("vae", vae_name), ckpt_cache_path)
vae = AutoencoderKL.from_pretrained(
pretrained_model_name_or_path=ckpt_cache_path,
torch_dtype=self.dtype,
cache_dir=self.tmp_dir,
)
return (vae,)
class DiffusersSchedulerLoader:
def __init__(self):
self.tmp_dir = folder_paths.get_temp_directory()
self.dtype = torch.float32
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"pipeline": ("PIPELINE", ),
"scheduler_name": (list(SCHEDULERS.keys()), ),
}
}
RETURN_TYPES = ("SCHEDULER",)
FUNCTION = "load_scheduler"
CATEGORY = "Diffusers"
def load_scheduler(self, pipeline, scheduler_name):
scheduler = SCHEDULERS[scheduler_name].from_pretrained(
pretrained_model_name_or_path=pipeline[1],
torch_dtype=self.dtype,
cache_dir=self.tmp_dir,
subfolder='scheduler'
)
return (scheduler,)
class DiffusersModelMakeup:
def __init__(self):
self.torch_device = get_torch_device()
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"pipeline": ("PIPELINE", ),
"scheduler": ("SCHEDULER", ),
"autoencoder": ("AUTOENCODER", ),
},
}
RETURN_TYPES = ("MAKED_PIPELINE",)
FUNCTION = "makeup_pipeline"
CATEGORY = "Diffusers"
def makeup_pipeline(self, pipeline, scheduler, autoencoder):
pipe = pipeline[0]
ckpt_cache_path = pipeline[1]
pipe.vae = autoencoder
pipe.scheduler = scheduler
pipe.safety_checker = None if pipe.safety_checker is None else lambda images, **kwargs: (images, [False])
pipe.enable_attention_slicing()
pipe = pipe.to(self.torch_device)
return ((pipe, ckpt_cache_path),)
class DiffusersClipTextEncode:
@classmethod
def INPUT_TYPES(s):
return {"required": {
"maked_pipeline": ("MAKED_PIPELINE", ),
"positive": ("STRING", {"multiline": True}),
"negative": ("STRING", {"multiline": True}),
}}
RETURN_TYPES = ("EMBEDS", "EMBEDS", "STRING", "STRING", )
RETURN_NAMES = ("positive_embeds", "negative_embeds", "positive", "negative", )
FUNCTION = "concat_embeds"
CATEGORY = "Diffusers"
def concat_embeds(self, maked_pipeline, positive, negative):
positive_embeds, negative_embeds = token_auto_concat_embeds(maked_pipeline[0], positive,negative)
return (positive_embeds, negative_embeds, positive, negative, )
class DiffusersSampler:
def __init__(self):
self.torch_device = get_torch_device()
@classmethod
def INPUT_TYPES(s):
return {"required": {
"maked_pipeline": ("MAKED_PIPELINE", ),
"positive_embeds": ("EMBEDS", ),
"negative_embeds": ("EMBEDS", ),
"width": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
"height": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
"cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
}}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "sample"
CATEGORY = "Diffusers"
def sample(self, maked_pipeline, positive_embeds, negative_embeds, height, width, steps, cfg, seed):
images = maked_pipeline[0](
prompt_embeds=positive_embeds,
height=height,
width=width,
num_inference_steps=steps,
guidance_scale=cfg,
negative_prompt_embeds=negative_embeds,
generator=torch.Generator(self.torch_device).manual_seed(seed)
).images
return (convert_images_to_tensors(images),)
# - Stream Diffusion -
class CreateIntListNode:
@classmethod
def INPUT_TYPES(s):
max_element = 10
return {
"required": {
"elements_count" : ("INT", {"default": 2, "min": 1, "max": max_element, "step": 1}),
},
"optional": {
f"element_{i}": ("INT", {"default": 0}) for i in range(1, max_element)
}
}
RETURN_TYPES = ("LIST",)
FUNCTION = "create_list"
CATEGORY = "Diffusers/StreamDiffusion"
def create_list(self, elements_count, **kwargs):
return ([value for key, value in kwargs.items()][:elements_count], )
class LcmLoraLoader:
@classmethod
def INPUT_TYPES(s):
return {"required": { "lora_name": (folder_paths.get_filename_list("loras"), ), }}
RETURN_TYPES = ("LCM_LORA",)
FUNCTION = "load_lora"
CATEGORY = "Diffusers/StreamDiffusion"
def load_lora(self, lora_name):
return (load_file(folder_paths.get_full_path("loras", lora_name)), )
class StreamDiffusionCreateStream:
def __init__(self):
self.dtype = torch.float32
self.torch_device = get_torch_device()
self.tmp_dir = folder_paths.get_temp_directory()
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"maked_pipeline": ("MAKED_PIPELINE", ),
"t_index_list": ("LIST", ),
"width": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
"height": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
"do_add_noise": ("BOOLEAN", {"default": True}),
"use_denoising_batch": ("BOOLEAN", {"default": True}),
"frame_buffer_size": ("INT", {"default": 1, "min": 1, "max": 10000}),
"cfg_type": (["none", "full", "self", "initialize"], {"default": "none"}),
"acceleration": (["none", "xformers", "tensorrt"], {"default": "tensorrt"}),
"lcm_lora" : ("LCM_LORA", ),
"tiny_vae" : ("STRING", {"default": "madebyollin/taesd"}),
"use_lcm_lora" : ("BOOLEAN", {"default": True}),
"use_tiny_vae" : ("BOOLEAN", {"default": True}),
},
}
RETURN_TYPES = ("STREAM",)
FUNCTION = "load_stream"
CATEGORY = "Diffusers/StreamDiffusion"
def load_stream(self, maked_pipeline, t_index_list, width, height, do_add_noise, use_denoising_batch, frame_buffer_size, cfg_type, acceleration, lcm_lora, tiny_vae, use_lcm_lora, use_tiny_vae):
model_id_or_path = maked_pipeline[1]
maked_pipeline: StableDiffusionPipeline = copy.deepcopy(maked_pipeline[0])
lcm_lora = copy.deepcopy(lcm_lora)
stream = StreamDiffusion(
pipe = maked_pipeline,
t_index_list = t_index_list,
torch_dtype = self.dtype,
width = width,
height = height,
do_add_noise = do_add_noise,
use_denoising_batch = use_denoising_batch,
frame_buffer_size = frame_buffer_size,
cfg_type = cfg_type,
)
stream.load_lcm_lora(lcm_lora)
stream.fuse_lora()
stream.vae = AutoencoderTiny.from_pretrained(
pretrained_model_name_or_path=tiny_vae,
torch_dtype=self.dtype,
cache_dir=self.tmp_dir,
).to(
device=maked_pipeline.device,
dtype=maked_pipeline.dtype
)
batch_size = (
len(t_index_list) * frame_buffer_size
if use_denoising_batch
else frame_buffer_size
)
try:
if acceleration == "xformers":
stream.pipe.enable_xformers_memory_efficient_attention()
if acceleration == "tensorrt":
from polygraphy import cuda
from streamdiffusion.acceleration.tensorrt import (
TorchVAEEncoder,
compile_unet,
compile_vae_decoder,
compile_vae_encoder,
)
from streamdiffusion.acceleration.tensorrt.engine import (
AutoencoderKLEngine,
UNet2DConditionModelEngine,
)
from streamdiffusion.acceleration.tensorrt.models import (
VAE,
UNet,
VAEEncoder,
)
def create_prefix(
model_id_or_path: str,
max_batch_size: int,
min_batch_size: int,
):
maybe_path = Path(model_id_or_path)
if maybe_path.exists():
return f"{maybe_path.stem}--lcm_lora-{use_lcm_lora}--tiny_vae-{use_tiny_vae}--max_batch-{max_batch_size}--min_batch-{min_batch_size}--mode-{self.mode}"
else:
return f"{model_id_or_path}--lcm_lora-{use_lcm_lora}--tiny_vae-{use_tiny_vae}--max_batch-{max_batch_size}--min_batch-{min_batch_size}--mode-{self.mode}"
engine_dir = Path(engine_dir)
unet_path = os.path.join(
engine_dir,
create_prefix(
model_id_or_path=model_id_or_path,
max_batch_size=stream.trt_unet_batch_size,
min_batch_size=stream.trt_unet_batch_size,
),
"unet.engine",
)
vae_encoder_path = os.path.join(
engine_dir,
create_prefix(
model_id_or_path=model_id_or_path,
max_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
min_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
),
"vae_encoder.engine",
)
vae_decoder_path = os.path.join(
engine_dir,
create_prefix(
model_id_or_path=model_id_or_path,
max_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
min_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
),
"vae_decoder.engine",
)
if not os.path.exists(unet_path):
os.makedirs(os.path.dirname(unet_path), exist_ok=True)
unet_model = UNet(
fp16=True,
device=stream.device,
max_batch_size=stream.trt_unet_batch_size,
min_batch_size=stream.trt_unet_batch_size,
embedding_dim=stream.text_encoder.config.hidden_size,
unet_dim=stream.unet.config.in_channels,
)
compile_unet(
stream.unet,
unet_model,
unet_path + ".onnx",
unet_path + ".opt.onnx",
unet_path,
opt_batch_size=stream.trt_unet_batch_size,
)
if not os.path.exists(vae_decoder_path):
os.makedirs(os.path.dirname(vae_decoder_path), exist_ok=True)
stream.vae.forward = stream.vae.decode
vae_decoder_model = VAE(
device=stream.device,
max_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
min_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
)
compile_vae_decoder(
stream.vae,
vae_decoder_model,
vae_decoder_path + ".onnx",
vae_decoder_path + ".opt.onnx",
vae_decoder_path,
opt_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
)
delattr(stream.vae, "forward")
if not os.path.exists(vae_encoder_path):
os.makedirs(os.path.dirname(vae_encoder_path), exist_ok=True)
vae_encoder = TorchVAEEncoder(stream.vae).to(torch.device("cuda"))
vae_encoder_model = VAEEncoder(
device=stream.device,
max_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
min_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
)
compile_vae_encoder(
vae_encoder,
vae_encoder_model,
vae_encoder_path + ".onnx",
vae_encoder_path + ".opt.onnx",
vae_encoder_path,
opt_batch_size=batch_size
if self.mode == "txt2img"
else stream.frame_bff_size,
)
cuda_steram = cuda.Stream()
vae_config = stream.vae.config
vae_dtype = stream.vae.dtype
stream.unet = UNet2DConditionModelEngine(
unet_path, cuda_steram, use_cuda_graph=False
)
stream.vae = AutoencoderKLEngine(
vae_encoder_path,
vae_decoder_path,
cuda_steram,
stream.pipe.vae_scale_factor,
use_cuda_graph=False,
)
setattr(stream.vae, "config", vae_config)
setattr(stream.vae, "dtype", vae_dtype)
gc.collect()
torch.cuda.empty_cache()
print("TensorRT acceleration enabled.")
if acceleration == "sfast":
from streamdiffusion.acceleration.sfast import (
accelerate_with_stable_fast,
)
stream = accelerate_with_stable_fast(stream)
print("StableFast acceleration enabled.")
except Exception:
traceback.print_exc()
print("Acceleration has failed. Falling back to normal mode.")
return (stream, )
class StreamDiffusionSampler:
def __init__(self):
self.torch_device = get_torch_device()
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"stream": ("STREAM", ),
"positive": ("STRING", {"multiline": True}),
"negative": ("STRING", {"multiline": True}),
"steps": ("INT", {"default": 50, "min": 1, "max": 10000}),
"cfg": ("FLOAT", {"default": 1.2, "min": 0.0, "max": 100.0}),
"delta": ("FLOAT", {"default": 1, "min": 0.0, "max": 1.0}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
"num": ("INT", {"default": 1, "min": 1, "max": 10000}),
"warmup": ("INT", {"default": 1, "min": 0, "max": 10000}),
},
"optional" : {
"image" : ("IMAGE", )
}
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "sample"
CATEGORY = "Diffusers/StreamDiffusion"
def sample(self, stream: StreamDiffusion, positive, negative, steps, cfg, delta, seed, num, warmup, image = None):
stream.prepare(
prompt = positive,
negative_prompt = negative,
num_inference_steps = steps,
guidance_scale = cfg,
delta = delta,
seed = seed
)
if image != None:
image = convert_tensors_to_images(image)
image = resize_images(image, (stream.width, stream.height))
for _ in range(warmup):
stream()
result = []
for _ in range(num):
x_outputs = []
if image is None:
x_outputs.append(stream.txt2img())
else:
stream(image[0])
for i in image[1:] + image[-1:]:
x_outputs.append(stream(i))
for x_output in x_outputs:
result.append(postprocess_image(x_output, output_type="pil")[0])
return (convert_images_to_tensors(result),)
class StreamDiffusionWarmup:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"stream": ("STREAM", ),
"negative": ("STRING", {"multiline": True}),
"steps": ("INT", {"default": 50, "min": 1, "max": 10000}),
"cfg": ("FLOAT", {"default": 1.2, "min": 0.0, "max": 100.0}),
"delta": ("FLOAT", {"default": 1, "min": 0.0, "max": 1.0}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
"warmup": ("INT", {"default": 1, "min": 0, "max": 10000}),
},
}
RETURN_TYPES = ("WARMUP_STREAM",)
FUNCTION = "stream_warmup"
CATEGORY = "Diffusers/StreamDiffusion"
def stream_warmup(self, stream: StreamDiffusion, negative, steps, cfg, delta, seed, warmup):
stream.prepare(
prompt="",
negative_prompt=negative,
num_inference_steps = steps,
guidance_scale = cfg,
delta = delta,
seed = seed
)
for _ in range(warmup):
stream()
return (stream, )
class StreamDiffusionFastSampler:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"warmup_stream": ("WARMUP_STREAM", ),
"positive": ("STRING", {"multiline": True}),
"num": ("INT", {"default": 1, "min": 1, "max": 10000}),
},
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "sample"
CATEGORY = "Diffusers/StreamDiffusion"
def sample(self, warmup_stream, positive, num):
stream: StreamDiffusion = warmup_stream
stream.update_prompt(positive)
result = []
for _ in range(num):
x_output = stream.txt2img()
result.append(postprocess_image(x_output, output_type="pil")[0])
return (convert_images_to_tensors(result),)
# - - - - - - - - - - - - - - - - - -
NODE_CLASS_MAPPINGS = {
"DiffusersPipelineLoader": DiffusersPipelineLoader,
"DiffusersVaeLoader": DiffusersVaeLoader,
"DiffusersSchedulerLoader": DiffusersSchedulerLoader,
"DiffusersModelMakeup": DiffusersModelMakeup,
"DiffusersClipTextEncode": DiffusersClipTextEncode,
"DiffusersSampler": DiffusersSampler,
"CreateIntListNode": CreateIntListNode,
"LcmLoraLoader": LcmLoraLoader,
"StreamDiffusionCreateStream": StreamDiffusionCreateStream,
"StreamDiffusionSampler": StreamDiffusionSampler,
"StreamDiffusionWarmup": StreamDiffusionWarmup,
"StreamDiffusionFastSampler": StreamDiffusionFastSampler,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"DiffusersPipelineLoader": "Diffusers Pipeline Loader",
"DiffusersVaeLoader": "Diffusers Vae Loader",
"DiffusersSchedulerLoader": "Diffusers Scheduler Loader",
"DiffusersModelMakeup": "Diffusers Model Makeup",
"DiffusersClipTextEncode": "Diffusers Clip Text Encode",
"DiffusersSampler": "Diffusers Sampler",
"CreateIntListNode": "Create Int List",
"LcmLoraLoader": "LCM Lora Loader",
"StreamDiffusionCreateStream": "StreamDiffusion Create Stream",
"StreamDiffusionSampler": "StreamDiffusion Sampler",
"StreamDiffusionWarmup": "StreamDiffusion Warmup",
"StreamDiffusionFastSampler": "StreamDiffusion Fast Sampler",
}