436 lines
15 KiB
Python
436 lines
15 KiB
Python
import copy
|
|
import os
|
|
import torch
|
|
from safetensors.torch import load_file
|
|
from .utils import SCHEDULERS, token_auto_concat_embeds, vae_pt_to_vae_diffuser, convert_images_to_tensors, convert_tensors_to_images, resize_images
|
|
from comfy.model_management import get_torch_device
|
|
import folder_paths
|
|
from streamdiffusion import StreamDiffusion
|
|
from streamdiffusion.image_utils import postprocess_image
|
|
from diffusers import StableDiffusionPipeline, AutoencoderKL, AutoencoderTiny
|
|
|
|
|
|
class DiffusersPipelineLoader:
|
|
def __init__(self):
|
|
self.tmp_dir = folder_paths.get_temp_directory()
|
|
self.dtype = torch.float32
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": { "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ), }}
|
|
|
|
RETURN_TYPES = ("PIPELINE", "AUTOENCODER", "SCHEDULER",)
|
|
|
|
FUNCTION = "create_pipeline"
|
|
|
|
CATEGORY = "Diffusers"
|
|
|
|
def create_pipeline(self, ckpt_name):
|
|
ckpt_cache_path = os.path.join(self.tmp_dir, ckpt_name)
|
|
|
|
StableDiffusionPipeline.from_single_file(
|
|
pretrained_model_link_or_path=folder_paths.get_full_path("checkpoints", ckpt_name),
|
|
torch_dtype=self.dtype,
|
|
cache_dir=self.tmp_dir,
|
|
).save_pretrained(ckpt_cache_path, safe_serialization=True)
|
|
|
|
pipe = StableDiffusionPipeline.from_pretrained(
|
|
pretrained_model_name_or_path=ckpt_cache_path,
|
|
torch_dtype=self.dtype,
|
|
cache_dir=self.tmp_dir,
|
|
)
|
|
return ((pipe, ckpt_cache_path), pipe.vae, pipe.scheduler)
|
|
|
|
class DiffusersVaeLoader:
|
|
def __init__(self):
|
|
self.tmp_dir = folder_paths.get_temp_directory()
|
|
self.dtype = torch.float32
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": { "vae_name": (folder_paths.get_filename_list("vae"), ), }}
|
|
|
|
RETURN_TYPES = ("AUTOENCODER",)
|
|
|
|
FUNCTION = "create_pipeline"
|
|
|
|
CATEGORY = "Diffusers"
|
|
|
|
def create_pipeline(self, vae_name):
|
|
ckpt_cache_path = os.path.join(self.tmp_dir, vae_name)
|
|
vae_pt_to_vae_diffuser(folder_paths.get_full_path("vae", vae_name), ckpt_cache_path)
|
|
|
|
vae = AutoencoderKL.from_pretrained(
|
|
pretrained_model_name_or_path=ckpt_cache_path,
|
|
torch_dtype=self.dtype,
|
|
cache_dir=self.tmp_dir,
|
|
)
|
|
|
|
return (vae,)
|
|
|
|
class DiffusersSchedulerLoader:
|
|
def __init__(self):
|
|
self.tmp_dir = folder_paths.get_temp_directory()
|
|
self.dtype = torch.float32
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"pipeline": ("PIPELINE", ),
|
|
"scheduler_name": (list(SCHEDULERS.keys()), ),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("SCHEDULER",)
|
|
|
|
FUNCTION = "load_scheduler"
|
|
|
|
CATEGORY = "Diffusers"
|
|
|
|
def load_scheduler(self, pipeline, scheduler_name):
|
|
scheduler = SCHEDULERS[scheduler_name].from_pretrained(
|
|
pretrained_model_name_or_path=pipeline[1],
|
|
torch_dtype=self.dtype,
|
|
cache_dir=self.tmp_dir,
|
|
subfolder='scheduler'
|
|
)
|
|
return (scheduler,)
|
|
|
|
class DiffusersModelMakeup:
|
|
def __init__(self):
|
|
self.torch_device = get_torch_device()
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"pipeline": ("PIPELINE", ),
|
|
"scheduler": ("SCHEDULER", ),
|
|
"autoencoder": ("AUTOENCODER", ),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("MAKED_PIPELINE",)
|
|
|
|
FUNCTION = "makeup_pipeline"
|
|
|
|
CATEGORY = "Diffusers"
|
|
|
|
def makeup_pipeline(self, pipeline, scheduler, autoencoder):
|
|
pipeline = pipeline[0]
|
|
pipeline.vae = autoencoder
|
|
pipeline.scheduler = scheduler
|
|
pipeline.safety_checker = None if pipeline.safety_checker is None else lambda images, **kwargs: (images, [False])
|
|
pipeline.enable_attention_slicing()
|
|
pipeline = pipeline.to(self.torch_device)
|
|
return (pipeline,)
|
|
|
|
class DiffusersClipTextEncode:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"maked_pipeline": ("MAKED_PIPELINE", ),
|
|
"positive": ("STRING", {"multiline": True}),
|
|
"negative": ("STRING", {"multiline": True}),
|
|
}}
|
|
|
|
RETURN_TYPES = ("EMBEDS", "EMBEDS", "STRING", "STRING", )
|
|
RETURN_NAMES = ("positive_embeds", "negative_embeds", "positive", "negative", )
|
|
|
|
FUNCTION = "concat_embeds"
|
|
|
|
CATEGORY = "Diffusers"
|
|
|
|
def concat_embeds(self, maked_pipeline, positive, negative):
|
|
positive_embeds, negative_embeds = token_auto_concat_embeds(maked_pipeline, positive,negative)
|
|
|
|
return (positive_embeds, negative_embeds, positive, negative, )
|
|
|
|
class DiffusersSampler:
|
|
def __init__(self):
|
|
self.torch_device = get_torch_device()
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": {
|
|
"maked_pipeline": ("MAKED_PIPELINE", ),
|
|
"positive_embeds": ("EMBEDS", ),
|
|
"negative_embeds": ("EMBEDS", ),
|
|
"width": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
|
|
"height": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
|
|
"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
|
|
"cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
|
|
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
|
|
}}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
|
|
FUNCTION = "sample"
|
|
|
|
CATEGORY = "Diffusers"
|
|
|
|
def sample(self, maked_pipeline, positive_embeds, negative_embeds, height, width, steps, cfg, seed):
|
|
images = maked_pipeline(
|
|
prompt_embeds=positive_embeds,
|
|
height=height,
|
|
width=width,
|
|
num_inference_steps=steps,
|
|
guidance_scale=cfg,
|
|
negative_prompt_embeds=negative_embeds,
|
|
generator=torch.Generator(self.torch_device).manual_seed(seed)
|
|
).images
|
|
return (convert_images_to_tensors(images),)
|
|
|
|
# - Stream Diffusion -
|
|
|
|
class CreateIntListNode:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
max_element = 10
|
|
return {
|
|
"required": {
|
|
"elements_count" : ("INT", {"default": 2, "min": 1, "max": max_element, "step": 1}),
|
|
},
|
|
"optional": {
|
|
f"element_{i}": ("INT", {"default": 0}) for i in range(1, max_element)
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("LIST",)
|
|
FUNCTION = "create_list"
|
|
|
|
CATEGORY = "Diffusers/StreamDiffusion"
|
|
|
|
def create_list(self, elements_count, **kwargs):
|
|
return ([value for key, value in kwargs.items()][:elements_count], )
|
|
|
|
class LcmLoraLoader:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {"required": { "lora_name": (folder_paths.get_filename_list("loras"), ), }}
|
|
|
|
RETURN_TYPES = ("LCM_LORA",)
|
|
FUNCTION = "load_lora"
|
|
|
|
CATEGORY = "Diffusers/StreamDiffusion"
|
|
|
|
def load_lora(self, lora_name):
|
|
return (load_file(folder_paths.get_full_path("loras", lora_name)), )
|
|
|
|
class StreamDiffusionCreateStream:
|
|
def __init__(self):
|
|
self.dtype = torch.float32
|
|
self.torch_device = get_torch_device()
|
|
self.tmp_dir = folder_paths.get_temp_directory()
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"maked_pipeline": ("MAKED_PIPELINE", ),
|
|
"t_index_list": ("LIST", ),
|
|
"width": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
|
|
"height": ("INT", {"default": 512, "min": 1, "max": 8192, "step": 1}),
|
|
"do_add_noise": ("BOOLEAN", {"default": True}),
|
|
"use_denoising_batch": ("BOOLEAN", {"default": True}),
|
|
"frame_buffer_size": ("INT", {"default": 1, "min": 1, "max": 10000}),
|
|
"cfg_type": (["none", "full", "self", "initialize"], {"default": "none"}),
|
|
"xformers_memory_efficient_attention": ("BOOLEAN", {"default": False}),
|
|
"lcm_lora" : ("LCM_LORA", ),
|
|
"tiny_vae" : ("STRING", {"default": "madebyollin/taesd"})
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("STREAM",)
|
|
FUNCTION = "load_stream"
|
|
|
|
CATEGORY = "Diffusers/StreamDiffusion"
|
|
|
|
def load_stream(self, maked_pipeline, t_index_list, width, height, do_add_noise, use_denoising_batch, frame_buffer_size, cfg_type, xformers_memory_efficient_attention, lcm_lora, tiny_vae):
|
|
maked_pipeline = copy.deepcopy(maked_pipeline)
|
|
lcm_lora = copy.deepcopy(lcm_lora)
|
|
stream = StreamDiffusion(
|
|
pipe = maked_pipeline,
|
|
t_index_list = t_index_list,
|
|
torch_dtype = self.dtype,
|
|
width = width,
|
|
height = height,
|
|
do_add_noise = do_add_noise,
|
|
use_denoising_batch = use_denoising_batch,
|
|
frame_buffer_size = frame_buffer_size,
|
|
cfg_type = cfg_type,
|
|
)
|
|
stream.load_lcm_lora(lcm_lora)
|
|
stream.fuse_lora()
|
|
stream.vae = AutoencoderTiny.from_pretrained(
|
|
pretrained_model_name_or_path=tiny_vae,
|
|
torch_dtype=self.dtype,
|
|
cache_dir=self.tmp_dir,
|
|
).to(
|
|
device=maked_pipeline.device,
|
|
dtype=maked_pipeline.dtype
|
|
)
|
|
|
|
if xformers_memory_efficient_attention:
|
|
maked_pipeline.enable_xformers_memory_efficient_attention()
|
|
return (stream, )
|
|
|
|
class StreamDiffusionSampler:
|
|
def __init__(self):
|
|
self.torch_device = get_torch_device()
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"stream": ("STREAM", ),
|
|
"positive": ("STRING", {"multiline": True}),
|
|
"negative": ("STRING", {"multiline": True}),
|
|
"steps": ("INT", {"default": 50, "min": 1, "max": 10000}),
|
|
"cfg": ("FLOAT", {"default": 1.2, "min": 0.0, "max": 100.0}),
|
|
"delta": ("FLOAT", {"default": 1, "min": 0.0, "max": 1.0}),
|
|
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
|
|
"num": ("INT", {"default": 1, "min": 1, "max": 10000}),
|
|
"warmup": ("INT", {"default": 1, "min": 0, "max": 10000}),
|
|
},
|
|
"optional" : {
|
|
"image" : ("IMAGE", )
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
|
|
FUNCTION = "sample"
|
|
|
|
CATEGORY = "Diffusers/StreamDiffusion"
|
|
|
|
def sample(self, stream: StreamDiffusion, positive, negative, steps, cfg, delta, seed, num, warmup, image = None):
|
|
stream.prepare(
|
|
prompt = positive,
|
|
negative_prompt = negative,
|
|
num_inference_steps = steps,
|
|
guidance_scale = cfg,
|
|
delta = delta,
|
|
seed = seed
|
|
)
|
|
|
|
if image != None:
|
|
image = convert_tensors_to_images(image)
|
|
image = resize_images(image, (stream.width, stream.height))
|
|
|
|
for _ in range(warmup):
|
|
stream()
|
|
|
|
result = []
|
|
for _ in range(num):
|
|
x_outputs = []
|
|
if image is None:
|
|
x_outputs.append(stream.txt2img())
|
|
else:
|
|
stream(image[0])
|
|
for i in image[1:] + image[-1:]:
|
|
x_outputs.append(stream(i))
|
|
for x_output in x_outputs:
|
|
result.append(postprocess_image(x_output, output_type="pil")[0])
|
|
|
|
return (convert_images_to_tensors(result),)
|
|
|
|
class StreamDiffusionWarmup:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"stream": ("STREAM", ),
|
|
"negative": ("STRING", {"multiline": True}),
|
|
"steps": ("INT", {"default": 50, "min": 1, "max": 10000}),
|
|
"cfg": ("FLOAT", {"default": 1.2, "min": 0.0, "max": 100.0}),
|
|
"delta": ("FLOAT", {"default": 1, "min": 0.0, "max": 1.0}),
|
|
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
|
|
"warmup": ("INT", {"default": 1, "min": 0, "max": 10000}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("WARMUP_STREAM",)
|
|
|
|
FUNCTION = "stream_warmup"
|
|
|
|
CATEGORY = "Diffusers/StreamDiffusion"
|
|
|
|
def stream_warmup(self, stream: StreamDiffusion, negative, steps, cfg, delta, seed, warmup):
|
|
stream.prepare(
|
|
prompt="",
|
|
negative_prompt=negative,
|
|
num_inference_steps = steps,
|
|
guidance_scale = cfg,
|
|
delta = delta,
|
|
seed = seed
|
|
)
|
|
|
|
for _ in range(warmup):
|
|
stream()
|
|
|
|
return (stream, )
|
|
|
|
|
|
class StreamDiffusionFastSampler:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"warmup_stream": ("WARMUP_STREAM", ),
|
|
"positive": ("STRING", {"multiline": True}),
|
|
"num": ("INT", {"default": 1, "min": 1, "max": 10000}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
|
|
FUNCTION = "sample"
|
|
|
|
CATEGORY = "Diffusers/StreamDiffusion"
|
|
|
|
def sample(self, warmup_stream, positive, num):
|
|
stream: StreamDiffusion = warmup_stream
|
|
|
|
stream.update_prompt(positive)
|
|
|
|
result = []
|
|
for _ in range(num):
|
|
x_output = stream.txt2img()
|
|
result.append(postprocess_image(x_output, output_type="pil")[0])
|
|
return (convert_images_to_tensors(result),)
|
|
|
|
# - - - - - - - - - - - - - - - - - -
|
|
|
|
|
|
NODE_CLASS_MAPPINGS = {
|
|
"DiffusersPipelineLoader": DiffusersPipelineLoader,
|
|
"DiffusersVaeLoader": DiffusersVaeLoader,
|
|
"DiffusersSchedulerLoader": DiffusersSchedulerLoader,
|
|
"DiffusersModelMakeup": DiffusersModelMakeup,
|
|
"DiffusersClipTextEncode": DiffusersClipTextEncode,
|
|
"DiffusersSampler": DiffusersSampler,
|
|
"CreateIntListNode": CreateIntListNode,
|
|
"LcmLoraLoader": LcmLoraLoader,
|
|
"StreamDiffusionCreateStream": StreamDiffusionCreateStream,
|
|
"StreamDiffusionSampler": StreamDiffusionSampler,
|
|
"StreamDiffusionWarmup": StreamDiffusionWarmup,
|
|
"StreamDiffusionFastSampler": StreamDiffusionFastSampler,
|
|
}
|
|
|
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
|
"DiffusersPipelineLoader": "Diffusers Pipeline Loader",
|
|
"DiffusersVaeLoader": "Diffusers Vae Loader",
|
|
"DiffusersSchedulerLoader": "Diffusers Scheduler Loader",
|
|
"DiffusersModelMakeup": "Diffusers Model Makeup",
|
|
"DiffusersClipTextEncode": "Diffusers Clip Text Encode",
|
|
"DiffusersSampler": "Diffusers Sampler",
|
|
"CreateIntListNode": "Create Int List",
|
|
"LcmLoraLoader": "LCM Lora Loader",
|
|
"StreamDiffusionCreateStream": "StreamDiffusion Create Stream",
|
|
"StreamDiffusionSampler": "StreamDiffusion Sampler",
|
|
"StreamDiffusionWarmup": "StreamDiffusion Warmup",
|
|
"StreamDiffusionFastSampler": "StreamDiffusion Fast Sampler",
|
|
}
|