nodes/storyicon-comfyui_musev_evolved
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: .

Browse Source 
Signed-off-by: storyicon <storyicon@foxmail.com>
This commit is contained in:
storyicon
2024-05-22 09:03:23 +00:00
parent b97e7f618e
commit bd981cf81f
3 changed files with 304 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodes.py
+6 -185
View File
@@ -1,188 +1,9 @@
import os
import folder_paths
comfy_path = os.path.dirname(folder_paths.__file__)
diffusers_path = folder_paths.get_folder_paths("diffusers")[0]
MuseVCheckPointDir = os.path.join(
diffusers_path, "TMElyralab/MuseV"
)
import sys
sys.path.insert(0,f'{comfy_path}/custom_nodes/ComfyUI-MuseV')
sys.path.insert(0,f'{comfy_path}/custom_nodes/ComfyUI-MuseV/MMCM')
sys.path.insert(0,f'{comfy_path}/custom_nodes/ComfyUI-MuseV/diffusers/src')
sys.path.insert(0,f'{comfy_path}/custom_nodes/ComfyUI-MuseV/controlnet_aux/src')
print(sys.path)
import numpy as np
import torch
from einops import repeat
from .MMCM.mmcm.utils.seed_util import set_all_seed
from .MMCM.mmcm.utils.task_util import fiss_tasks, generate_tasks as generate_tasks_from_table
from musev.pipelines.pipeline_controlnet_predictor import (
DiffusersPipelinePredictor,
)
from musev.models.unet_loader import load_unet_by_name
from musev import logger
logger.setLevel("INFO")
file_dir = os.path.dirname(__file__)
PROJECT_DIR = file_dir
DATA_DIR = os.path.join(PROJECT_DIR, "data")
class MuseVImg2Vid:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE", {"default":12}),
"time_size": ("INT",{"default":12}),
"seed": ("INT", {"default":1234}),
"video_num_inference_steps": ("INT", {"default":10}),
"video_guidance_scale": ("FLOAT", {"default":3.5, "round": False, "step":0.01}),
"w_ind_noise": ("FLOAT", {"default":0.5, "round": False, "step":0.01}),
"image_weight": ("FLOAT", {"default":0.001, "round": False, "step":0.001}),
"motion_speed": ("FLOAT", {"default":8.0, "round": False, "step":0.01}),
"context_frames": ("INT", {"default":12}),
"context_stride": ("INT", {"default":1}),
"context_overlap": ("INT", {"default":4}),
"positive_prompt": ("STRING", {"multiline": True, "default": "(masterpiece, best quality, highres:1),(1girl, solo:1),(beautiful face, soft skin, costume:1),(eye blinks:1.8),(head wave:1.3)"}),
"negative_prompt": ("STRING", {"multiline": True, "default": "badhandv4, ng_deepnegative_v1_75t, (((multiple heads))), (((bad body))), (((two people))), ((extra arms)), ((deformed body)), (((sexy))), paintings,(((two heads))), ((big head)),sketches, (worst quality:2), (low quality:2), (normal quality:2), lowres, ((monochrome)), ((grayscale)), skin spots, acnes, skin blemishes, age spot, glans, (((nsfw))), nipples, extra fingers, (extra legs), (long neck), mutated hands, (fused fingers), (too many fingers)"}),
}
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "main"
CATEGORY = "MuseV"
def main(
self,
image,
time_size,
seed,
w_ind_noise,
context_frames,
context_stride,
context_overlap,
video_num_inference_steps,
video_guidance_scale,
positive_prompt,
negative_prompt,
image_weight,
motion_speed,
):
condition_image = 255.0 * image[0].cpu().numpy()
condition_image = np.clip(condition_image, 0, 255).astype(np.uint8)
condition_image = repeat(condition_image, "h w c-> b c t h w", b=1, t=1)
width = condition_image.shape[4]
height = condition_image.shape[3]
sd_model_path = os.path.join(folder_paths.models_dir, 'diffusers/TMElyralab/MuseV/t2i/sd1.5/majicmixRealv6Fp16')
sd_unet_model = os.path.join(folder_paths.models_dir, 'diffusers/TMElyralab/MuseV/motion/musev_referencenet')
negative_embedding = [
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/badhandv4.pt"),
"badhandv4"
],
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/ng_deepnegative_v1_75t.pt"),
"ng_deepnegative_v1_75t"
],
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/EasyNegativeV2.safetensors"),
"EasyNegativeV2"
],
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/bad_prompt_version2-neg.pt"),
"bad_prompt_version2-neg"
]
]
vae_path = os.path.join(folder_paths.models_dir, 'diffusers/TMElyralab/MuseV/vae/sd-vae-ft-mse')
unet = load_unet_by_name(
model_name='musev_referencenet',
sd_unet_model=sd_unet_model,
sd_model=sd_model_path,
cross_attention_dim=768,
need_t2i_facein=False,
strict=True,
need_t2i_ip_adapter_face=False,
)
sd_predictor = DiffusersPipelinePredictor(
sd_model_path=sd_model_path,
unet=unet,
lora_dict=None,
lcm_lora_dct=None,
device='cuda',
dtype=torch.float16,
negative_embedding=negative_embedding,
referencenet=None,
ip_adapter_image_proj=None,
vision_clip_extractor=None,
facein_image_proj=None,
face_emb_extractor=None,
vae_model=vae_path,
ip_adapter_face_emb_extractor=None,
ip_adapter_face_image_proj=None,
)
cpu_generator, gpu_generator = set_all_seed(seed)
out_videos = sd_predictor.run_pipe_text2video(
video_length=time_size,
prompt=positive_prompt,
width=width,
height=height,
generator=gpu_generator,
noise_type='video_fusion',
negative_prompt=negative_prompt,
video_negative_prompt=negative_prompt,
max_batch_num=1,
strength=0.8,
need_img_based_video_noise=True,
video_num_inference_steps=video_num_inference_steps,
condition_images=condition_image,
fix_condition_images=False,
video_guidance_scale=video_guidance_scale,
guidance_scale=7.5,
num_inference_steps=30,
redraw_condition_image=False,
img_weight=image_weight,
w_ind_noise=w_ind_noise,
n_vision_condition=1,
motion_speed=motion_speed,
need_hist_match=False,
video_guidance_scale_end=None,
video_guidance_scale_method='linear',
vision_condition_latent_index=None,
refer_image=None,
fixed_refer_image=True,
redraw_condition_image_with_referencenet=True,
ip_adapter_image=None,
refer_face_image=None,
fixed_refer_face_image=True,
facein_scale=1.0,
redraw_condition_image_with_facein=True,
ip_adapter_face_scale=1.0,
redraw_condition_image_with_ip_adapter_face=True,
fixed_ip_adapter_image=True,
ip_adapter_scale=1.0,
redraw_condition_image_with_ipdapter=True,
prompt_only_use_image_prompt=False,
# serial_denoise parameter start
record_mid_video_noises=False,
record_mid_video_latents=False,
video_overlap=1,
# serial_denoise parameter end
# parallel_denoise parameter start
context_schedule='uniform_v2',
context_frames=context_frames,
context_stride=context_stride,
context_overlap=context_overlap,
context_batch_size=1,
interpolation_factor=1,
# parallel_denoise parameter end
)
return torch.from_numpy(out_videos).permute(0,2,3,4,1)
from .v1 import MuseVPredictorV1, MuseVImg2VidV1
from .utils import AnimationZoom, ImageSelector
NODE_CLASS_MAPPINGS = {
"MuseVImg2Vid (comfyui_musev_evolved)": MuseVImg2Vid,
"MuseVPredictor V1 (comfyui_musev_evolved)": MuseVPredictorV1,
"MuseVImg2Vid V1 (comfyui_musev_evolved)": MuseVImg2VidV1,
"AnimationZoom (comfyui_musev_evolved)": AnimationZoom,
"ImageSelector (comfyui_musev_evolved)": ImageSelector,
}
utils.py
+93
View File
@@ -0,0 +1,93 @@
import numpy as np
from PIL import Image
import torch
class AnimationZoom:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE", ),
"scale": ("FLOAT", {"default":0.5, "round": False, "step":0.01}),
"frame": ("INT", {"default": 10}),
}
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "main"
CATEGORY = "MuseV Evolved"
def main(self, image, scale, frame):
pil_image = Image.fromarray((image* 255).byte().numpy()[0]).convert('RGB')
w, h = pil_image.size
scales = np.linspace(1, scale, frame)
frames = []
for scale in scales:
res = Image.new("RGB", (w, h), color="white")
offset_x = int(w/2 - int(w * scale / 2))
offset_y = int(h/2 - int(h * scale / 2))
res.paste(pil_image.resize((int(w * scale), int(h * scale)), Image.Resampling.BILINEAR), (offset_x, offset_y))
tensor = torch.from_numpy(np.array(res)).float().div(255)
frames.append(tensor)
return (torch.stack(frames), )
class ImageSelector:
def __init__(self):
pass
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"images": ("IMAGE", ),
"selected_indexes": ("STRING", {
"multiline": False,
"default": "1,2,3"
}),
},
}
RETURN_TYPES = ("IMAGE", )
FUNCTION = "run"
OUTPUT_NODE = False
CATEGORY = "MuseV Evolved"
def run(self, images: torch.Tensor, selected_indexes: str):
shape = images.shape
len_first_dim = shape[0]
selected_index: list[int] = []
total_indexes: list[int] = list(range(len_first_dim))
for s in selected_indexes.strip().split(','):
try:
if ":" in s:
_li = s.strip().split(':', maxsplit=1)
_start = _li[0]
_end = _li[1]
if _start and _end:
selected_index.extend(
total_indexes[int(_start):int(_end)]
)
elif _start:
selected_index.extend(
total_indexes[int(_start):]
)
elif _end:
selected_index.extend(
total_indexes[:int(_end)]
)
else:
x: int = int(s.strip())
if x < len_first_dim:
selected_index.append(x)
except:
pass
if selected_index:
print(f"ImageSelector: selected: {len(selected_index)} images")
return (images[selected_index, :, :, :], )
print(f"ImageSelector: selected no images, passthrough")
return (images, )
v1.py
+205
View File
@@ -0,0 +1,205 @@
import os
import folder_paths
import numpy as np
from PIL import Image
import torch
diffusers_path = folder_paths.get_folder_paths("diffusers")[0]
MuseVCheckPointDir = os.path.join(
diffusers_path, "TMElyralab/MuseV"
)
current_dir = os.path.dirname(__file__)
import sys
sys.path.insert(0, current_dir)
sys.path.insert(0, os.path.join(current_dir, "MMCM"))
sys.path.insert(0, os.path.join(current_dir, "diffusers/src"))
sys.path.insert(0, os.path.join(current_dir, "controlnet_aux/src"))
from einops import repeat
from .MMCM.mmcm.utils.seed_util import set_all_seed
from .MMCM.mmcm.utils.task_util import fiss_tasks, generate_tasks as generate_tasks_from_table
from musev.pipelines.pipeline_controlnet_predictor import (
DiffusersPipelinePredictor,
)
from musev.models.unet_loader import load_unet_by_name
from musev import logger
logger.setLevel("INFO")
file_dir = os.path.dirname(__file__)
PROJECT_DIR = file_dir
DATA_DIR = os.path.join(PROJECT_DIR, "data")
class MuseVPredictorV1:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {}
}
RETURN_TYPES = ("MUSEV_PREDICTOR",)
FUNCTION = "main"
CATEGORY = "MuseV Evolved"
def main(self):
sd_model_path = os.path.join(folder_paths.models_dir, 'diffusers/TMElyralab/MuseV/t2i/sd1.5/majicmixRealv6Fp16')
sd_unet_model = os.path.join(folder_paths.models_dir, 'diffusers/TMElyralab/MuseV/motion/musev_referencenet')
vae_path = os.path.join(folder_paths.models_dir, 'diffusers/TMElyralab/MuseV/vae/sd-vae-ft-mse')
negative_embedding = [
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/badhandv4.pt"),
"badhandv4"
],
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/ng_deepnegative_v1_75t.pt"),
"ng_deepnegative_v1_75t"
],
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/EasyNegativeV2.safetensors"),
"EasyNegativeV2"
],
[
os.path.join(folder_paths.models_dir, "diffusers/TMElyralab/MuseV/embedding/bad_prompt_version2-neg.pt"),
"bad_prompt_version2-neg"
]
]
unet = load_unet_by_name(
model_name='musev_referencenet',
sd_unet_model=sd_unet_model,
sd_model=sd_model_path,
cross_attention_dim=768,
need_t2i_facein=False,
strict=True,
need_t2i_ip_adapter_face=False,
)
sd_predictor = DiffusersPipelinePredictor(
sd_model_path=sd_model_path,
unet=unet,
lora_dict=None,
lcm_lora_dct=None,
device='cuda',
dtype=torch.float16,
negative_embedding=negative_embedding,
referencenet=None,
ip_adapter_image_proj=None,
vision_clip_extractor=None,
facein_image_proj=None,
face_emb_extractor=None,
vae_model=vae_path,
ip_adapter_face_emb_extractor=None,
ip_adapter_face_image_proj=None,
)
return (sd_predictor, )
class MuseVImg2VidV1:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE", ),
"musev_predictor": ("MUSEV_PREDICTOR", ),
"time_size": ("INT", {"default":12}),
"seed": ("INT", {"default":1234}),
"video_num_inference_steps": ("INT", {"default":10}),
"video_guidance_scale": ("FLOAT", {"default":3.5, "round": False, "step":0.01}),
"w_ind_noise": ("FLOAT", {"default":0.5, "round": False, "step":0.01}),
"image_weight": ("FLOAT", {"default":0.001, "round": False, "step":0.001}),
"motion_speed": ("FLOAT", {"default":8.0, "round": False, "step":0.01}),
"context_frames": ("INT", {"default":12}),
"context_stride": ("INT", {"default":1}),
"context_overlap": ("INT", {"default":4}),
"output_shift_first_frame": ("BOOLEAN", {"default":True}),
"positive_prompt": ("STRING", {"multiline": True, "default": "(masterpiece, best quality, highres:1),(1girl, solo:1),(beautiful face, soft skin, costume:1),(eye blinks:1.8),(head wave:1.3)"}),
"negative_prompt": ("STRING", {"multiline": True, "default": "badhandv4, ng_deepnegative_v1_75t, (((multiple heads))), (((bad body))), (((two people))), ((extra arms)), ((deformed body)), (((sexy))), paintings,(((two heads))), ((big head)),sketches, (worst quality:2), (low quality:2), (normal quality:2), lowres, ((monochrome)), ((grayscale)), skin spots, acnes, skin blemishes, age spot, glans, (((nsfw))), nipples, extra fingers, (extra legs), (long neck), mutated hands, (fused fingers), (too many fingers)"}),
}
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "main"
CATEGORY = "MuseV Evolved"
def main(
self,
image,
musev_predictor,
time_size,
seed,
w_ind_noise,
context_frames,
context_stride,
context_overlap,
video_num_inference_steps,
video_guidance_scale,
output_shift_first_frame,
positive_prompt,
negative_prompt,
image_weight,
motion_speed,
):
cpu_generator, gpu_generator = set_all_seed(seed)
condition_image = 255.0 * image[0].cpu().numpy()
condition_image = np.clip(condition_image, 0, 255).astype(np.uint8)
condition_image = repeat(condition_image, "h w c-> b c t h w", b=1, t=1)
width = condition_image.shape[4]
height = condition_image.shape[3]
out_videos = musev_predictor.run_pipe_text2video(
video_length=time_size,
prompt=positive_prompt,
width=width,
height=height,
generator=gpu_generator,
noise_type='video_fusion',
negative_prompt=negative_prompt,
video_negative_prompt=negative_prompt,
max_batch_num=1,
strength=0.8,
need_img_based_video_noise=True,
video_num_inference_steps=video_num_inference_steps,
condition_images=condition_image,
fix_condition_images=False,
video_guidance_scale=video_guidance_scale,
guidance_scale=7.5,
num_inference_steps=30,
redraw_condition_image=False,
img_weight=image_weight,
w_ind_noise=w_ind_noise,
n_vision_condition=1,
motion_speed=motion_speed,
need_hist_match=False,
video_guidance_scale_end=None,
video_guidance_scale_method='linear',
vision_condition_latent_index=None,
refer_image=None,
fixed_refer_image=True,
redraw_condition_image_with_referencenet=True,
ip_adapter_image=None,
refer_face_image=None,
fixed_refer_face_image=True,
facein_scale=1.0,
redraw_condition_image_with_facein=True,
ip_adapter_face_scale=1.0,
redraw_condition_image_with_ip_adapter_face=True,
fixed_ip_adapter_image=True,
ip_adapter_scale=1.0,
redraw_condition_image_with_ipdapter=True,
prompt_only_use_image_prompt=False,
# serial_denoise parameter start
record_mid_video_noises=False,
record_mid_video_latents=False,
video_overlap=1,
# serial_denoise parameter end
# parallel_denoise parameter start
context_schedule='uniform_v2',
context_frames=context_frames,
context_stride=context_stride,
context_overlap=context_overlap,
context_batch_size=1,
interpolation_factor=1,
# parallel_denoise parameter end
)
video = torch.from_numpy(out_videos).permute(0,2,3,4,1)
if output_shift_first_frame:
video = video[:, 1:, :, :, :]
return video