188 lines
8.6 KiB
Python
188 lines
8.6 KiB
Python
import os,sys
|
|
now_dir = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.append(now_dir)
|
|
import time
|
|
import folder_paths
|
|
import numpy as np
|
|
from PIL import Image
|
|
from skimage import img_as_ubyte
|
|
from huggingface_hub import snapshot_download
|
|
from fancyvideo.pipelines.fancyvideo_infer_pipeline import InferPipeline
|
|
pretrained_models = os.path.join(folder_paths.models_dir,"AIFSH","FancyVideo")
|
|
models_dir = os.path.join(pretrained_models,"resources","models")
|
|
out_dir = folder_paths.get_output_directory()
|
|
|
|
def load_models():
|
|
if not os.path.exists(os.path.join(models_dir,"fancyvideo_ckpts/vae_3d_61_frames/mp_rank_00_model_states.pt")):
|
|
snapshot_download(repo_id="qihoo360/FancyVideo",local_dir=pretrained_models,
|
|
ignore_patterns=["*md"])
|
|
|
|
|
|
if not os.path.exists(os.path.join(models_dir,"stable-diffusion-v1-5/unet/diffusion_pytorch_model.bin")):
|
|
snapshot_download(repo_id="stable-diffusion-v1-5/stable-diffusion-v1-5",
|
|
local_dir=os.path.join(models_dir,"stable-diffusion-v1-5"),
|
|
ignore_patterns=["v1-5*","*.safetensors","*fp16*","*non_ema*"])
|
|
|
|
class FancyVideoI2VNode:
|
|
def __init__(self):
|
|
load_models()
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required":{
|
|
"first_frame":("IMAGE",),
|
|
"prompt":("TEXT",),
|
|
"base_model":(["realisticVisionV60B1_v51VAE","toonyou_beta3","realcartoon3d_v15","pixarsRendermanInspo_mk1"],),
|
|
"resolution":(["768*768","768*1024","1024*768"],),
|
|
"video_length":([16,32],),
|
|
"fps":("INT",{
|
|
"default":25
|
|
}),
|
|
"cond_motion_score":("FLOAT",{
|
|
"default":3.0
|
|
}),
|
|
"seed":("INT",{
|
|
"default":42
|
|
}),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("VIDEO",)
|
|
#RETURN_NAMES = ("image_output_name",)
|
|
|
|
FUNCTION = "gen_video"
|
|
|
|
#OUTPUT_NODE = False
|
|
|
|
CATEGORY = "AIFSH_FancyVideo"
|
|
|
|
def image2np(self,image,resolution):
|
|
image_np = image.numpy()[0] * 255
|
|
image_np = image_np.astype(np.uint8)
|
|
image = Image.fromarray(image_np)
|
|
image = image.resize((resolution[1],resolution[0]),)
|
|
return np.array(image)
|
|
|
|
def gen_video(self,first_frame,prompt,base_model,resolution,
|
|
video_length,fps,cond_motion_score,seed):
|
|
text_to_video_mm_path = os.path.join(models_dir,f"fancyvideo_ckpts/vae_3d_{61 if video_length==16 else 125}_frames/mp_rank_00_model_states.pt")
|
|
infer_pipeline = InferPipeline(
|
|
text_to_video_mm_path=text_to_video_mm_path,
|
|
base_model_path=os.path.join(models_dir,"sd_v1-5_base_models",f"{base_model}.safetensors"),
|
|
res_adapter_type="res_adapter_v2",
|
|
trained_keys=["motion_modules.", "conv_in.weight", "fps_embedding.", "motion_embedding."],
|
|
model_path=models_dir,
|
|
vae_type="vae_3d",
|
|
use_fps_embedding=True,
|
|
use_motion_embedding=True,
|
|
common_positive_prompt="Best quality, masterpiece, ultra high res, photorealistic, Ultra realistic illustration, hyperrealistic, 8k",
|
|
common_negative_prompt="(low quality:1.3), (worst quality:1.3),poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face,Facial blurring,a large crowd, many people,advertising, information, news, watermark, text, username, signature,out of frame, low res, error, cropped, worst quality, low quality, artifacts, ugly, duplicate, morbid, mutilated, extra fingers, mutated hands, poorly drawn hands, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, nsfw, breast, naked, eroticism",
|
|
)
|
|
|
|
output_fps = fps # the fps to export video
|
|
cond_fps = fps # condition fps
|
|
cond_motion_score = cond_motion_score # condition motion score
|
|
use_noise_scheduler_snr = True
|
|
seed = seed
|
|
|
|
dst_path = os.path.join(out_dir,f"fancyvideo_{time.time_ns()}.mp4")
|
|
resolution_dict = {
|
|
"768*768":(768,768),
|
|
"768*1024":(1024,768),
|
|
"1024*768":(768,1024)
|
|
}
|
|
resolution = resolution_dict[resolution]
|
|
reference_image = self.image2np(first_frame,resolution)
|
|
print(reference_image.shape)
|
|
reference_image, video, prompt = infer_pipeline.t2v_process_one_prompt(prompt=prompt, reference_image=reference_image, seed=seed, video_length=video_length, resolution=resolution, use_noise_scheduler_snr=use_noise_scheduler_snr, fps=cond_fps, motion_score=cond_motion_score,)
|
|
frame_list = []
|
|
for frame in video:
|
|
frame = img_as_ubyte(frame.cpu().permute(1, 2, 0).float().detach().numpy())
|
|
frame_list.append(frame)
|
|
infer_pipeline.save_video(frame_list=frame_list, fps=output_fps, dst_path=dst_path)
|
|
return (dst_path,)
|
|
|
|
class FancyVideoV2VNode:
|
|
|
|
def __init__(self):
|
|
load_models()
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required":{
|
|
"video":("VIDEO",),
|
|
"prompt":("TEXT",),
|
|
"base_model":(["realisticVisionV60B1_v51VAE","toonyou_beta3","realcartoon3d_v15","pixarsRendermanInspo_mk1"],),
|
|
"resolution":(["768*768","768*1024","1024*768"],),
|
|
"infer_mode":(["extending","backtracking"],),
|
|
"fps":("INT",{
|
|
"default":25
|
|
}),
|
|
"cond_motion_score":("FLOAT",{
|
|
"default":3.0
|
|
}),
|
|
"seed":("INT",{
|
|
"default":42
|
|
}),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("VIDEO",)
|
|
#RETURN_NAMES = ("image_output_name",)
|
|
|
|
FUNCTION = "gen_video"
|
|
|
|
#OUTPUT_NODE = False
|
|
|
|
CATEGORY = "AIFSH_FancyVideo"
|
|
|
|
|
|
def gen_video(self,video,prompt,base_model,resolution,
|
|
infer_mode,fps,cond_motion_score,seed):
|
|
text_to_video_mm_path = os.path.join(models_dir,f"fancyvideo_ckpts/video_{infer_mode}/mp_rank_00_model_states.pt")
|
|
infer_pipeline = InferPipeline(
|
|
text_to_video_mm_path=text_to_video_mm_path,
|
|
base_model_path=os.path.join(models_dir,"sd_v1-5_base_models",f"{base_model}.safetensors"),
|
|
res_adapter_type="res_adapter_v2",
|
|
trained_keys=["motion_modules.", "conv_in.weight", "fps_embedding.", "motion_embedding."],
|
|
model_path=models_dir,
|
|
vae_type="vae_3d",
|
|
use_fps_embedding=True,
|
|
use_motion_embedding=True,
|
|
common_positive_prompt="Best quality, masterpiece, ultra high res, photorealistic, Ultra realistic illustration, hyperrealistic, 8k",
|
|
common_negative_prompt="(low quality:1.3), (worst quality:1.3),poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face,Facial blurring,a large crowd, many people,advertising, information, news, watermark, text, username, signature,out of frame, low res, error, cropped, worst quality, low quality, artifacts, ugly, duplicate, morbid, mutilated, extra fingers, mutated hands, poorly drawn hands, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, nsfw, breast, naked, eroticism"
|
|
)
|
|
|
|
output_fps = fps # the fps to export video
|
|
cond_fps = fps # condition fps
|
|
cond_motion_score = cond_motion_score # condition motion score
|
|
use_noise_scheduler_snr = True
|
|
seed = seed
|
|
|
|
dst_path = os.path.join(out_dir,f"fancyvideo_{time.time_ns()}.mp4")
|
|
resolution_dict = {
|
|
"768*768":(768,768),
|
|
"768*1024":(1024,768),
|
|
"1024*768":(768,1024)
|
|
}
|
|
resolution = resolution_dict[resolution]
|
|
|
|
video = infer_pipeline.video_expansion_process_one_prompt(infer_mode="video_"+infer_mode, prompt=prompt, reference_video_path=video, seed=seed, video_length=32, resolution=resolution, use_noise_scheduler_snr=use_noise_scheduler_snr, fps=cond_fps, motion_score=cond_motion_score,)
|
|
frame_list = []
|
|
for frame in video:
|
|
frame = img_as_ubyte(frame.cpu().permute(1, 2, 0).float().detach().numpy())
|
|
frame_list.append(frame)
|
|
infer_pipeline.save_video(frame_list=frame_list, fps=output_fps, dst_path=dst_path)
|
|
return (dst_path,)
|
|
|
|
NODE_CLASS_MAPPINGS = {
|
|
"FancyVideoI2VNode": FancyVideoI2VNode,
|
|
"FancyVideoV2VNode":FancyVideoV2VNode,
|
|
}
|
|
|
|
|
|
|
|
|