Cleanup
This commit is contained in:
kijai
@@ -113,7 +113,8 @@ class DynamiCrafterI2V:
|
||||
if H % 64 != 0:
|
||||
H = H - (H % 64)
|
||||
if orig_H % 64 != 0 or orig_W % 64 != 0:
|
||||
image = comfy.utils.lanczos(image, W, H)
|
||||
image = F.interpolate(image, size=(H, W), mode="bicubic")
|
||||
|
||||
B, C, H, W = image.shape
|
||||
noise_shape = [B, self.model.model.diffusion_model.out_channels, frames, H // 8, W // 8]
|
||||
|
||||
@@ -124,6 +125,8 @@ class DynamiCrafterI2V:
|
||||
if image2 is not None:
|
||||
image2 = image2 * 2 - 1
|
||||
image2 = image2.permute(0, 3, 1, 2).to(dtype).to(device)
|
||||
if image2.shape != image.shape:
|
||||
image2 = F.interpolate(image, size=(H, W), mode="bicubic")
|
||||
z2 = get_latent_z(self.model, image2.unsqueeze(2)) #bc,1,hw
|
||||
img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
|
||||
img_tensor_repeat = torch.zeros_like(img_tensor_repeat)
|
||||
@@ -142,6 +145,7 @@ class DynamiCrafterI2V:
|
||||
cond_images = self.model.embedder(image)
|
||||
img_emb = self.model.image_proj_model(cond_images)
|
||||
imtext_cond = torch.cat([text_emb, img_emb], dim=1)
|
||||
del cond_images, img_emb, text_emb
|
||||
|
||||
fs = torch.tensor([fs], dtype=torch.long, device=self.model.device)
|
||||
cond = {"c_crossattn": [imtext_cond], "c_concat": [img_tensor_repeat]}
|
||||
@@ -187,7 +191,7 @@ class DynamiCrafterI2V:
|
||||
conditioning=cond,
|
||||
batch_size=noise_shape[0],
|
||||
shape=noise_shape[1:],
|
||||
verbose=False,
|
||||
verbose=True,
|
||||
unconditional_guidance_scale=cfg,
|
||||
unconditional_conditioning=uc,
|
||||
eta=eta,
|
||||
@@ -211,11 +215,14 @@ class DynamiCrafterI2V:
|
||||
video = torch.clamp(video.float(), -1., 1.)
|
||||
video = (video + 1.0) / 2.0
|
||||
video = video.squeeze(0).permute(1, 2, 3, 0)
|
||||
del decoded_images, samples
|
||||
|
||||
if not keep_model_loaded:
|
||||
self.model.to('cpu')
|
||||
mm.soft_empty_cache()
|
||||
|
||||
if video.shape[1] != orig_H or video.shape[2] != orig_W:
|
||||
video = F.interpolate(video.permute(0, 3, 1, 2), size=(orig_H, orig_W), mode="bicubic")
|
||||
video = video.permute(0, 2, 3, 1)
|
||||
last_image = video[-1].unsqueeze(0)
|
||||
return (video, last_image)
|
||||
|
||||
@@ -259,8 +266,7 @@ class DynamiCrafterBatchInterpolation:
|
||||
if H % 64 != 0:
|
||||
H = H - (H % 64)
|
||||
if orig_H % 64 != 0 or orig_W % 64 != 0:
|
||||
images = comfy.utils.lanczos(images, W, H)
|
||||
|
||||
images = F.interpolate(images, size=(H, W), mode="bicubic")
|
||||
|
||||
out = []
|
||||
autocast_condition = (dtype != torch.float32) and not comfy.model_management.is_device_mps(device)
|
||||
@@ -359,7 +365,11 @@ class DynamiCrafterBatchInterpolation:
|
||||
self.model.to('cpu')
|
||||
mm.soft_empty_cache()
|
||||
out_video = torch.cat(out, dim=0)
|
||||
|
||||
|
||||
if out_video.shape[1] != orig_H or out_video.shape[2] != orig_W:
|
||||
out_video = F.interpolate(out_video.permute(0, 3, 1, 2), size=(orig_H, orig_W), mode="bicubic")
|
||||
out_video = video.permute(0, 2, 3, 1)
|
||||
|
||||
last_image = out_video[-1].unsqueeze(0)
|
||||
return (out_video, last_image)
|
||||
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
decord>=0.6.0
|
||||
einops>=0.3.0
|
||||
imageio>=2.9.0
|
||||
numpy>=1.24.2
|
||||
omegaconf>=2.1.1
|
||||
opencv_python
|
||||
pandas>=2.0.0
|
||||
Pillow>=9.5.0
|
||||
pytorch_lightning>=1.8.3
|
||||
PyYAML>=6.0
|
||||
|
||||
+2
-173
@@ -1,102 +1,10 @@
|
||||
import os, sys, glob
|
||||
import numpy as np
|
||||
#import sys
|
||||
from collections import OrderedDict
|
||||
from decord import VideoReader, cpu
|
||||
import cv2
|
||||
|
||||
import torch
|
||||
import torchvision
|
||||
sys.path.insert(1, os.path.join(sys.path[0], '..', '..'))
|
||||
from ...lvdm.models.samplers.ddim import DDIMSampler
|
||||
#sys.path.insert(1, os.path.join(sys.path[0], '..', '..'))
|
||||
from einops import rearrange
|
||||
|
||||
|
||||
def batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=50, ddim_eta=1.0,\
|
||||
cfg_scale=1.0, temporal_cfg_scale=None, **kwargs):
|
||||
ddim_sampler = DDIMSampler(model)
|
||||
uncond_type = model.uncond_type
|
||||
batch_size = noise_shape[0]
|
||||
fs = cond["fs"]
|
||||
del cond["fs"]
|
||||
if noise_shape[-1] == 32:
|
||||
timestep_spacing = "uniform"
|
||||
guidance_rescale = 0.0
|
||||
else:
|
||||
timestep_spacing = "uniform_trailing"
|
||||
guidance_rescale = 0.7
|
||||
## construct unconditional guidance
|
||||
if cfg_scale != 1.0:
|
||||
if uncond_type == "empty_seq":
|
||||
prompts = batch_size * [""]
|
||||
#prompts = N * T * [""] ## if is_imgbatch=True
|
||||
uc_emb = model.get_learned_conditioning(prompts)
|
||||
elif uncond_type == "zero_embed":
|
||||
c_emb = cond["c_crossattn"][0] if isinstance(cond, dict) else cond
|
||||
uc_emb = torch.zeros_like(c_emb)
|
||||
|
||||
## process image embedding token
|
||||
if hasattr(model, 'embedder'):
|
||||
uc_img = torch.zeros(noise_shape[0],3,224,224).to(model.device)
|
||||
## img: b c h w >> b l c
|
||||
uc_img = model.embedder(uc_img)
|
||||
uc_img = model.image_proj_model(uc_img)
|
||||
uc_emb = torch.cat([uc_emb, uc_img], dim=1)
|
||||
|
||||
if isinstance(cond, dict):
|
||||
uc = {key:cond[key] for key in cond.keys()}
|
||||
uc.update({'c_crossattn': [uc_emb]})
|
||||
else:
|
||||
uc = uc_emb
|
||||
else:
|
||||
uc = None
|
||||
|
||||
x_T = None
|
||||
batch_variants = []
|
||||
|
||||
for _ in range(n_samples):
|
||||
if ddim_sampler is not None:
|
||||
kwargs.update({"clean_cond": True})
|
||||
samples, _ = ddim_sampler.sample(S=ddim_steps,
|
||||
conditioning=cond,
|
||||
batch_size=noise_shape[0],
|
||||
shape=noise_shape[1:],
|
||||
verbose=False,
|
||||
unconditional_guidance_scale=cfg_scale,
|
||||
unconditional_conditioning=uc,
|
||||
eta=ddim_eta,
|
||||
temporal_length=noise_shape[2],
|
||||
conditional_guidance_scale_temporal=temporal_cfg_scale,
|
||||
x_T=x_T,
|
||||
fs=fs,
|
||||
timestep_spacing=timestep_spacing,
|
||||
guidance_rescale=guidance_rescale,
|
||||
**kwargs
|
||||
)
|
||||
## reconstruct from latent to pixel space
|
||||
batch_images = model.decode_first_stage(samples)
|
||||
batch_variants.append(batch_images)
|
||||
## batch, <samples>, c, t, h, w
|
||||
batch_variants = torch.stack(batch_variants, dim=1)
|
||||
return batch_variants
|
||||
|
||||
|
||||
def get_filelist(data_dir, ext='*'):
|
||||
file_list = glob.glob(os.path.join(data_dir, '*.%s'%ext))
|
||||
file_list.sort()
|
||||
return file_list
|
||||
|
||||
def get_dirlist(path):
|
||||
list = []
|
||||
if (os.path.exists(path)):
|
||||
files = os.listdir(path)
|
||||
for file in files:
|
||||
m = os.path.join(path,file)
|
||||
if (os.path.isdir(m)):
|
||||
list.append(m)
|
||||
list.sort()
|
||||
return list
|
||||
|
||||
|
||||
def load_model_checkpoint(model, ckpt):
|
||||
def load_checkpoint(model, ckpt, full_strict):
|
||||
state_dict = torch.load(ckpt, map_location="cpu")
|
||||
@@ -139,85 +47,6 @@ def load_prompts(prompt_file):
|
||||
f.close()
|
||||
return prompt_list
|
||||
|
||||
|
||||
def load_video_batch(filepath_list, frame_stride, video_size=(256,256), video_frames=16):
|
||||
'''
|
||||
Notice about some special cases:
|
||||
1. video_frames=-1 means to take all the frames (with fs=1)
|
||||
2. when the total video frames is less than required, padding strategy will be used (repreated last frame)
|
||||
'''
|
||||
fps_list = []
|
||||
batch_tensor = []
|
||||
assert frame_stride > 0, "valid frame stride should be a positive interge!"
|
||||
for filepath in filepath_list:
|
||||
padding_num = 0
|
||||
vidreader = VideoReader(filepath, ctx=cpu(0), width=video_size[1], height=video_size[0])
|
||||
fps = vidreader.get_avg_fps()
|
||||
total_frames = len(vidreader)
|
||||
max_valid_frames = (total_frames-1) // frame_stride + 1
|
||||
if video_frames < 0:
|
||||
## all frames are collected: fs=1 is a must
|
||||
required_frames = total_frames
|
||||
frame_stride = 1
|
||||
else:
|
||||
required_frames = video_frames
|
||||
query_frames = min(required_frames, max_valid_frames)
|
||||
frame_indices = [frame_stride*i for i in range(query_frames)]
|
||||
|
||||
## [t,h,w,c] -> [c,t,h,w]
|
||||
frames = vidreader.get_batch(frame_indices)
|
||||
frame_tensor = torch.tensor(frames.asnumpy()).permute(3, 0, 1, 2).float()
|
||||
frame_tensor = (frame_tensor / 255. - 0.5) * 2
|
||||
if max_valid_frames < required_frames:
|
||||
padding_num = required_frames - max_valid_frames
|
||||
frame_tensor = torch.cat([frame_tensor, *([frame_tensor[:,-1:,:,:]]*padding_num)], dim=1)
|
||||
print(f'{os.path.split(filepath)[1]} is not long enough: {padding_num} frames padded.')
|
||||
batch_tensor.append(frame_tensor)
|
||||
sample_fps = int(fps/frame_stride)
|
||||
fps_list.append(sample_fps)
|
||||
|
||||
return torch.stack(batch_tensor, dim=0)
|
||||
|
||||
from PIL import Image
|
||||
def load_image_batch(filepath_list, image_size=(256,256)):
|
||||
batch_tensor = []
|
||||
for filepath in filepath_list:
|
||||
_, filename = os.path.split(filepath)
|
||||
_, ext = os.path.splitext(filename)
|
||||
if ext == '.mp4':
|
||||
vidreader = VideoReader(filepath, ctx=cpu(0), width=image_size[1], height=image_size[0])
|
||||
frame = vidreader.get_batch([0])
|
||||
img_tensor = torch.tensor(frame.asnumpy()).squeeze(0).permute(2, 0, 1).float()
|
||||
elif ext == '.png' or ext == '.jpg':
|
||||
img = Image.open(filepath).convert("RGB")
|
||||
rgb_img = np.array(img, np.float32)
|
||||
#bgr_img = cv2.imread(filepath, cv2.IMREAD_COLOR)
|
||||
#bgr_img = cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB)
|
||||
rgb_img = cv2.resize(rgb_img, (image_size[1],image_size[0]), interpolation=cv2.INTER_LINEAR)
|
||||
img_tensor = torch.from_numpy(rgb_img).permute(2, 0, 1).float()
|
||||
else:
|
||||
print(f'ERROR: <{ext}> image loading only support format: [mp4], [png], [jpg]')
|
||||
raise NotImplementedError
|
||||
img_tensor = (img_tensor / 255. - 0.5) * 2
|
||||
batch_tensor.append(img_tensor)
|
||||
return torch.stack(batch_tensor, dim=0)
|
||||
|
||||
|
||||
def save_videos(batch_tensors, savedir, filenames, fps=10):
|
||||
# b,samples,c,t,h,w
|
||||
n_samples = batch_tensors.shape[1]
|
||||
for idx, vid_tensor in enumerate(batch_tensors):
|
||||
video = vid_tensor.detach().cpu()
|
||||
video = torch.clamp(video.float(), -1., 1.)
|
||||
video = video.permute(2, 0, 1, 3, 4) # t,n,c,h,w
|
||||
frame_grids = [torchvision.utils.make_grid(framesheet, nrow=int(n_samples)) for framesheet in video] #[3, 1*h, n*w]
|
||||
grid = torch.stack(frame_grids, dim=0) # stack in temporal dim [t, 3, n*h, w]
|
||||
grid = (grid + 1.0) / 2.0
|
||||
grid = (grid * 255).to(torch.uint8).permute(0, 2, 3, 1)
|
||||
savepath = os.path.join(savedir, f"{filenames[idx]}.mp4")
|
||||
torchvision.io.write_video(savepath, grid, fps=fps, video_codec='h264', options={'crf': '10'})
|
||||
|
||||
|
||||
def get_latent_z(model, videos):
|
||||
b, c, t, h, w = videos.shape
|
||||
x = rearrange(videos, 'b c t h w -> (b t) c h w')
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
from omegaconf import OmegaConf
|
||||
import torch
|
||||
from scripts.evaluation.funcs import load_model_checkpoint, save_videos, batch_ddim_sampling, get_latent_z
|
||||
from utils.utils import instantiate_from_config
|
||||
from huggingface_hub import hf_hub_download
|
||||
from einops import repeat
|
||||
import torchvision.transforms as transforms
|
||||
from pytorch_lightning import seed_everything
|
||||
|
||||
|
||||
class Image2Video():
|
||||
def __init__(self,result_dir='./tmp/',gpu_num=1,resolution='256_256') -> None:
|
||||
self.resolution = (int(resolution.split('_')[0]), int(resolution.split('_')[1])) #hw
|
||||
self.download_model()
|
||||
|
||||
self.result_dir = result_dir
|
||||
if not os.path.exists(self.result_dir):
|
||||
os.mkdir(self.result_dir)
|
||||
ckpt_path='checkpoints/dynamicrafter_'+resolution.split('_')[1]+'_v1/model.ckpt'
|
||||
config_file='configs/inference_'+resolution.split('_')[1]+'_v1.0.yaml'
|
||||
config = OmegaConf.load(config_file)
|
||||
model_config = config.pop("model", OmegaConf.create())
|
||||
model_config['params']['unet_config']['params']['use_checkpoint']=False
|
||||
model_list = []
|
||||
for gpu_id in range(gpu_num):
|
||||
model = instantiate_from_config(model_config)
|
||||
# model = model.cuda(gpu_id)
|
||||
assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
|
||||
model = load_model_checkpoint(model, ckpt_path)
|
||||
model.eval().half()
|
||||
model_list.append(model)
|
||||
self.model_list = model_list
|
||||
self.save_fps = 8
|
||||
|
||||
def get_image(self, image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123):
|
||||
seed_everything(seed)
|
||||
transform = transforms.Compose([
|
||||
transforms.Resize(min(self.resolution)),
|
||||
transforms.CenterCrop(self.resolution),
|
||||
])
|
||||
torch.cuda.empty_cache()
|
||||
print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
|
||||
start = time.time()
|
||||
gpu_id=0
|
||||
if steps > 60:
|
||||
steps = 60
|
||||
model = self.model_list[gpu_id]
|
||||
model = model.cuda()
|
||||
batch_size=1
|
||||
channels = model.model.diffusion_model.out_channels
|
||||
frames = model.temporal_length
|
||||
h, w = self.resolution[0] // 8, self.resolution[1] // 8
|
||||
noise_shape = [batch_size, channels, frames, h, w]
|
||||
|
||||
# text cond
|
||||
with torch.no_grad(), torch.cuda.amp.autocast():
|
||||
text_emb = model.get_learned_conditioning([prompt])
|
||||
|
||||
# img cond
|
||||
img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
|
||||
img_tensor = (img_tensor / 255. - 0.5) * 2
|
||||
|
||||
image_tensor_resized = transform(img_tensor) #3,h,w
|
||||
videos = image_tensor_resized.unsqueeze(0) # bchw
|
||||
|
||||
z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw
|
||||
|
||||
img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
|
||||
|
||||
cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc
|
||||
img_emb = model.image_proj_model(cond_images)
|
||||
|
||||
imtext_cond = torch.cat([text_emb, img_emb], dim=1)
|
||||
|
||||
fs = torch.tensor([fs], dtype=torch.long, device=model.device)
|
||||
cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]}
|
||||
|
||||
## inference
|
||||
batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
|
||||
## b,samples,c,t,h,w
|
||||
prompt_str = prompt.replace("/", "_slash_") if "/" in prompt else prompt
|
||||
prompt_str = prompt_str.replace(" ", "_") if " " in prompt else prompt_str
|
||||
prompt_str=prompt_str[:40]
|
||||
if len(prompt_str) == 0:
|
||||
prompt_str = 'empty_prompt'
|
||||
|
||||
save_videos(batch_samples, self.result_dir, filenames=[prompt_str], fps=self.save_fps)
|
||||
print(f"Saved in {prompt_str}. Time used: {(time.time() - start):.2f} seconds")
|
||||
model = model.cpu()
|
||||
return os.path.join(self.result_dir, f"{prompt_str}.mp4")
|
||||
|
||||
def download_model(self):
|
||||
REPO_ID = 'Doubiiu/DynamiCrafter_'+str(self.resolution[1]) if self.resolution[1]!=256 else 'Doubiiu/DynamiCrafter'
|
||||
filename_list = ['model.ckpt']
|
||||
if not os.path.exists('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/'):
|
||||
os.makedirs('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/')
|
||||
for filename in filename_list:
|
||||
local_file = os.path.join('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/', filename)
|
||||
if not os.path.exists(local_file):
|
||||
hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/', local_dir_use_symlinks=False)
|
||||
|
||||
if __name__ == '__main__':
|
||||
i2v = Image2Video()
|
||||
video_path = i2v.get_image('prompts/art.png','man fishing in a boat at sunset')
|
||||
print('done', video_path)
|
||||
@@ -1,131 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
from omegaconf import OmegaConf
|
||||
import torch
|
||||
from scripts.evaluation.funcs import load_model_checkpoint, save_videos, batch_ddim_sampling, get_latent_z
|
||||
from utils.utils import instantiate_from_config
|
||||
from huggingface_hub import hf_hub_download
|
||||
from einops import repeat
|
||||
import torchvision.transforms as transforms
|
||||
from pytorch_lightning import seed_everything
|
||||
|
||||
|
||||
class Image2Video():
|
||||
def __init__(self,result_dir='./tmp/',gpu_num=1,resolution='256_256') -> None:
|
||||
self.resolution = (int(resolution.split('_')[0]), int(resolution.split('_')[1])) #hw
|
||||
self.download_model()
|
||||
|
||||
self.result_dir = result_dir
|
||||
if not os.path.exists(self.result_dir):
|
||||
os.mkdir(self.result_dir)
|
||||
ckpt_path='checkpoints/dynamicrafter_'+resolution.split('_')[1]+'_interp_v1/model.ckpt'
|
||||
config_file='configs/inference_'+resolution.split('_')[1]+'_v1.0.yaml'
|
||||
config = OmegaConf.load(config_file)
|
||||
model_config = config.pop("model", OmegaConf.create())
|
||||
model_config['params']['unet_config']['params']['use_checkpoint']=False
|
||||
model_list = []
|
||||
for gpu_id in range(gpu_num):
|
||||
model = instantiate_from_config(model_config)
|
||||
# model = model.cuda(gpu_id)
|
||||
assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
|
||||
model = load_model_checkpoint(model, ckpt_path)
|
||||
model.eval()
|
||||
model_list.append(model)
|
||||
self.model_list = model_list
|
||||
self.save_fps = 8
|
||||
|
||||
def get_image(self, image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123, image2=None):
|
||||
seed_everything(seed)
|
||||
transform = transforms.Compose([
|
||||
transforms.Resize(min(self.resolution)),
|
||||
transforms.CenterCrop(self.resolution),
|
||||
])
|
||||
torch.cuda.empty_cache()
|
||||
print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
|
||||
start = time.time()
|
||||
gpu_id=0
|
||||
if steps > 60:
|
||||
steps = 60
|
||||
model = self.model_list[gpu_id]
|
||||
model = model.cuda()
|
||||
batch_size=1
|
||||
channels = model.model.diffusion_model.out_channels
|
||||
frames = model.temporal_length
|
||||
h, w = self.resolution[0] // 8, self.resolution[1] // 8
|
||||
noise_shape = [batch_size, channels, frames, h, w]
|
||||
|
||||
# text cond
|
||||
with torch.no_grad(), torch.cuda.amp.autocast():
|
||||
text_emb = model.get_learned_conditioning([prompt])
|
||||
|
||||
# img cond
|
||||
img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
|
||||
img_tensor = (img_tensor / 255. - 0.5) * 2
|
||||
|
||||
image_tensor_resized = transform(img_tensor) #3,h,w
|
||||
videos = image_tensor_resized.unsqueeze(0) # bchw
|
||||
|
||||
z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw
|
||||
|
||||
|
||||
if image2 is not None:
|
||||
img_tensor2 = torch.from_numpy(image2).permute(2, 0, 1).float().to(model.device)
|
||||
img_tensor2 = (img_tensor2 / 255. - 0.5) * 2
|
||||
|
||||
image_tensor_resized2 = transform(img_tensor2) #3,h,w
|
||||
videos2 = image_tensor_resized2.unsqueeze(0) # bchw
|
||||
|
||||
z2 = get_latent_z(model, videos2.unsqueeze(2)) #bc,1,hw
|
||||
|
||||
img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
|
||||
|
||||
img_tensor_repeat = torch.zeros_like(img_tensor_repeat)
|
||||
|
||||
## old
|
||||
img_tensor_repeat[:,:,:1,:,:] = z
|
||||
if image2 is not None:
|
||||
img_tensor_repeat[:,:,-1:,:,:] = z2
|
||||
else:
|
||||
img_tensor_repeat[:,:,-1:,:,:] = z
|
||||
|
||||
|
||||
cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc
|
||||
img_emb = model.image_proj_model(cond_images)
|
||||
|
||||
imtext_cond = torch.cat([text_emb, img_emb], dim=1)
|
||||
|
||||
fs = torch.tensor([fs], dtype=torch.long, device=model.device)
|
||||
cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]}
|
||||
|
||||
## inference
|
||||
batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
|
||||
|
||||
## remove the last frame
|
||||
if image2 is None:
|
||||
batch_samples = batch_samples[:,:,:,:-1,...]
|
||||
## b,samples,c,t,h,w
|
||||
prompt_str = prompt.replace("/", "_slash_") if "/" in prompt else prompt
|
||||
prompt_str = prompt_str.replace(" ", "_") if " " in prompt else prompt_str
|
||||
prompt_str=prompt_str[:40]
|
||||
if len(prompt_str) == 0:
|
||||
prompt_str = 'empty_prompt'
|
||||
|
||||
save_videos(batch_samples, self.result_dir, filenames=[prompt_str], fps=self.save_fps)
|
||||
print(f"Saved in {prompt_str}. Time used: {(time.time() - start):.2f} seconds")
|
||||
model = model.cpu()
|
||||
return os.path.join(self.result_dir, f"{prompt_str}.mp4")
|
||||
|
||||
def download_model(self):
|
||||
REPO_ID = 'Doubiiu/DynamiCrafter_'+str(self.resolution[1])+'_Interp'
|
||||
filename_list = ['model.ckpt']
|
||||
if not os.path.exists('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/'):
|
||||
os.makedirs('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/')
|
||||
for filename in filename_list:
|
||||
local_file = os.path.join('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/', filename)
|
||||
if not os.path.exists(local_file):
|
||||
hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/', local_dir_use_symlinks=False)
|
||||
|
||||
if __name__ == '__main__':
|
||||
i2v = Image2Video()
|
||||
video_path = i2v.get_image('prompts/art.png','man fishing in a boat at sunset')
|
||||
print('done', video_path)
|
||||
@@ -1,61 +0,0 @@
|
||||
version=$1 ##1024, 512, 256
|
||||
seed=123
|
||||
name=dynamicrafter_$1_seed${seed}
|
||||
|
||||
ckpt=checkpoints/dynamicrafter_$1_v1/model.ckpt
|
||||
config=configs/inference_$1_v1.0.yaml
|
||||
|
||||
prompt_dir=prompts/$1/
|
||||
res_dir="results"
|
||||
|
||||
if [ "$1" == "256" ]; then
|
||||
H=256
|
||||
FS=3 ## This model adopts frame stride=3, range recommended: 1-6 (larger value -> larger motion)
|
||||
elif [ "$1" == "512" ]; then
|
||||
H=320
|
||||
FS=24 ## This model adopts FPS=24, range recommended: 15-30 (smaller value -> larger motion)
|
||||
elif [ "$1" == "1024" ]; then
|
||||
H=576
|
||||
FS=10 ## This model adopts FPS=10, range recommended: 15-5 (smaller value -> larger motion)
|
||||
else
|
||||
echo "Invalid input. Please enter 256, 512, or 1024."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$1" == "256" ]; then
|
||||
CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \
|
||||
--seed ${seed} \
|
||||
--ckpt_path $ckpt \
|
||||
--config $config \
|
||||
--savedir $res_dir/$name \
|
||||
--n_samples 1 \
|
||||
--bs 1 --height ${H} --width $1 \
|
||||
--unconditional_guidance_scale 7.5 \
|
||||
--ddim_steps 50 \
|
||||
--ddim_eta 1.0 \
|
||||
--prompt_dir $prompt_dir \
|
||||
--text_input \
|
||||
--video_length 16 \
|
||||
--frame_stride ${FS}
|
||||
else
|
||||
CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \
|
||||
--seed ${seed} \
|
||||
--ckpt_path $ckpt \
|
||||
--config $config \
|
||||
--savedir $res_dir/$name \
|
||||
--n_samples 1 \
|
||||
--bs 1 --height ${H} --width $1 \
|
||||
--unconditional_guidance_scale 7.5 \
|
||||
--ddim_steps 50 \
|
||||
--ddim_eta 1.0 \
|
||||
--prompt_dir $prompt_dir \
|
||||
--text_input \
|
||||
--video_length 16 \
|
||||
--frame_stride ${FS} \
|
||||
--timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae
|
||||
fi
|
||||
|
||||
|
||||
## multi-cond CFG: the <unconditional_guidance_scale> is s_txt, <cfg_img> is s_img
|
||||
#--multiple_cond_cfg --cfg_img 7.5
|
||||
#--loop
|
||||
@@ -1,47 +0,0 @@
|
||||
version=$1 # interp or loop
|
||||
ckpt=checkpoints/dynamicrafter_512_interp_v1/model.ckpt
|
||||
config=configs/inference_512_v1.0.yaml
|
||||
|
||||
prompt_dir=prompts/512_$1/
|
||||
res_dir="results"
|
||||
|
||||
FS=5 ## This model adopts FPS=5, range recommended: 5-30 (smaller value -> larger motion)
|
||||
|
||||
|
||||
if [ "$1" == "interp" ]; then
|
||||
seed=12306
|
||||
name=dynamicrafter_512_$1_seed${seed}
|
||||
CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \
|
||||
--seed ${seed} \
|
||||
--ckpt_path $ckpt \
|
||||
--config $config \
|
||||
--savedir $res_dir/$name \
|
||||
--n_samples 1 \
|
||||
--bs 1 --height 320 --width 512 \
|
||||
--unconditional_guidance_scale 7.5 \
|
||||
--ddim_steps 50 \
|
||||
--ddim_eta 1.0 \
|
||||
--prompt_dir $prompt_dir \
|
||||
--text_input \
|
||||
--video_length 16 \
|
||||
--frame_stride ${FS} \
|
||||
--timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae --interp
|
||||
else
|
||||
seed=234
|
||||
name=dynamicrafter_512_$1_seed${seed}
|
||||
CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \
|
||||
--seed ${seed} \
|
||||
--ckpt_path $ckpt \
|
||||
--config $config \
|
||||
--savedir $res_dir/$name \
|
||||
--n_samples 1 \
|
||||
--bs 1 --height 320 --width 512 \
|
||||
--unconditional_guidance_scale 7.5 \
|
||||
--ddim_steps 50 \
|
||||
--ddim_eta 1.0 \
|
||||
--prompt_dir $prompt_dir \
|
||||
--text_input \
|
||||
--video_length 16 \
|
||||
--frame_stride ${FS} \
|
||||
--timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae --loop
|
||||
fi
|
||||
@@ -1,102 +0,0 @@
|
||||
version=$1 ##1024, 512, 256
|
||||
seed=123
|
||||
|
||||
name=dynamicrafter_$1_mp_seed${seed}
|
||||
|
||||
ckpt=checkpoints/dynamicrafter_$1_v1/model.ckpt
|
||||
config=configs/inference_$1_v1.0.yaml
|
||||
|
||||
prompt_dir=prompts/$1/
|
||||
res_dir="results"
|
||||
|
||||
if [ "$1" == "256" ]; then
|
||||
H=256
|
||||
FS=3 ## This model adopts frame stride=3
|
||||
elif [ "$1" == "512" ]; then
|
||||
H=320
|
||||
FS=24 ## This model adopts FPS=24
|
||||
elif [ "$1" == "1024" ]; then
|
||||
H=576
|
||||
FS=10 ## This model adopts FPS=10
|
||||
else
|
||||
echo "Invalid input. Please enter 256, 512, or 1024."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# if [ "$1" == "256" ]; then
|
||||
# CUDA_VISIBLE_DEVICES=2 python3 scripts/evaluation/inference.py \
|
||||
# --seed 123 \
|
||||
# --ckpt_path $ckpt \
|
||||
# --config $config \
|
||||
# --savedir $res_dir/$name \
|
||||
# --n_samples 1 \
|
||||
# --bs 1 --height ${H} --width $1 \
|
||||
# --unconditional_guidance_scale 7.5 \
|
||||
# --ddim_steps 50 \
|
||||
# --ddim_eta 1.0 \
|
||||
# --prompt_dir $prompt_dir \
|
||||
# --text_input \
|
||||
# --video_length 16 \
|
||||
# --frame_stride ${FS}
|
||||
# else
|
||||
# CUDA_VISIBLE_DEVICES=2 python3 scripts/evaluation/inference.py \
|
||||
# --seed 123 \
|
||||
# --ckpt_path $ckpt \
|
||||
# --config $config \
|
||||
# --savedir $res_dir/$name \
|
||||
# --n_samples 1 \
|
||||
# --bs 1 --height ${H} --width $1 \
|
||||
# --unconditional_guidance_scale 7.5 \
|
||||
# --ddim_steps 50 \
|
||||
# --ddim_eta 1.0 \
|
||||
# --prompt_dir $prompt_dir \
|
||||
# --text_input \
|
||||
# --video_length 16 \
|
||||
# --frame_stride ${FS} \
|
||||
# --timestep_spacing 'uniform_trailing' --guidance_rescale 0.7
|
||||
# fi
|
||||
|
||||
|
||||
## multi-cond CFG: the <unconditional_guidance_scale> is s_txt, <cfg_img> is s_img
|
||||
#--multiple_cond_cfg --cfg_img 7.5
|
||||
#--loop
|
||||
|
||||
## inference using single node with multi-GPUs:
|
||||
if [ "$1" == "256" ]; then
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3 -m torch.distributed.launch \
|
||||
--nproc_per_node=8 --nnodes=1 --master_addr=127.0.0.1 --master_port=23456 --node_rank=0 \
|
||||
scripts/evaluation/ddp_wrapper.py \
|
||||
--module 'inference' \
|
||||
--seed ${seed} \
|
||||
--ckpt_path $ckpt \
|
||||
--config $config \
|
||||
--savedir $res_dir/$name \
|
||||
--n_samples 1 \
|
||||
--bs 1 --height ${H} --width $1 \
|
||||
--unconditional_guidance_scale 7.5 \
|
||||
--ddim_steps 50 \
|
||||
--ddim_eta 1.0 \
|
||||
--prompt_dir $prompt_dir \
|
||||
--text_input \
|
||||
--video_length 16 \
|
||||
--frame_stride ${FS}
|
||||
else
|
||||
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3 -m torch.distributed.launch \
|
||||
--nproc_per_node=8 --nnodes=1 --master_addr=127.0.0.1 --master_port=23456 --node_rank=0 \
|
||||
scripts/evaluation/ddp_wrapper.py \
|
||||
--module 'inference' \
|
||||
--seed ${seed} \
|
||||
--ckpt_path $ckpt \
|
||||
--config $config \
|
||||
--savedir $res_dir/$name \
|
||||
--n_samples 1 \
|
||||
--bs 1 --height ${H} --width $1 \
|
||||
--unconditional_guidance_scale 7.5 \
|
||||
--ddim_steps 50 \
|
||||
--ddim_eta 1.0 \
|
||||
--prompt_dir $prompt_dir \
|
||||
--text_input \
|
||||
--video_length 16 \
|
||||
--frame_stride ${FS} \
|
||||
--timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae
|
||||
fi
|
||||
@@ -1,11 +1,9 @@
|
||||
import importlib
|
||||
import numpy as np
|
||||
import cv2
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import os
|
||||
|
||||
|
||||
def count_params(model, verbose=False):
|
||||
total_params = sum(p.numel() for p in model.parameters())
|
||||
if verbose:
|
||||
@@ -49,26 +47,11 @@ def load_npz_from_dir(data_dir):
|
||||
data = np.concatenate(data, axis=0)
|
||||
return data
|
||||
|
||||
|
||||
def load_npz_from_paths(data_paths):
|
||||
data = [np.load(data_path)['arr_0'] for data_path in data_paths]
|
||||
data = np.concatenate(data, axis=0)
|
||||
return data
|
||||
|
||||
|
||||
def resize_numpy_image(image, max_resolution=512 * 512, resize_short_edge=None):
|
||||
h, w = image.shape[:2]
|
||||
if resize_short_edge is not None:
|
||||
k = resize_short_edge / min(h, w)
|
||||
else:
|
||||
k = max_resolution / (h * w)
|
||||
k = k**0.5
|
||||
h = int(np.round(h * k / 64)) * 64
|
||||
w = int(np.round(w * k / 64)) * 64
|
||||
image = cv2.resize(image, (w, h), interpolation=cv2.INTER_LANCZOS4)
|
||||
return image
|
||||
|
||||
|
||||
def setup_dist(args):
|
||||
if dist.is_initialized():
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user