From 1314f82e0be46170136e57cd15b35e6e03ac69e4 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Sun, 17 Mar 2024 00:52:44 +0200 Subject: [PATCH] Cleanup --- nodes.py | 22 +++- requirements.txt | 4 - scripts/evaluation/funcs.py | 175 +------------------------ scripts/gradio/i2v_test.py | 107 --------------- scripts/gradio/i2v_test_application.py | 131 ------------------ scripts/run.sh | 61 --------- scripts/run_application.sh | 47 ------- scripts/run_mp.sh | 102 -------------- utils/utils.py | 17 --- 9 files changed, 18 insertions(+), 648 deletions(-) delete mode 100644 scripts/gradio/i2v_test.py delete mode 100644 scripts/gradio/i2v_test_application.py delete mode 100644 scripts/run.sh delete mode 100644 scripts/run_application.sh delete mode 100644 scripts/run_mp.sh diff --git a/nodes.py b/nodes.py index debc2d1..65edbe2 100644 --- a/nodes.py +++ b/nodes.py @@ -113,7 +113,8 @@ class DynamiCrafterI2V: if H % 64 != 0: H = H - (H % 64) if orig_H % 64 != 0 or orig_W % 64 != 0: - image = comfy.utils.lanczos(image, W, H) + image = F.interpolate(image, size=(H, W), mode="bicubic") + B, C, H, W = image.shape noise_shape = [B, self.model.model.diffusion_model.out_channels, frames, H // 8, W // 8] @@ -124,6 +125,8 @@ class DynamiCrafterI2V: if image2 is not None: image2 = image2 * 2 - 1 image2 = image2.permute(0, 3, 1, 2).to(dtype).to(device) + if image2.shape != image.shape: + image2 = F.interpolate(image, size=(H, W), mode="bicubic") z2 = get_latent_z(self.model, image2.unsqueeze(2)) #bc,1,hw img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames) img_tensor_repeat = torch.zeros_like(img_tensor_repeat) @@ -142,6 +145,7 @@ class DynamiCrafterI2V: cond_images = self.model.embedder(image) img_emb = self.model.image_proj_model(cond_images) imtext_cond = torch.cat([text_emb, img_emb], dim=1) + del cond_images, img_emb, text_emb fs = torch.tensor([fs], dtype=torch.long, device=self.model.device) cond = {"c_crossattn": [imtext_cond], "c_concat": [img_tensor_repeat]} @@ -187,7 +191,7 @@ class DynamiCrafterI2V: conditioning=cond, batch_size=noise_shape[0], shape=noise_shape[1:], - verbose=False, + verbose=True, unconditional_guidance_scale=cfg, unconditional_conditioning=uc, eta=eta, @@ -211,11 +215,14 @@ class DynamiCrafterI2V: video = torch.clamp(video.float(), -1., 1.) video = (video + 1.0) / 2.0 video = video.squeeze(0).permute(1, 2, 3, 0) + del decoded_images, samples if not keep_model_loaded: self.model.to('cpu') mm.soft_empty_cache() - + if video.shape[1] != orig_H or video.shape[2] != orig_W: + video = F.interpolate(video.permute(0, 3, 1, 2), size=(orig_H, orig_W), mode="bicubic") + video = video.permute(0, 2, 3, 1) last_image = video[-1].unsqueeze(0) return (video, last_image) @@ -259,8 +266,7 @@ class DynamiCrafterBatchInterpolation: if H % 64 != 0: H = H - (H % 64) if orig_H % 64 != 0 or orig_W % 64 != 0: - images = comfy.utils.lanczos(images, W, H) - + images = F.interpolate(images, size=(H, W), mode="bicubic") out = [] autocast_condition = (dtype != torch.float32) and not comfy.model_management.is_device_mps(device) @@ -359,7 +365,11 @@ class DynamiCrafterBatchInterpolation: self.model.to('cpu') mm.soft_empty_cache() out_video = torch.cat(out, dim=0) - + + if out_video.shape[1] != orig_H or out_video.shape[2] != orig_W: + out_video = F.interpolate(out_video.permute(0, 3, 1, 2), size=(orig_H, orig_W), mode="bicubic") + out_video = video.permute(0, 2, 3, 1) + last_image = out_video[-1].unsqueeze(0) return (out_video, last_image) diff --git a/requirements.txt b/requirements.txt index 1d230a1..36b783e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,6 @@ -decord>=0.6.0 einops>=0.3.0 -imageio>=2.9.0 numpy>=1.24.2 omegaconf>=2.1.1 -opencv_python -pandas>=2.0.0 Pillow>=9.5.0 pytorch_lightning>=1.8.3 PyYAML>=6.0 diff --git a/scripts/evaluation/funcs.py b/scripts/evaluation/funcs.py index 63088ca..f1c330d 100644 --- a/scripts/evaluation/funcs.py +++ b/scripts/evaluation/funcs.py @@ -1,102 +1,10 @@ -import os, sys, glob -import numpy as np +#import sys from collections import OrderedDict -from decord import VideoReader, cpu -import cv2 - import torch -import torchvision -sys.path.insert(1, os.path.join(sys.path[0], '..', '..')) -from ...lvdm.models.samplers.ddim import DDIMSampler +#sys.path.insert(1, os.path.join(sys.path[0], '..', '..')) from einops import rearrange -def batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=50, ddim_eta=1.0,\ - cfg_scale=1.0, temporal_cfg_scale=None, **kwargs): - ddim_sampler = DDIMSampler(model) - uncond_type = model.uncond_type - batch_size = noise_shape[0] - fs = cond["fs"] - del cond["fs"] - if noise_shape[-1] == 32: - timestep_spacing = "uniform" - guidance_rescale = 0.0 - else: - timestep_spacing = "uniform_trailing" - guidance_rescale = 0.7 - ## construct unconditional guidance - if cfg_scale != 1.0: - if uncond_type == "empty_seq": - prompts = batch_size * [""] - #prompts = N * T * [""] ## if is_imgbatch=True - uc_emb = model.get_learned_conditioning(prompts) - elif uncond_type == "zero_embed": - c_emb = cond["c_crossattn"][0] if isinstance(cond, dict) else cond - uc_emb = torch.zeros_like(c_emb) - - ## process image embedding token - if hasattr(model, 'embedder'): - uc_img = torch.zeros(noise_shape[0],3,224,224).to(model.device) - ## img: b c h w >> b l c - uc_img = model.embedder(uc_img) - uc_img = model.image_proj_model(uc_img) - uc_emb = torch.cat([uc_emb, uc_img], dim=1) - - if isinstance(cond, dict): - uc = {key:cond[key] for key in cond.keys()} - uc.update({'c_crossattn': [uc_emb]}) - else: - uc = uc_emb - else: - uc = None - - x_T = None - batch_variants = [] - - for _ in range(n_samples): - if ddim_sampler is not None: - kwargs.update({"clean_cond": True}) - samples, _ = ddim_sampler.sample(S=ddim_steps, - conditioning=cond, - batch_size=noise_shape[0], - shape=noise_shape[1:], - verbose=False, - unconditional_guidance_scale=cfg_scale, - unconditional_conditioning=uc, - eta=ddim_eta, - temporal_length=noise_shape[2], - conditional_guidance_scale_temporal=temporal_cfg_scale, - x_T=x_T, - fs=fs, - timestep_spacing=timestep_spacing, - guidance_rescale=guidance_rescale, - **kwargs - ) - ## reconstruct from latent to pixel space - batch_images = model.decode_first_stage(samples) - batch_variants.append(batch_images) - ## batch, , c, t, h, w - batch_variants = torch.stack(batch_variants, dim=1) - return batch_variants - - -def get_filelist(data_dir, ext='*'): - file_list = glob.glob(os.path.join(data_dir, '*.%s'%ext)) - file_list.sort() - return file_list - -def get_dirlist(path): - list = [] - if (os.path.exists(path)): - files = os.listdir(path) - for file in files: - m = os.path.join(path,file) - if (os.path.isdir(m)): - list.append(m) - list.sort() - return list - - def load_model_checkpoint(model, ckpt): def load_checkpoint(model, ckpt, full_strict): state_dict = torch.load(ckpt, map_location="cpu") @@ -139,85 +47,6 @@ def load_prompts(prompt_file): f.close() return prompt_list - -def load_video_batch(filepath_list, frame_stride, video_size=(256,256), video_frames=16): - ''' - Notice about some special cases: - 1. video_frames=-1 means to take all the frames (with fs=1) - 2. when the total video frames is less than required, padding strategy will be used (repreated last frame) - ''' - fps_list = [] - batch_tensor = [] - assert frame_stride > 0, "valid frame stride should be a positive interge!" - for filepath in filepath_list: - padding_num = 0 - vidreader = VideoReader(filepath, ctx=cpu(0), width=video_size[1], height=video_size[0]) - fps = vidreader.get_avg_fps() - total_frames = len(vidreader) - max_valid_frames = (total_frames-1) // frame_stride + 1 - if video_frames < 0: - ## all frames are collected: fs=1 is a must - required_frames = total_frames - frame_stride = 1 - else: - required_frames = video_frames - query_frames = min(required_frames, max_valid_frames) - frame_indices = [frame_stride*i for i in range(query_frames)] - - ## [t,h,w,c] -> [c,t,h,w] - frames = vidreader.get_batch(frame_indices) - frame_tensor = torch.tensor(frames.asnumpy()).permute(3, 0, 1, 2).float() - frame_tensor = (frame_tensor / 255. - 0.5) * 2 - if max_valid_frames < required_frames: - padding_num = required_frames - max_valid_frames - frame_tensor = torch.cat([frame_tensor, *([frame_tensor[:,-1:,:,:]]*padding_num)], dim=1) - print(f'{os.path.split(filepath)[1]} is not long enough: {padding_num} frames padded.') - batch_tensor.append(frame_tensor) - sample_fps = int(fps/frame_stride) - fps_list.append(sample_fps) - - return torch.stack(batch_tensor, dim=0) - -from PIL import Image -def load_image_batch(filepath_list, image_size=(256,256)): - batch_tensor = [] - for filepath in filepath_list: - _, filename = os.path.split(filepath) - _, ext = os.path.splitext(filename) - if ext == '.mp4': - vidreader = VideoReader(filepath, ctx=cpu(0), width=image_size[1], height=image_size[0]) - frame = vidreader.get_batch([0]) - img_tensor = torch.tensor(frame.asnumpy()).squeeze(0).permute(2, 0, 1).float() - elif ext == '.png' or ext == '.jpg': - img = Image.open(filepath).convert("RGB") - rgb_img = np.array(img, np.float32) - #bgr_img = cv2.imread(filepath, cv2.IMREAD_COLOR) - #bgr_img = cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB) - rgb_img = cv2.resize(rgb_img, (image_size[1],image_size[0]), interpolation=cv2.INTER_LINEAR) - img_tensor = torch.from_numpy(rgb_img).permute(2, 0, 1).float() - else: - print(f'ERROR: <{ext}> image loading only support format: [mp4], [png], [jpg]') - raise NotImplementedError - img_tensor = (img_tensor / 255. - 0.5) * 2 - batch_tensor.append(img_tensor) - return torch.stack(batch_tensor, dim=0) - - -def save_videos(batch_tensors, savedir, filenames, fps=10): - # b,samples,c,t,h,w - n_samples = batch_tensors.shape[1] - for idx, vid_tensor in enumerate(batch_tensors): - video = vid_tensor.detach().cpu() - video = torch.clamp(video.float(), -1., 1.) - video = video.permute(2, 0, 1, 3, 4) # t,n,c,h,w - frame_grids = [torchvision.utils.make_grid(framesheet, nrow=int(n_samples)) for framesheet in video] #[3, 1*h, n*w] - grid = torch.stack(frame_grids, dim=0) # stack in temporal dim [t, 3, n*h, w] - grid = (grid + 1.0) / 2.0 - grid = (grid * 255).to(torch.uint8).permute(0, 2, 3, 1) - savepath = os.path.join(savedir, f"{filenames[idx]}.mp4") - torchvision.io.write_video(savepath, grid, fps=fps, video_codec='h264', options={'crf': '10'}) - - def get_latent_z(model, videos): b, c, t, h, w = videos.shape x = rearrange(videos, 'b c t h w -> (b t) c h w') diff --git a/scripts/gradio/i2v_test.py b/scripts/gradio/i2v_test.py deleted file mode 100644 index d0066a9..0000000 --- a/scripts/gradio/i2v_test.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -import time -from omegaconf import OmegaConf -import torch -from scripts.evaluation.funcs import load_model_checkpoint, save_videos, batch_ddim_sampling, get_latent_z -from utils.utils import instantiate_from_config -from huggingface_hub import hf_hub_download -from einops import repeat -import torchvision.transforms as transforms -from pytorch_lightning import seed_everything - - -class Image2Video(): - def __init__(self,result_dir='./tmp/',gpu_num=1,resolution='256_256') -> None: - self.resolution = (int(resolution.split('_')[0]), int(resolution.split('_')[1])) #hw - self.download_model() - - self.result_dir = result_dir - if not os.path.exists(self.result_dir): - os.mkdir(self.result_dir) - ckpt_path='checkpoints/dynamicrafter_'+resolution.split('_')[1]+'_v1/model.ckpt' - config_file='configs/inference_'+resolution.split('_')[1]+'_v1.0.yaml' - config = OmegaConf.load(config_file) - model_config = config.pop("model", OmegaConf.create()) - model_config['params']['unet_config']['params']['use_checkpoint']=False - model_list = [] - for gpu_id in range(gpu_num): - model = instantiate_from_config(model_config) - # model = model.cuda(gpu_id) - assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!" - model = load_model_checkpoint(model, ckpt_path) - model.eval().half() - model_list.append(model) - self.model_list = model_list - self.save_fps = 8 - - def get_image(self, image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123): - seed_everything(seed) - transform = transforms.Compose([ - transforms.Resize(min(self.resolution)), - transforms.CenterCrop(self.resolution), - ]) - torch.cuda.empty_cache() - print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))) - start = time.time() - gpu_id=0 - if steps > 60: - steps = 60 - model = self.model_list[gpu_id] - model = model.cuda() - batch_size=1 - channels = model.model.diffusion_model.out_channels - frames = model.temporal_length - h, w = self.resolution[0] // 8, self.resolution[1] // 8 - noise_shape = [batch_size, channels, frames, h, w] - - # text cond - with torch.no_grad(), torch.cuda.amp.autocast(): - text_emb = model.get_learned_conditioning([prompt]) - - # img cond - img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device) - img_tensor = (img_tensor / 255. - 0.5) * 2 - - image_tensor_resized = transform(img_tensor) #3,h,w - videos = image_tensor_resized.unsqueeze(0) # bchw - - z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw - - img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames) - - cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc - img_emb = model.image_proj_model(cond_images) - - imtext_cond = torch.cat([text_emb, img_emb], dim=1) - - fs = torch.tensor([fs], dtype=torch.long, device=model.device) - cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]} - - ## inference - batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale) - ## b,samples,c,t,h,w - prompt_str = prompt.replace("/", "_slash_") if "/" in prompt else prompt - prompt_str = prompt_str.replace(" ", "_") if " " in prompt else prompt_str - prompt_str=prompt_str[:40] - if len(prompt_str) == 0: - prompt_str = 'empty_prompt' - - save_videos(batch_samples, self.result_dir, filenames=[prompt_str], fps=self.save_fps) - print(f"Saved in {prompt_str}. Time used: {(time.time() - start):.2f} seconds") - model = model.cpu() - return os.path.join(self.result_dir, f"{prompt_str}.mp4") - - def download_model(self): - REPO_ID = 'Doubiiu/DynamiCrafter_'+str(self.resolution[1]) if self.resolution[1]!=256 else 'Doubiiu/DynamiCrafter' - filename_list = ['model.ckpt'] - if not os.path.exists('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/'): - os.makedirs('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/') - for filename in filename_list: - local_file = os.path.join('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/', filename) - if not os.path.exists(local_file): - hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_v1/', local_dir_use_symlinks=False) - -if __name__ == '__main__': - i2v = Image2Video() - video_path = i2v.get_image('prompts/art.png','man fishing in a boat at sunset') - print('done', video_path) \ No newline at end of file diff --git a/scripts/gradio/i2v_test_application.py b/scripts/gradio/i2v_test_application.py deleted file mode 100644 index e616ad0..0000000 --- a/scripts/gradio/i2v_test_application.py +++ /dev/null @@ -1,131 +0,0 @@ -import os -import time -from omegaconf import OmegaConf -import torch -from scripts.evaluation.funcs import load_model_checkpoint, save_videos, batch_ddim_sampling, get_latent_z -from utils.utils import instantiate_from_config -from huggingface_hub import hf_hub_download -from einops import repeat -import torchvision.transforms as transforms -from pytorch_lightning import seed_everything - - -class Image2Video(): - def __init__(self,result_dir='./tmp/',gpu_num=1,resolution='256_256') -> None: - self.resolution = (int(resolution.split('_')[0]), int(resolution.split('_')[1])) #hw - self.download_model() - - self.result_dir = result_dir - if not os.path.exists(self.result_dir): - os.mkdir(self.result_dir) - ckpt_path='checkpoints/dynamicrafter_'+resolution.split('_')[1]+'_interp_v1/model.ckpt' - config_file='configs/inference_'+resolution.split('_')[1]+'_v1.0.yaml' - config = OmegaConf.load(config_file) - model_config = config.pop("model", OmegaConf.create()) - model_config['params']['unet_config']['params']['use_checkpoint']=False - model_list = [] - for gpu_id in range(gpu_num): - model = instantiate_from_config(model_config) - # model = model.cuda(gpu_id) - assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!" - model = load_model_checkpoint(model, ckpt_path) - model.eval() - model_list.append(model) - self.model_list = model_list - self.save_fps = 8 - - def get_image(self, image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123, image2=None): - seed_everything(seed) - transform = transforms.Compose([ - transforms.Resize(min(self.resolution)), - transforms.CenterCrop(self.resolution), - ]) - torch.cuda.empty_cache() - print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))) - start = time.time() - gpu_id=0 - if steps > 60: - steps = 60 - model = self.model_list[gpu_id] - model = model.cuda() - batch_size=1 - channels = model.model.diffusion_model.out_channels - frames = model.temporal_length - h, w = self.resolution[0] // 8, self.resolution[1] // 8 - noise_shape = [batch_size, channels, frames, h, w] - - # text cond - with torch.no_grad(), torch.cuda.amp.autocast(): - text_emb = model.get_learned_conditioning([prompt]) - - # img cond - img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device) - img_tensor = (img_tensor / 255. - 0.5) * 2 - - image_tensor_resized = transform(img_tensor) #3,h,w - videos = image_tensor_resized.unsqueeze(0) # bchw - - z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw - - - if image2 is not None: - img_tensor2 = torch.from_numpy(image2).permute(2, 0, 1).float().to(model.device) - img_tensor2 = (img_tensor2 / 255. - 0.5) * 2 - - image_tensor_resized2 = transform(img_tensor2) #3,h,w - videos2 = image_tensor_resized2.unsqueeze(0) # bchw - - z2 = get_latent_z(model, videos2.unsqueeze(2)) #bc,1,hw - - img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames) - - img_tensor_repeat = torch.zeros_like(img_tensor_repeat) - - ## old - img_tensor_repeat[:,:,:1,:,:] = z - if image2 is not None: - img_tensor_repeat[:,:,-1:,:,:] = z2 - else: - img_tensor_repeat[:,:,-1:,:,:] = z - - - cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc - img_emb = model.image_proj_model(cond_images) - - imtext_cond = torch.cat([text_emb, img_emb], dim=1) - - fs = torch.tensor([fs], dtype=torch.long, device=model.device) - cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]} - - ## inference - batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale) - - ## remove the last frame - if image2 is None: - batch_samples = batch_samples[:,:,:,:-1,...] - ## b,samples,c,t,h,w - prompt_str = prompt.replace("/", "_slash_") if "/" in prompt else prompt - prompt_str = prompt_str.replace(" ", "_") if " " in prompt else prompt_str - prompt_str=prompt_str[:40] - if len(prompt_str) == 0: - prompt_str = 'empty_prompt' - - save_videos(batch_samples, self.result_dir, filenames=[prompt_str], fps=self.save_fps) - print(f"Saved in {prompt_str}. Time used: {(time.time() - start):.2f} seconds") - model = model.cpu() - return os.path.join(self.result_dir, f"{prompt_str}.mp4") - - def download_model(self): - REPO_ID = 'Doubiiu/DynamiCrafter_'+str(self.resolution[1])+'_Interp' - filename_list = ['model.ckpt'] - if not os.path.exists('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/'): - os.makedirs('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/') - for filename in filename_list: - local_file = os.path.join('./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/', filename) - if not os.path.exists(local_file): - hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_'+str(self.resolution[1])+'_interp_v1/', local_dir_use_symlinks=False) - -if __name__ == '__main__': - i2v = Image2Video() - video_path = i2v.get_image('prompts/art.png','man fishing in a boat at sunset') - print('done', video_path) \ No newline at end of file diff --git a/scripts/run.sh b/scripts/run.sh deleted file mode 100644 index f3a9dca..0000000 --- a/scripts/run.sh +++ /dev/null @@ -1,61 +0,0 @@ -version=$1 ##1024, 512, 256 -seed=123 -name=dynamicrafter_$1_seed${seed} - -ckpt=checkpoints/dynamicrafter_$1_v1/model.ckpt -config=configs/inference_$1_v1.0.yaml - -prompt_dir=prompts/$1/ -res_dir="results" - -if [ "$1" == "256" ]; then - H=256 - FS=3 ## This model adopts frame stride=3, range recommended: 1-6 (larger value -> larger motion) -elif [ "$1" == "512" ]; then - H=320 - FS=24 ## This model adopts FPS=24, range recommended: 15-30 (smaller value -> larger motion) -elif [ "$1" == "1024" ]; then - H=576 - FS=10 ## This model adopts FPS=10, range recommended: 15-5 (smaller value -> larger motion) -else - echo "Invalid input. Please enter 256, 512, or 1024." - exit 1 -fi - -if [ "$1" == "256" ]; then -CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \ ---seed ${seed} \ ---ckpt_path $ckpt \ ---config $config \ ---savedir $res_dir/$name \ ---n_samples 1 \ ---bs 1 --height ${H} --width $1 \ ---unconditional_guidance_scale 7.5 \ ---ddim_steps 50 \ ---ddim_eta 1.0 \ ---prompt_dir $prompt_dir \ ---text_input \ ---video_length 16 \ ---frame_stride ${FS} -else -CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \ ---seed ${seed} \ ---ckpt_path $ckpt \ ---config $config \ ---savedir $res_dir/$name \ ---n_samples 1 \ ---bs 1 --height ${H} --width $1 \ ---unconditional_guidance_scale 7.5 \ ---ddim_steps 50 \ ---ddim_eta 1.0 \ ---prompt_dir $prompt_dir \ ---text_input \ ---video_length 16 \ ---frame_stride ${FS} \ ---timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae -fi - - -## multi-cond CFG: the is s_txt, is s_img -#--multiple_cond_cfg --cfg_img 7.5 -#--loop \ No newline at end of file diff --git a/scripts/run_application.sh b/scripts/run_application.sh deleted file mode 100644 index ae8943f..0000000 --- a/scripts/run_application.sh +++ /dev/null @@ -1,47 +0,0 @@ -version=$1 # interp or loop -ckpt=checkpoints/dynamicrafter_512_interp_v1/model.ckpt -config=configs/inference_512_v1.0.yaml - -prompt_dir=prompts/512_$1/ -res_dir="results" - -FS=5 ## This model adopts FPS=5, range recommended: 5-30 (smaller value -> larger motion) - - -if [ "$1" == "interp" ]; then -seed=12306 -name=dynamicrafter_512_$1_seed${seed} -CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \ ---seed ${seed} \ ---ckpt_path $ckpt \ ---config $config \ ---savedir $res_dir/$name \ ---n_samples 1 \ ---bs 1 --height 320 --width 512 \ ---unconditional_guidance_scale 7.5 \ ---ddim_steps 50 \ ---ddim_eta 1.0 \ ---prompt_dir $prompt_dir \ ---text_input \ ---video_length 16 \ ---frame_stride ${FS} \ ---timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae --interp -else -seed=234 -name=dynamicrafter_512_$1_seed${seed} -CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/inference.py \ ---seed ${seed} \ ---ckpt_path $ckpt \ ---config $config \ ---savedir $res_dir/$name \ ---n_samples 1 \ ---bs 1 --height 320 --width 512 \ ---unconditional_guidance_scale 7.5 \ ---ddim_steps 50 \ ---ddim_eta 1.0 \ ---prompt_dir $prompt_dir \ ---text_input \ ---video_length 16 \ ---frame_stride ${FS} \ ---timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae --loop -fi diff --git a/scripts/run_mp.sh b/scripts/run_mp.sh deleted file mode 100644 index 2ae2001..0000000 --- a/scripts/run_mp.sh +++ /dev/null @@ -1,102 +0,0 @@ -version=$1 ##1024, 512, 256 -seed=123 - -name=dynamicrafter_$1_mp_seed${seed} - -ckpt=checkpoints/dynamicrafter_$1_v1/model.ckpt -config=configs/inference_$1_v1.0.yaml - -prompt_dir=prompts/$1/ -res_dir="results" - -if [ "$1" == "256" ]; then - H=256 - FS=3 ## This model adopts frame stride=3 -elif [ "$1" == "512" ]; then - H=320 - FS=24 ## This model adopts FPS=24 -elif [ "$1" == "1024" ]; then - H=576 - FS=10 ## This model adopts FPS=10 -else - echo "Invalid input. Please enter 256, 512, or 1024." - exit 1 -fi - -# if [ "$1" == "256" ]; then -# CUDA_VISIBLE_DEVICES=2 python3 scripts/evaluation/inference.py \ -# --seed 123 \ -# --ckpt_path $ckpt \ -# --config $config \ -# --savedir $res_dir/$name \ -# --n_samples 1 \ -# --bs 1 --height ${H} --width $1 \ -# --unconditional_guidance_scale 7.5 \ -# --ddim_steps 50 \ -# --ddim_eta 1.0 \ -# --prompt_dir $prompt_dir \ -# --text_input \ -# --video_length 16 \ -# --frame_stride ${FS} -# else -# CUDA_VISIBLE_DEVICES=2 python3 scripts/evaluation/inference.py \ -# --seed 123 \ -# --ckpt_path $ckpt \ -# --config $config \ -# --savedir $res_dir/$name \ -# --n_samples 1 \ -# --bs 1 --height ${H} --width $1 \ -# --unconditional_guidance_scale 7.5 \ -# --ddim_steps 50 \ -# --ddim_eta 1.0 \ -# --prompt_dir $prompt_dir \ -# --text_input \ -# --video_length 16 \ -# --frame_stride ${FS} \ -# --timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 -# fi - - -## multi-cond CFG: the is s_txt, is s_img -#--multiple_cond_cfg --cfg_img 7.5 -#--loop - -## inference using single node with multi-GPUs: -if [ "$1" == "256" ]; then -CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3 -m torch.distributed.launch \ ---nproc_per_node=8 --nnodes=1 --master_addr=127.0.0.1 --master_port=23456 --node_rank=0 \ -scripts/evaluation/ddp_wrapper.py \ ---module 'inference' \ ---seed ${seed} \ ---ckpt_path $ckpt \ ---config $config \ ---savedir $res_dir/$name \ ---n_samples 1 \ ---bs 1 --height ${H} --width $1 \ ---unconditional_guidance_scale 7.5 \ ---ddim_steps 50 \ ---ddim_eta 1.0 \ ---prompt_dir $prompt_dir \ ---text_input \ ---video_length 16 \ ---frame_stride ${FS} -else -CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3 -m torch.distributed.launch \ ---nproc_per_node=8 --nnodes=1 --master_addr=127.0.0.1 --master_port=23456 --node_rank=0 \ -scripts/evaluation/ddp_wrapper.py \ ---module 'inference' \ ---seed ${seed} \ ---ckpt_path $ckpt \ ---config $config \ ---savedir $res_dir/$name \ ---n_samples 1 \ ---bs 1 --height ${H} --width $1 \ ---unconditional_guidance_scale 7.5 \ ---ddim_steps 50 \ ---ddim_eta 1.0 \ ---prompt_dir $prompt_dir \ ---text_input \ ---video_length 16 \ ---frame_stride ${FS} \ ---timestep_spacing 'uniform_trailing' --guidance_rescale 0.7 --perframe_ae -fi \ No newline at end of file diff --git a/utils/utils.py b/utils/utils.py index cece051..4be5489 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -1,11 +1,9 @@ import importlib import numpy as np -import cv2 import torch import torch.distributed as dist import os - def count_params(model, verbose=False): total_params = sum(p.numel() for p in model.parameters()) if verbose: @@ -49,26 +47,11 @@ def load_npz_from_dir(data_dir): data = np.concatenate(data, axis=0) return data - def load_npz_from_paths(data_paths): data = [np.load(data_path)['arr_0'] for data_path in data_paths] data = np.concatenate(data, axis=0) return data - -def resize_numpy_image(image, max_resolution=512 * 512, resize_short_edge=None): - h, w = image.shape[:2] - if resize_short_edge is not None: - k = resize_short_edge / min(h, w) - else: - k = max_resolution / (h * w) - k = k**0.5 - h = int(np.round(h * k / 64)) * 64 - w = int(np.round(w * k / 64)) * 64 - image = cv2.resize(image, (w, h), interpolation=cv2.INTER_LANCZOS4) - return image - - def setup_dist(args): if dist.is_initialized(): return