nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Deleted unnecessary files

Browse Source
This commit is contained in:
Peiyuan Zhang
2024-10-28 19:21:00 +00:00
parent cd6f8f14f1
commit 7ec6f8f720
9 changed files with 12 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -4
View File
@@ -19,13 +19,11 @@ apt-get update && apt install screen && pip install watch gpustat
## Prepare Data & Models
We've prepared some debug data to facilitate development. To make sure the training pipeline is correct, train on the debug data and make sure the model overfit on it (feed it the same text prompt and see if the output video is the same as the training data)
```
python scripts/download_hf.py --repo_id=Stealths-Video/dummyVid --local_dir=data/dummyVid --repo_type=dataset
python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/checkpoint.ckpt --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model
python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model
python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name 29x480p/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model
python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name 29x480p/diffusion_pytorch_model.safetensors --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model
python scripts/download_hf.py --repo_id=Stealths-Video/mochi --local_dir=data/mochi --repo_type=model
python scripts/download_hf.py --repo_id=Stealths-Video/Mochi-Synthetic-Data --local_dir=data/mochi --repo_type=model
```
```
fastvideo/__init__.py
-1
View File
@@ -1 +0,0 @@
#
fastvideo/dataset/__init__.py
+1 -5
View File
@@ -3,7 +3,6 @@ from transformers import AutoTokenizer
from torchvision import transforms
from fastvideo.dataset.t2v_datasets import T2V_dataset
from fastvideo.dataset.inpaint_datasets import Inpaint_dataset
from fastvideo.dataset.transform import Normalize255, TemporalRandomCrop,CenterCropResizeVideo
from fastvideo.models.causalvideovae import ae_norm, ae_denorm
@@ -28,10 +27,7 @@ def getdataset(args):
tokenizer = AutoTokenizer.from_pretrained(args.text_encoder_name, cache_dir=args.cache_dir)
if args.dataset == 't2v':
return T2V_dataset(args, transform=transform, temporal_sample=temporal_sample, tokenizer=tokenizer,
transform_topcrop=transform_topcrop, video_length_tolerance_range=args.video_length_tolerance_range)
elif args.dataset == 'inpaint' or args.dataset == 'i2v':
return Inpaint_dataset(args, transform=transform, temporal_sample=temporal_sample, tokenizer=tokenizer,
transform_topcrop=transform_topcrop)
transform_topcrop=transform_topcrop)
raise NotImplementedError(args.dataset)
fastvideo/dataset/inpaint_datasets.py
-116
View File
@@ -1,116 +0,0 @@
import os, random
import numpy as np
import torch
from accelerate.logging import get_logger
from fastvideo.utils.utils import text_preprocessing
from fastvideo.dataset.t2v_datasets import SingletonMeta, DataSetProg
from fastvideo.dataset.t2v_datasets import T2V_dataset
logger = get_logger(__name__)
dataset_prog = DataSetProg()
class Meta_dataset(T2V_dataset):
def __init__(self, args, transform, temporal_sample, tokenizer, transform_topcrop):
super().__init__(args, transform, temporal_sample, tokenizer, transform_topcrop)
if self.num_frames != 1:
# inpaint
# The proportion of executing the i2v task.
self.i2v_ratio = args.i2v_ratio
self.transition_ratio = args.transition_ratio
self.v2v_ratio = args.v2v_ratio
self.clear_video_ratio = args.clear_video_ratio
assert self.i2v_ratio + self.transition_ratio + self.v2v_ratio + self.clear_video_ratio < 1, 'The sum of i2v_ratio, transition_ratio, v2v_ratio and clear video ratio should be less than 1.'
self.default_text_ratio = args.default_text_ratio
self.default_text = f"The {'video' if self.num_frames != 1 else 'image'} showcases a scene with coherent and clear visuals."
def get_mask_masked_video(self, video):
# video shape (T, C, H, W)
# 1 means masked, 0 means not masked
t, c, h, w = video.shape
mask = torch.ones_like(video, device=video.device, dtype=video.dtype)
rand_num = random.random()
# i2v
if rand_num < self.i2v_ratio:
mask[0] = 0
# transition
elif rand_num < self.i2v_ratio + self.transition_ratio:
mask[0] = 0
mask[-1] = 0
# video continuation
elif rand_num < self.i2v_ratio + self.transition_ratio + self.v2v_ratio:
end_idx = random.randint(1, t)
mask[:end_idx] = 0
# clear video
elif rand_num < self.i2v_ratio + self.transition_ratio + self.v2v_ratio + self.clear_video_ratio:
mask[:] = 0
# random mask
else:
idx_to_select = random.randint(0, t - 1)
selected_indices = random.sample(range(0, t), idx_to_select)
mask[selected_indices] = 0
masked_video = video * (mask < 0.5)
# save_video(masked_video.permute(0, 2, 3, 1).cpu().numpy(), 'masked_video.mp4')
return dict(mask=mask, masked_video=masked_video)
def drop(self, text):
rand_num = random.random()
rand_num_text = random.random()
if rand_num < self.cfg:
text = self.default_text if rand_num_text < self.default_text_ratio else ''
return dict(text=text)
class Inpaint_dataset(Meta_dataset):
def get_video(self, idx):
video_path = dataset_prog.cap_list[idx]['path']
assert os.path.exists(video_path), f"file {video_path} do not exist!"
frame_indice = dataset_prog.cap_list[idx]['sample_frame_index']
video = self.decord_read(video_path, predefine_num_frames=len(frame_indice))
h, w = video.shape[-2:]
assert h / w <= 17 / 16 and h / w >= 8 / 16, f'Only videos with a ratio (h/w) less than 17/16 and more than 8/16 are supported. But video ({video_path}) found ratio is {round(h / w, 2)} with the shape of {video.shape}'
t = video.shape[0]
video = self.transform(video) # T C H W -> T C H W
# inpaint
inpaint_cond_data = self.get_mask_masked_video(video)
mask, masked_video = inpaint_cond_data['mask'], inpaint_cond_data['masked_video']
video = torch.cat([video, masked_video, mask], dim=1) # T 3*C H W
# video = torch.rand(221, 3, 480, 640)
video = video.transpose(0, 1) # T C H W -> C T H W
text = dataset_prog.cap_list[idx]['cap']
if not isinstance(text, list):
text = [text]
text = [random.choice(text)]
text = text_preprocessing(text, support_Chinese=self.support_Chinese)
text = self.drop(text)['text']
text_tokens_and_mask = self.tokenizer(
text,
max_length=self.model_max_length,
padding='max_length',
truncation=True,
return_attention_mask=True,
add_special_tokens=True,
return_tensors='pt'
)
input_ids = text_tokens_and_mask['input_ids']
cond_mask = text_tokens_and_mask['attention_mask']
return dict(pixel_values=video, input_ids=input_ids, cond_mask=cond_mask)
fastvideo/dataset/t2v_datasets.py
+2 -2
View File
@@ -113,7 +113,7 @@ def filter_resolution(h, w, max_h_div_w_ratio=17/16, min_h_div_w_ratio=8 / 16):
class T2V_dataset(Dataset):
def __init__(self, args, transform, temporal_sample, tokenizer, transform_topcrop, video_length_tolerance_range):
def __init__(self, args, transform, temporal_sample, tokenizer, transform_topcrop):
self.data = args.data
self.num_frames = args.num_frames
self.train_fps = args.train_fps
@@ -130,7 +130,7 @@ class T2V_dataset(Dataset):
self.drop_short_ratio = args.drop_short_ratio
assert self.speed_factor >= 1
self.v_decoder = DecordInit()
self.video_length_tolerance_range = video_length_tolerance_range
self.video_length_tolerance_range = args.video_length_tolerance_range
self.support_Chinese = True
if not ('mt5' in args.text_encoder_name):
self.support_Chinese = False
fastvideo/dataset/transform.py
+2 -2
View File
@@ -426,7 +426,7 @@ class CenterCropVideo:
return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}"
class Normalize255:
class Normalize:
"""
Normalize the video clip by mean subtraction and division by standard deviation
Args:
@@ -588,4 +588,4 @@ if __name__ == '__main__':
for i in range(target_video_len):
save_image(select_vframes_trans[i], os.path.join('./test000', '%04d.png' % i), normalize=True,
value_range=(-1, 1))
value_range=(-1, 1))
fastvideo/train/train_t2v_diffusers.py → fastvideo/train.py
+3 -6
View File
@@ -47,10 +47,6 @@ from diffusers.utils import check_min_version, is_wandb_available
from fastvideo.utils.ema import EMAModel
from fastvideo.dataset import getdataset
from fastvideo.models import CausalVAEModelWrapper
from fastvideo.models.text_encoder import get_text_warpper
from fastvideo.models.causalvideovae import ae_stride_config, ae_channel_config
from fastvideo.models.diffusion import Diffusion_models, Diffusion_models_class
from fastvideo.utils.dataset_utils import Collate, LengthGroupedSampler
@@ -92,14 +88,14 @@ def log_validation(args, model, vae, text_encoder, tokenizer, accelerator, weigh
logs = {
f"{'ema_' if ema else ''}validation": [
wandb.Image(image, caption=f"{i}: {prompt}")
for i, (image, prompt) in enumerate(zip(images, validation_prompt))
for i, (image, prompt) in enumerate(zip(images, args.validaiton_prompt))
]
}
else:
logs = {
f"{'ema_' if ema else ''}validation": [
wandb.Video(video, caption=f"{i}: {prompt}", fps=24)
for i, (video, prompt) in enumerate(zip(videos, validation_prompt))
for i, (video, prompt) in enumerate(zip(videos, args.validaiton_prompt))
]
}
tracker.log(logs, step=global_step)
@@ -745,6 +741,7 @@ if __name__ == "__main__":
# dataset & dataloader
parser.add_argument("--dataset", type=str, required=True)
parser.add_argument("--data", type=str, required='')
parser.add_argument("--cache_vae_latent", action="store_true")
parser.add_argument("--train_fps", type=int, default=24)
parser.add_argument("--drop_short_ratio", type=float, default=1.0)
parser.add_argument("--speed_factor", type=float, default=1.0)
fastvideo/utils/lora_utils.py
-82
View File
@@ -1,82 +0,0 @@
from peft import get_peft_model, PeftModel
import os
from copy import deepcopy
import torch
from fastvideo.utils.ema import EMAModel
class EMAModel_LoRA(EMAModel):
def __init__(self, lora_config, **kwargs):
super().__init__(**kwargs)
self.lora_config = lora_config
@classmethod
def from_pretrained(cls, path, model_cls, lora_config, origin_model_path) -> "EMAModel":
# 1. load origin model
origin_model = model_cls.from_pretrained(origin_model_path) # origin_model
# 2. convert to lora model automatically and load lora weight
lora_model = PeftModel.from_pretrained(origin_model, path) # lora_origin_model
# 3. ema the whole lora_model
ema_model = cls(lora_config, parameters=lora_model.parameters(), model_cls=model_cls, model_config=origin_model.config)
# 4. load ema_kwargs, e.g decay...
ema_kwargs = torch.load(os.path.join(path, 'ema_kwargs.pt'))
ema_model.load_state_dict(ema_kwargs)
return ema_model
def save_pretrained(self, path):
if self.model_cls is None:
raise ValueError("`save_pretrained` can only be used if `model_cls` was defined at __init__.")
if self.model_config is None:
raise ValueError("`save_pretrained` can only be used if `model_config` was defined at __init__.")
# 1. init a base model randomly
model = self.model_cls.from_config(self.model_config)
# 2. convert lora_model
lora_model = get_peft_model(model, self.lora_config)
# 3. ema_lora_model weight to lora_model
self.copy_to(lora_model.parameters())
# 4. save lora weight
lora_model.save_pretrained(path) # only lora weight
# 5. save ema_kwargs, e.g decay...
state_dict = self.state_dict() # lora_model weight
state_dict.pop("shadow_params", None)
torch.save(state_dict, os.path.join(path, 'ema_kwargs.pt'))
def maybe_zero_3(param, ignore_status=False, name=None):
from deepspeed import zero
from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
if hasattr(param, "ds_id"):
if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:
if not ignore_status:
logging.warning(f"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}")
with zero.GatheredParameters([param]):
param = param.data.detach().cpu().clone()
else:
param = param.detach().cpu().clone()
return param
# Borrowed from peft.utils.get_peft_model_state_dict
def get_peft_state_maybe_zero_3(named_params, bias):
if bias == "none":
to_return = {k: t for k, t in named_params if "lora_" in k}
elif bias == "all":
to_return = {k: t for k, t in named_params if "lora_" in k or "bias" in k}
elif bias == "lora_only":
to_return = {}
maybe_lora_bias = {}
lora_bias_names = set()
for k, t in named_params:
if "lora_" in k:
to_return[k] = t
bias_name = k.split("lora_")[0] + "bias"
lora_bias_names.add(bias_name)
elif "bias" in k:
maybe_lora_bias[k] = t
for k, t in maybe_lora_bias:
if bias_name in lora_bias_names:
to_return[bias_name] = t
else:
raise NotImplementedError
to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()}
return to_return
t2v_debug_multi.sh
+2 -2
View File
@@ -2,9 +2,9 @@
accelerate launch \
--config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
fastvideo/train/train_t2v_diffusers.py \
fastvideo/train.py \
--cache_dir "data/.cache" \
--model OpenSoraT2V-mT5-Base-S/122 \
--model data/mochi \
--text_encoder_name google/mt5-base \
--dataset t2v \
--data "data/dummyVid/merge_single.txt" \