From 7ec6f8f720ea3d2eff464acc9405b5e2cdd036ea Mon Sep 17 00:00:00 2001 From: Peiyuan Zhang Date: Mon, 28 Oct 2024 19:21:00 +0000 Subject: [PATCH] Deleted unnecessary files --- README.md | 6 +- fastvideo/__init__.py | 1 - fastvideo/dataset/__init__.py | 6 +- fastvideo/dataset/inpaint_datasets.py | 116 ------------------ fastvideo/dataset/t2v_datasets.py | 4 +- fastvideo/dataset/transform.py | 4 +- .../train_t2v_diffusers.py => train.py} | 9 +- fastvideo/utils/lora_utils.py | 82 ------------- t2v_debug_multi.sh | 4 +- 9 files changed, 12 insertions(+), 220 deletions(-) delete mode 100644 fastvideo/__init__.py delete mode 100644 fastvideo/dataset/inpaint_datasets.py rename fastvideo/{train/train_t2v_diffusers.py => train.py} (99%) delete mode 100644 fastvideo/utils/lora_utils.py diff --git a/README.md b/README.md index 4f46973fd..573ad8dcc 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,11 @@ apt-get update && apt install screen && pip install watch gpustat ## Prepare Data & Models We've prepared some debug data to facilitate development. To make sure the training pipeline is correct, train on the debug data and make sure the model overfit on it (feed it the same text prompt and see if the output video is the same as the training data) + ``` python scripts/download_hf.py --repo_id=Stealths-Video/dummyVid --local_dir=data/dummyVid --repo_type=dataset -python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/checkpoint.ckpt --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model -python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model -python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name 29x480p/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model -python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name 29x480p/diffusion_pytorch_model.safetensors --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model python scripts/download_hf.py --repo_id=Stealths-Video/mochi --local_dir=data/mochi --repo_type=model +python scripts/download_hf.py --repo_id=Stealths-Video/Mochi-Synthetic-Data --local_dir=data/mochi --repo_type=model ``` ``` diff --git a/fastvideo/__init__.py b/fastvideo/__init__.py deleted file mode 100644 index 4e768b56d..000000000 --- a/fastvideo/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# \ No newline at end of file diff --git a/fastvideo/dataset/__init__.py b/fastvideo/dataset/__init__.py index 1f2237b5c..c01d0f15d 100644 --- a/fastvideo/dataset/__init__.py +++ b/fastvideo/dataset/__init__.py @@ -3,7 +3,6 @@ from transformers import AutoTokenizer from torchvision import transforms from fastvideo.dataset.t2v_datasets import T2V_dataset -from fastvideo.dataset.inpaint_datasets import Inpaint_dataset from fastvideo.dataset.transform import Normalize255, TemporalRandomCrop,CenterCropResizeVideo from fastvideo.models.causalvideovae import ae_norm, ae_denorm @@ -28,10 +27,7 @@ def getdataset(args): tokenizer = AutoTokenizer.from_pretrained(args.text_encoder_name, cache_dir=args.cache_dir) if args.dataset == 't2v': return T2V_dataset(args, transform=transform, temporal_sample=temporal_sample, tokenizer=tokenizer, - transform_topcrop=transform_topcrop, video_length_tolerance_range=args.video_length_tolerance_range) - elif args.dataset == 'inpaint' or args.dataset == 'i2v': - return Inpaint_dataset(args, transform=transform, temporal_sample=temporal_sample, tokenizer=tokenizer, - transform_topcrop=transform_topcrop) + transform_topcrop=transform_topcrop) raise NotImplementedError(args.dataset) diff --git a/fastvideo/dataset/inpaint_datasets.py b/fastvideo/dataset/inpaint_datasets.py deleted file mode 100644 index ce0959774..000000000 --- a/fastvideo/dataset/inpaint_datasets.py +++ /dev/null @@ -1,116 +0,0 @@ - -import os, random -import numpy as np - -import torch -from accelerate.logging import get_logger - -from fastvideo.utils.utils import text_preprocessing - -from fastvideo.dataset.t2v_datasets import SingletonMeta, DataSetProg -from fastvideo.dataset.t2v_datasets import T2V_dataset - -logger = get_logger(__name__) - -dataset_prog = DataSetProg() - - -class Meta_dataset(T2V_dataset): - def __init__(self, args, transform, temporal_sample, tokenizer, transform_topcrop): - super().__init__(args, transform, temporal_sample, tokenizer, transform_topcrop) - - if self.num_frames != 1: - # inpaint - # The proportion of executing the i2v task. - self.i2v_ratio = args.i2v_ratio - self.transition_ratio = args.transition_ratio - self.v2v_ratio = args.v2v_ratio - self.clear_video_ratio = args.clear_video_ratio - assert self.i2v_ratio + self.transition_ratio + self.v2v_ratio + self.clear_video_ratio < 1, 'The sum of i2v_ratio, transition_ratio, v2v_ratio and clear video ratio should be less than 1.' - - self.default_text_ratio = args.default_text_ratio - self.default_text = f"The {'video' if self.num_frames != 1 else 'image'} showcases a scene with coherent and clear visuals." - - def get_mask_masked_video(self, video): - # video shape (T, C, H, W) - # 1 means masked, 0 means not masked - t, c, h, w = video.shape - mask = torch.ones_like(video, device=video.device, dtype=video.dtype) - - rand_num = random.random() - # i2v - if rand_num < self.i2v_ratio: - mask[0] = 0 - # transition - elif rand_num < self.i2v_ratio + self.transition_ratio: - mask[0] = 0 - mask[-1] = 0 - # video continuation - elif rand_num < self.i2v_ratio + self.transition_ratio + self.v2v_ratio: - end_idx = random.randint(1, t) - mask[:end_idx] = 0 - # clear video - elif rand_num < self.i2v_ratio + self.transition_ratio + self.v2v_ratio + self.clear_video_ratio: - mask[:] = 0 - # random mask - else: - idx_to_select = random.randint(0, t - 1) - selected_indices = random.sample(range(0, t), idx_to_select) - mask[selected_indices] = 0 - masked_video = video * (mask < 0.5) - - # save_video(masked_video.permute(0, 2, 3, 1).cpu().numpy(), 'masked_video.mp4') - return dict(mask=mask, masked_video=masked_video) - - def drop(self, text): - rand_num = random.random() - rand_num_text = random.random() - - if rand_num < self.cfg: - text = self.default_text if rand_num_text < self.default_text_ratio else '' - - return dict(text=text) - - -class Inpaint_dataset(Meta_dataset): - def get_video(self, idx): - video_path = dataset_prog.cap_list[idx]['path'] - assert os.path.exists(video_path), f"file {video_path} do not exist!" - frame_indice = dataset_prog.cap_list[idx]['sample_frame_index'] - video = self.decord_read(video_path, predefine_num_frames=len(frame_indice)) - - h, w = video.shape[-2:] - assert h / w <= 17 / 16 and h / w >= 8 / 16, f'Only videos with a ratio (h/w) less than 17/16 and more than 8/16 are supported. But video ({video_path}) found ratio is {round(h / w, 2)} with the shape of {video.shape}' - t = video.shape[0] - video = self.transform(video) # T C H W -> T C H W - - # inpaint - inpaint_cond_data = self.get_mask_masked_video(video) - mask, masked_video = inpaint_cond_data['mask'], inpaint_cond_data['masked_video'] - video = torch.cat([video, masked_video, mask], dim=1) # T 3*C H W - - # video = torch.rand(221, 3, 480, 640) - - video = video.transpose(0, 1) # T C H W -> C T H W - text = dataset_prog.cap_list[idx]['cap'] - if not isinstance(text, list): - text = [text] - text = [random.choice(text)] - - text = text_preprocessing(text, support_Chinese=self.support_Chinese) - - text = self.drop(text)['text'] - - text_tokens_and_mask = self.tokenizer( - text, - max_length=self.model_max_length, - padding='max_length', - truncation=True, - return_attention_mask=True, - add_special_tokens=True, - return_tensors='pt' - ) - input_ids = text_tokens_and_mask['input_ids'] - cond_mask = text_tokens_and_mask['attention_mask'] - return dict(pixel_values=video, input_ids=input_ids, cond_mask=cond_mask) - \ No newline at end of file diff --git a/fastvideo/dataset/t2v_datasets.py b/fastvideo/dataset/t2v_datasets.py index 08e89b5ec..c15d19571 100644 --- a/fastvideo/dataset/t2v_datasets.py +++ b/fastvideo/dataset/t2v_datasets.py @@ -113,7 +113,7 @@ def filter_resolution(h, w, max_h_div_w_ratio=17/16, min_h_div_w_ratio=8 / 16): class T2V_dataset(Dataset): - def __init__(self, args, transform, temporal_sample, tokenizer, transform_topcrop, video_length_tolerance_range): + def __init__(self, args, transform, temporal_sample, tokenizer, transform_topcrop): self.data = args.data self.num_frames = args.num_frames self.train_fps = args.train_fps @@ -130,7 +130,7 @@ class T2V_dataset(Dataset): self.drop_short_ratio = args.drop_short_ratio assert self.speed_factor >= 1 self.v_decoder = DecordInit() - self.video_length_tolerance_range = video_length_tolerance_range + self.video_length_tolerance_range = args.video_length_tolerance_range self.support_Chinese = True if not ('mt5' in args.text_encoder_name): self.support_Chinese = False diff --git a/fastvideo/dataset/transform.py b/fastvideo/dataset/transform.py index ca68d085d..df19d603a 100644 --- a/fastvideo/dataset/transform.py +++ b/fastvideo/dataset/transform.py @@ -426,7 +426,7 @@ class CenterCropVideo: return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}" -class Normalize255: +class Normalize: """ Normalize the video clip by mean subtraction and division by standard deviation Args: @@ -588,4 +588,4 @@ if __name__ == '__main__': for i in range(target_video_len): save_image(select_vframes_trans[i], os.path.join('./test000', '%04d.png' % i), normalize=True, - value_range=(-1, 1)) + value_range=(-1, 1)) \ No newline at end of file diff --git a/fastvideo/train/train_t2v_diffusers.py b/fastvideo/train.py similarity index 99% rename from fastvideo/train/train_t2v_diffusers.py rename to fastvideo/train.py index 9314845af..b122fbc0e 100644 --- a/fastvideo/train/train_t2v_diffusers.py +++ b/fastvideo/train.py @@ -47,10 +47,6 @@ from diffusers.utils import check_min_version, is_wandb_available from fastvideo.utils.ema import EMAModel from fastvideo.dataset import getdataset -from fastvideo.models import CausalVAEModelWrapper -from fastvideo.models.text_encoder import get_text_warpper -from fastvideo.models.causalvideovae import ae_stride_config, ae_channel_config -from fastvideo.models.diffusion import Diffusion_models, Diffusion_models_class from fastvideo.utils.dataset_utils import Collate, LengthGroupedSampler @@ -92,14 +88,14 @@ def log_validation(args, model, vae, text_encoder, tokenizer, accelerator, weigh logs = { f"{'ema_' if ema else ''}validation": [ wandb.Image(image, caption=f"{i}: {prompt}") - for i, (image, prompt) in enumerate(zip(images, validation_prompt)) + for i, (image, prompt) in enumerate(zip(images, args.validaiton_prompt)) ] } else: logs = { f"{'ema_' if ema else ''}validation": [ wandb.Video(video, caption=f"{i}: {prompt}", fps=24) - for i, (video, prompt) in enumerate(zip(videos, validation_prompt)) + for i, (video, prompt) in enumerate(zip(videos, args.validaiton_prompt)) ] } tracker.log(logs, step=global_step) @@ -745,6 +741,7 @@ if __name__ == "__main__": # dataset & dataloader parser.add_argument("--dataset", type=str, required=True) parser.add_argument("--data", type=str, required='') + parser.add_argument("--cache_vae_latent", action="store_true") parser.add_argument("--train_fps", type=int, default=24) parser.add_argument("--drop_short_ratio", type=float, default=1.0) parser.add_argument("--speed_factor", type=float, default=1.0) diff --git a/fastvideo/utils/lora_utils.py b/fastvideo/utils/lora_utils.py deleted file mode 100644 index 23862e9c0..000000000 --- a/fastvideo/utils/lora_utils.py +++ /dev/null @@ -1,82 +0,0 @@ - -from peft import get_peft_model, PeftModel -import os -from copy import deepcopy -import torch -from fastvideo.utils.ema import EMAModel - -class EMAModel_LoRA(EMAModel): - def __init__(self, lora_config, **kwargs): - super().__init__(**kwargs) - self.lora_config = lora_config - - @classmethod - def from_pretrained(cls, path, model_cls, lora_config, origin_model_path) -> "EMAModel": - # 1. load origin model - origin_model = model_cls.from_pretrained(origin_model_path) # origin_model - # 2. convert to lora model automatically and load lora weight - lora_model = PeftModel.from_pretrained(origin_model, path) # lora_origin_model - # 3. ema the whole lora_model - ema_model = cls(lora_config, parameters=lora_model.parameters(), model_cls=model_cls, model_config=origin_model.config) - # 4. load ema_kwargs, e.g decay... - ema_kwargs = torch.load(os.path.join(path, 'ema_kwargs.pt')) - ema_model.load_state_dict(ema_kwargs) - return ema_model - - def save_pretrained(self, path): - if self.model_cls is None: - raise ValueError("`save_pretrained` can only be used if `model_cls` was defined at __init__.") - - if self.model_config is None: - raise ValueError("`save_pretrained` can only be used if `model_config` was defined at __init__.") - # 1. init a base model randomly - model = self.model_cls.from_config(self.model_config) - # 2. convert lora_model - lora_model = get_peft_model(model, self.lora_config) - # 3. ema_lora_model weight to lora_model - self.copy_to(lora_model.parameters()) - # 4. save lora weight - lora_model.save_pretrained(path) # only lora weight - # 5. save ema_kwargs, e.g decay... - state_dict = self.state_dict() # lora_model weight - state_dict.pop("shadow_params", None) - torch.save(state_dict, os.path.join(path, 'ema_kwargs.pt')) - - -def maybe_zero_3(param, ignore_status=False, name=None): - from deepspeed import zero - from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus - if hasattr(param, "ds_id"): - if param.ds_status == ZeroParamStatus.NOT_AVAILABLE: - if not ignore_status: - logging.warning(f"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}") - with zero.GatheredParameters([param]): - param = param.data.detach().cpu().clone() - else: - param = param.detach().cpu().clone() - return param - -# Borrowed from peft.utils.get_peft_model_state_dict -def get_peft_state_maybe_zero_3(named_params, bias): - if bias == "none": - to_return = {k: t for k, t in named_params if "lora_" in k} - elif bias == "all": - to_return = {k: t for k, t in named_params if "lora_" in k or "bias" in k} - elif bias == "lora_only": - to_return = {} - maybe_lora_bias = {} - lora_bias_names = set() - for k, t in named_params: - if "lora_" in k: - to_return[k] = t - bias_name = k.split("lora_")[0] + "bias" - lora_bias_names.add(bias_name) - elif "bias" in k: - maybe_lora_bias[k] = t - for k, t in maybe_lora_bias: - if bias_name in lora_bias_names: - to_return[bias_name] = t - else: - raise NotImplementedError - to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()} - return to_return diff --git a/t2v_debug_multi.sh b/t2v_debug_multi.sh index fd3308164..b6f51d018 100644 --- a/t2v_debug_multi.sh +++ b/t2v_debug_multi.sh @@ -2,9 +2,9 @@ accelerate launch \ --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \ - fastvideo/train/train_t2v_diffusers.py \ + fastvideo/train.py \ --cache_dir "data/.cache" \ - --model OpenSoraT2V-mT5-Base-S/122 \ + --model data/mochi \ --text_encoder_name google/mt5-base \ --dataset t2v \ --data "data/dummyVid/merge_single.txt" \