From b6b1a2aa92adb8f0ad204ef51edff424b4b140c0 Mon Sep 17 00:00:00 2001 From: Peiyuan Zhang Date: Sat, 26 Oct 2024 17:53:51 +0000 Subject: [PATCH] Update T5Base --- README.md | 12 ++-- .../diffusion/opensora/modeling_opensora.py | 6 ++ opensora/models/text_encoder/__init__.py | 1 + pyproject.toml | 3 +- scripts/text_condition/overfit.sh | 67 +++++++++++++++++++ t2v_debug_multi.sh | 12 ++-- 6 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 scripts/text_condition/overfit.sh diff --git a/README.md b/README.md index 0d17df40d..4003089b6 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,12 @@ conda activate fastvideo pip3 install torch==2.5.0 torchvision --index-url https://download.pytorch.org/whl/cu121 pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu121 pip install ninja -pip install flash-attn --no-build-isolation +pip install flash-attn --no-build-isolationf ``` ``` -pip install -e . -pip install -e ".[train]" +pip install -e . && pip install -e ".[train]" +apt-get update && apt install screen && pip install watch gpustat ``` ## Prepare Data & Models @@ -27,7 +27,11 @@ python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --fi python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model ``` - +``` +from transformers import AutoTokenizer, AutoModelForSeq2SeqLM +tokenizer = AutoTokenizer.from_pretrained("google/mt5-xxl", cache_dir="data/.cache") +model = AutoModelForSeq2SeqLM.from_pretrained("google/mt5-xxl", cache_dir="data/.cache") +``` ## Debug Training ``` bash t2v_debug_single.sh diff --git a/opensora/models/diffusion/opensora/modeling_opensora.py b/opensora/models/diffusion/opensora/modeling_opensora.py index eb28f5972..5e0309e84 100644 --- a/opensora/models/diffusion/opensora/modeling_opensora.py +++ b/opensora/models/diffusion/opensora/modeling_opensora.py @@ -603,6 +603,10 @@ class OpenSoraT2V(ModelMixin, ConfigMixin): # output = output[:, :, 1:] return output +def OpenSoraT2V_mT5_Baes_S_122(**kwargs): + return OpenSoraT2V(num_layers=28, attention_head_dim=96, num_attention_heads=16, patch_size_t=1, patch_size=2, + norm_type="ada_norm_single", caption_channels=768, cross_attention_dim=1536, **kwargs) + def OpenSoraT2V_S_122(**kwargs): return OpenSoraT2V(num_layers=28, attention_head_dim=96, num_attention_heads=16, patch_size_t=1, patch_size=2, norm_type="ada_norm_single", caption_channels=4096, cross_attention_dim=1536, **kwargs) @@ -620,6 +624,7 @@ def OpenSoraT2V_ROPE_L_122(**kwargs): norm_type="ada_norm_single", caption_channels=4096, cross_attention_dim=2304, **kwargs) OpenSora_models = { + "OpenSoraT2V-mT5-Base-S/122": OpenSoraT2V_mT5_Baes_S_122, "OpenSoraT2V-S/122": OpenSoraT2V_S_122, # 1.1B "OpenSoraT2V-B/122": OpenSoraT2V_B_122, "OpenSoraT2V-L/122": OpenSoraT2V_L_122, @@ -627,6 +632,7 @@ OpenSora_models = { } OpenSora_models_class = { + "OpenSoraT2V-mT5-Base-S/122": OpenSoraT2V, "OpenSoraT2V-S/122": OpenSoraT2V, "OpenSoraT2V-B/122": OpenSoraT2V, "OpenSoraT2V-L/122": OpenSoraT2V, diff --git a/opensora/models/text_encoder/__init__.py b/opensora/models/text_encoder/__init__.py index 7e7ef2244..9dc29b688 100644 --- a/opensora/models/text_encoder/__init__.py +++ b/opensora/models/text_encoder/__init__.py @@ -37,6 +37,7 @@ class CLIPWrapper(nn.Module): text_encoder = { + 'google/mt5-base': T5Wrapper, 'google/mt5-xl': T5Wrapper, 'google/mt5-xxl': T5Wrapper, 'google/umt5-xl': T5Wrapper, diff --git a/pyproject.toml b/pyproject.toml index d98a75a9b..40c001d3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,8 @@ dependencies = [ "regex==2024.9.11", "requests==2.31.0", "scikit-learn==1.5.2", "scipy==1.14.1", "six==1.16.0", "test-tube==0.7.5", "timm==1.0.11", "torchdiffeq==0.2.4", "torchmetrics==1.5.1", "tqdm==4.66.5", "urllib3==2.2.0", "uvicorn==0.32.0", "scikit-video==1.1.11", "imageio-ffmpeg==0.5.1", "sentencepiece==0.2.0", "beautifulsoup4==4.12.3", "ftfy==6.3.0", -"moviepy==1.0.3", "wandb==0.18.5", "tensorboard==2.18.0", "pydantic==2.9.2", "gradio==5.3.0", "huggingface_hub==0.26.1", "protobuf==5.28.3" +"moviepy==1.0.3", "wandb==0.18.5", "tensorboard==2.18.0", "pydantic==2.9.2", "gradio==5.3.0", "huggingface_hub==0.26.1", "protobuf==5.28.3", +"watch", "gpustat" ] [project.optional-dependencies] diff --git a/scripts/text_condition/overfit.sh b/scripts/text_condition/overfit.sh new file mode 100644 index 000000000..16074d479 --- /dev/null +++ b/scripts/text_condition/overfit.sh @@ -0,0 +1,67 @@ +# export WANDB_MODE="offline" + +export ENTITY="linbin" +export HF_DATASETS_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 +export PDSH_RCMD_TYPE=ssh +# NCCL setting +export GLOO_SOCKET_IFNAME=eth0 +export NCCL_SOCKET_IFNAME=eth0 +export NCCL_IB_HCA=mlx5_10:1,mlx5_11:1,mlx5_12:1,mlx5_13:1 +export NCCL_IB_GID_INDEX=3 +export NCCL_IB_TC=162 +export NCCL_IB_TIMEOUT=22 +export NCCL_PXN_DISABLE=0 +export NCCL_IB_QPS_PER_CONNECTION=4 +export NCCL_ALGO=Ring +export OMP_NUM_THREADS=1 +export MKL_NUM_THREADS=1 + +accelerate launch \ + --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \ + opensora/train/train_t2v_diffusers.py \ + --model OpenSoraT2V-ROPE-L/122 \ + --text_encoder_name google/mt5-xxl \ + --cache_dir "data/.cache" \ + --dataset t2v \ + --data "data/dummyVid/merge.txt" \ + --ae CausalVAEModel_D4_4x8x8 \ + --ae_path "data/Open-Sora-Plan-v1.2.0/vae/" \ + --sample_rate 1 \ + --num_frames 29 \ + --max_height 480 \ + --max_width 640 \ + --pretrained /ephemeral/hao.zhang/codefolder/FastVideo-OSP/data/Open-Sora-Plan-v1.2.0/29x480p/diffusion_pytorch_model.safetensors \ + --interpolation_scale_t 1.0 \ + --interpolation_scale_h 1.0 \ + --interpolation_scale_w 1.0 \ + --attention_mode xformers \ + --gradient_checkpointing \ + --train_batch_size=1 \ + --dataloader_num_workers 10 \ + --gradient_accumulation_steps=8 \ + --max_train_steps=1000000 \ + --learning_rate=1e-4 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --mixed_precision="bf16" \ + --report_to="wandb" \ + --checkpointing_steps=25 \ + --allow_tf32 \ + --model_max_length 512 \ + --use_image_num 0 \ + --tile_overlap_factor 0.125 \ + --snr_gamma 5.0 \ + --use_ema \ + --ema_start_step 0 \ + --cfg 0.1 \ + --noise_offset 0.02 \ + --use_rope \ + --resume_from_checkpoint="latest" \ + --ema_decay 0.999 \ + --enable_tiling \ + --speed_factor 1.0 \ + --group_frame \ + --sp_size 4 \ + --train_sp_batch_size 1 \ + --output_dir="outputs/debug_1node_vid_newpipeline_acc" \ No newline at end of file diff --git a/t2v_debug_multi.sh b/t2v_debug_multi.sh index 7d2e7abbb..b811e1bd6 100644 --- a/t2v_debug_multi.sh +++ b/t2v_debug_multi.sh @@ -3,12 +3,11 @@ accelerate launch \ --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \ opensora/train/train_t2v_diffusers.py \ - --model OpenSoraT2V-ROPE-L/122 \ - --pretrained data/Open-Sora-Plan-v1.2.0/29x480p/diffusion_pytorch_model.safetensors \ - --text_encoder_name google/mt5-xxl \ --cache_dir "data/.cache" \ + --model OpenSoraT2V-mT5-Base-S/122 \ + --text_encoder_name google/mt5-base \ --dataset t2v \ - --data "data/dummyVid/merge.txt" \ + --data "data/dummyVid/merge_single.txt" \ --ae CausalVAEModel_D4_4x8x8 \ --ae_path "data/Open-Sora-Plan-v1.2.0/vae" \ --sample_rate 1 \ @@ -24,7 +23,7 @@ accelerate launch \ --gradient_checkpointing \ --train_batch_size=1 \ --dataloader_num_workers 1 \ - --gradient_accumulation_steps=1 \ + --gradient_accumulation_steps=8 \ --max_train_steps=1000000 \ --learning_rate=1e-4 \ --lr_scheduler="constant" \ @@ -46,7 +45,6 @@ accelerate launch \ --ema_decay 0.999 \ --enable_tiling \ --speed_factor 1.0 \ - --group_frame \ - --sp_size 2 \ + --sp_size 1 \ --train_sp_batch_size 1 \ --output_dir="outputs/debug" \ \ No newline at end of file