Update T5Base

README.md

@@ -9,12 +9,12 @@ conda activate fastvideo
 pip3 install torch==2.5.0 torchvision  --index-url https://download.pytorch.org/whl/cu121
 pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu121
 pip install ninja
-pip install flash-attn --no-build-isolation
+pip install flash-attn --no-build-isolationf
 ```
 
 ```
-pip install -e .
-pip install -e ".[train]"
+pip install -e . && pip install -e ".[train]"
+apt-get update && apt install screen && pip install watch gpustat
 ```
 
 ## Prepare Data & Models
@@ -27,7 +27,11 @@ python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0  --fi
 python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0  --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model 
 ```
 
-
+```
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+tokenizer = AutoTokenizer.from_pretrained("google/mt5-xxl", cache_dir="data/.cache")
+model = AutoModelForSeq2SeqLM.from_pretrained("google/mt5-xxl", cache_dir="data/.cache")
+```
 ## Debug Training
 ```
 bash t2v_debug_single.sh

opensora/models/diffusion/opensora/modeling_opensora.py

@@ -603,6 +603,10 @@ class OpenSoraT2V(ModelMixin, ConfigMixin):
         #     output = output[:, :, 1:]
         return output
      
+def OpenSoraT2V_mT5_Baes_S_122(**kwargs):
+    return OpenSoraT2V(num_layers=28, attention_head_dim=96, num_attention_heads=16, patch_size_t=1, patch_size=2,
+                       norm_type="ada_norm_single", caption_channels=768, cross_attention_dim=1536, **kwargs)
+    
 def OpenSoraT2V_S_122(**kwargs):
     return OpenSoraT2V(num_layers=28, attention_head_dim=96, num_attention_heads=16, patch_size_t=1, patch_size=2,
                        norm_type="ada_norm_single", caption_channels=4096, cross_attention_dim=1536, **kwargs)
@@ -620,6 +624,7 @@ def OpenSoraT2V_ROPE_L_122(**kwargs):
                        norm_type="ada_norm_single", caption_channels=4096, cross_attention_dim=2304, **kwargs)
 
 OpenSora_models = {
+    "OpenSoraT2V-mT5-Base-S/122": OpenSoraT2V_mT5_Baes_S_122,
     "OpenSoraT2V-S/122": OpenSoraT2V_S_122,  #       1.1B
     "OpenSoraT2V-B/122": OpenSoraT2V_B_122,
     "OpenSoraT2V-L/122": OpenSoraT2V_L_122,
@@ -627,6 +632,7 @@ OpenSora_models = {
 }
 
 OpenSora_models_class = {
+    "OpenSoraT2V-mT5-Base-S/122": OpenSoraT2V,
     "OpenSoraT2V-S/122": OpenSoraT2V,
     "OpenSoraT2V-B/122": OpenSoraT2V,
     "OpenSoraT2V-L/122": OpenSoraT2V,

opensora/models/text_encoder/__init__.py

@@ -37,6 +37,7 @@ class CLIPWrapper(nn.Module):
 
 
 text_encoder = {
+    'google/mt5-base': T5Wrapper,
     'google/mt5-xl': T5Wrapper,
     'google/mt5-xxl': T5Wrapper,
     'google/umt5-xl': T5Wrapper,

pyproject.toml

@@ -20,7 +20,8 @@ dependencies = [
 "regex==2024.9.11", "requests==2.31.0", "scikit-learn==1.5.2", "scipy==1.14.1", "six==1.16.0", "test-tube==0.7.5", 
 "timm==1.0.11", "torchdiffeq==0.2.4", "torchmetrics==1.5.1", "tqdm==4.66.5", "urllib3==2.2.0", "uvicorn==0.32.0", 
 "scikit-video==1.1.11", "imageio-ffmpeg==0.5.1", "sentencepiece==0.2.0", "beautifulsoup4==4.12.3", "ftfy==6.3.0", 
-"moviepy==1.0.3", "wandb==0.18.5", "tensorboard==2.18.0", "pydantic==2.9.2", "gradio==5.3.0", "huggingface_hub==0.26.1", "protobuf==5.28.3"
+"moviepy==1.0.3", "wandb==0.18.5", "tensorboard==2.18.0", "pydantic==2.9.2", "gradio==5.3.0", "huggingface_hub==0.26.1", "protobuf==5.28.3",
+"watch", "gpustat"
 ]
 
 [project.optional-dependencies]

scripts/text_condition/overfit.sh

@@ -0,0 +1,67 @@
+# export WANDB_MODE="offline"
+
+export ENTITY="linbin"
+export HF_DATASETS_OFFLINE=1 
+export TRANSFORMERS_OFFLINE=1
+export PDSH_RCMD_TYPE=ssh
+# NCCL setting
+export GLOO_SOCKET_IFNAME=eth0
+export NCCL_SOCKET_IFNAME=eth0
+export NCCL_IB_HCA=mlx5_10:1,mlx5_11:1,mlx5_12:1,mlx5_13:1
+export NCCL_IB_GID_INDEX=3
+export NCCL_IB_TC=162
+export NCCL_IB_TIMEOUT=22
+export NCCL_PXN_DISABLE=0
+export NCCL_IB_QPS_PER_CONNECTION=4
+export NCCL_ALGO=Ring
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+
+accelerate launch \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
+    opensora/train/train_t2v_diffusers.py \
+    --model OpenSoraT2V-ROPE-L/122 \
+    --text_encoder_name google/mt5-xxl \
+    --cache_dir "data/.cache" \
+    --dataset t2v \
+    --data "data/dummyVid/merge.txt" \
+    --ae CausalVAEModel_D4_4x8x8 \
+    --ae_path "data/Open-Sora-Plan-v1.2.0/vae/" \
+    --sample_rate 1 \
+    --num_frames 29 \
+    --max_height 480 \
+    --max_width 640 \
+    --pretrained /ephemeral/hao.zhang/codefolder/FastVideo-OSP/data/Open-Sora-Plan-v1.2.0/29x480p/diffusion_pytorch_model.safetensors \
+    --interpolation_scale_t 1.0 \
+    --interpolation_scale_h 1.0 \
+    --interpolation_scale_w 1.0 \
+    --attention_mode xformers \
+    --gradient_checkpointing \
+    --train_batch_size=1 \
+    --dataloader_num_workers 10 \
+    --gradient_accumulation_steps=8 \
+    --max_train_steps=1000000 \
+    --learning_rate=1e-4 \
+    --lr_scheduler="constant" \
+    --lr_warmup_steps=0 \
+    --mixed_precision="bf16" \
+    --report_to="wandb" \
+    --checkpointing_steps=25 \
+    --allow_tf32 \
+    --model_max_length 512 \
+    --use_image_num 0 \
+    --tile_overlap_factor 0.125 \
+    --snr_gamma 5.0 \
+    --use_ema \
+    --ema_start_step 0 \
+    --cfg 0.1 \
+    --noise_offset 0.02 \
+    --use_rope \
+    --resume_from_checkpoint="latest" \
+    --ema_decay 0.999 \
+    --enable_tiling \
+    --speed_factor 1.0 \
+    --group_frame \
+    --sp_size 4 \
+    --train_sp_batch_size 1 \
+    --output_dir="outputs/debug_1node_vid_newpipeline_acc" 

t2v_debug_multi.sh

@@ -3,12 +3,11 @@
 accelerate launch \
     --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
     opensora/train/train_t2v_diffusers.py \
-    --model OpenSoraT2V-ROPE-L/122 \
-    --pretrained data/Open-Sora-Plan-v1.2.0/29x480p/diffusion_pytorch_model.safetensors \
-    --text_encoder_name google/mt5-xxl \
     --cache_dir "data/.cache" \
+    --model OpenSoraT2V-mT5-Base-S/122 \
+    --text_encoder_name google/mt5-base \
     --dataset t2v \
-    --data "data/dummyVid/merge.txt" \
+    --data "data/dummyVid/merge_single.txt" \
     --ae CausalVAEModel_D4_4x8x8 \
     --ae_path "data/Open-Sora-Plan-v1.2.0/vae" \
     --sample_rate 1 \
@@ -24,7 +23,7 @@ accelerate launch \
     --gradient_checkpointing \
     --train_batch_size=1 \
     --dataloader_num_workers 1 \
-    --gradient_accumulation_steps=1 \
+    --gradient_accumulation_steps=8 \
     --max_train_steps=1000000 \
     --learning_rate=1e-4 \
     --lr_scheduler="constant" \
@@ -46,7 +45,6 @@ accelerate launch \
     --ema_decay 0.999 \
     --enable_tiling \
     --speed_factor 1.0 \
-    --group_frame \
-    --sp_size 2 \
+    --sp_size 1 \
     --train_sp_batch_size 1 \
     --output_dir="outputs/debug" \