nodes/hao-ai-lab-FastVideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[cleanup] (#72)

Browse Source 
Co-authored-by: foreverpiano <pianoqwz@gmail.com>
This commit is contained in:
Zhang Peiyuan
2024-12-05 21:02:10 -08:00
committed by GitHub
co-authored by foreverpiano
parent 8cfdf58a17
commit 8d41d505fe
90 changed files with 380 additions and 4566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/experiment_1.sh
-56
View File
@@ -1,56 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--uncond_prompt_dir "data/validation_embeddings/uncond_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 250\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="data/outputs/lq_euler_50_thresh_0.025"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale 4.5 \
--num_euler_timesteps 50
gsutil cp data/outputs/lq_euler_50/checkpoint-4000 gs://vid_gen/runlong_temp_folder_for_pandas70m_debugging/fastvid/lq_euler_50/checkpoint-4000
scripts/experiment_10.sh
-49
View File
@@ -1,49 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 4\
--dataloader_num_workers 4\
--gradient_accumulation_steps=2\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thresh0.05_bs32"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "2.5,3.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.05
scripts/experiment_11.sh
-45
View File
@@ -1,45 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-7\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="data/outputs/lq_euler_50_thresh0.05_lr_1e-7"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "2.5,3.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.05
scripts/experiment_12.sh
-44
View File
@@ -1,44 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/shift1_euler_50"\
--tracker_project_name PCM \
--num_frames 163 \
--shift 1 \
--validation_guidance_scale "2.5,3.5,4.5" \
--num_euler_timesteps 50
scripts/experiment_13.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_14.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_cfg_4.5"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--distill_cfg 4.5
scripts/experiment_15.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.15_lrg_0.75"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.15 \
--linear_range 0.75
scripts/experiment_16.sh
-54
View File
@@ -1,54 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_ema_0.95_decay0.0"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--use_ema \
--ema_decay 0.95 \
--weight_decay 0.0
scripts/experiment_17-node1.sh
-57
View File
@@ -1,57 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=lq_euler_50_thres0.1_lrg_0.75_cfg0.0
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 1 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "1.5,2.5,4.5,6.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--not_apply_cfg_solver
scripts/experiment_17-node8.sh
-55
View File
@@ -1,55 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=lq_euler_50_thres0.1_lrg_0.75_cfg0.0
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 8 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 24\
--sp_size 8\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 139 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "1.5,2.5,4.5,6.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--not_apply_cfg_solver
scripts/experiment_18-node1.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=shift16_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 1 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--shift 16 \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50
scripts/experiment_18-node8.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=shift16_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 8 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 24\
--sp_size 8\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 139 \
--shift 16 \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50
scripts/experiment_19-node1.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_shift16_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 1 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 4 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--shift 16 \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50
scripts/experiment_19-node8.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_shift16_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 8 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 24\
--sp_size 8\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 4 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 139 \
--shift 16 \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50
scripts/experiment_2.sh
-48
View File
@@ -1,48 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--uncond_prompt_dir "data/validation_embeddings/uncond_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 250\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="data/outputs/lq_euler_50_thresh0.05"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale 4.5 \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.05
gsutil cp data/outputs/lq_euler_50_thresh0.05/checkpoint-4000 gs://vid_gen/runlong_temp_folder_for_pandas70m_debugging/fastvid/lq_euler_50_thresh0.05/checkpoint-4000
scripts/experiment_20-node1.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_shift12_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 1 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 4 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--shift 12 \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50
scripts/experiment_20-node8.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_shift12_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 8 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 24\
--sp_size 8\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 4 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 139 \
--shift 12 \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50
scripts/experiment_21-node1.sh
-54
View File
@@ -1,54 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_lq_euler_50_thresh0.1_lrg_0.75
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 1 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 4 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "1.5,2.5,4.5,6.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75
scripts/experiment_21-node8.sh
-54
View File
@@ -1,54 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_lq_euler_50_thresh0.1_lrg_0.75
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 8 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 24\
--sp_size 8\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 4 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 139 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "1.5,2.5,4.5,6.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75
scripts/experiment_22-node1.sh
-55
View File
@@ -1,55 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=lq_euler_50_thresh0.1_lrg_0.75_phase1
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 1 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 8\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-7\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 4 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "1.5,2.5,4.5,6.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_22-node8.sh
-55
View File
@@ -1,55 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=lq_euler_50_thresh0.1_lrg_0.75_phase1
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 8 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 24\
--sp_size 8\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-7\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 139 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "1.5,2.5,4.5,6.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_23-node8.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=lq_euler_50_thres0.1_lrg_0.75_bs_64
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 8 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1
scripts/experiment_24-node1.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_shift16_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 1 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=5e-7\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1
scripts/experiment_24-node8.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=offline
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
CACHE_DIR=/data/.cache
EXPERIMENT=4step_infer_shift16_euler_50
OUTPUT_DIR=$DATA_DIR/outputs/$EXPERIMENT
export WANDB_DIR=$DATA_DIR/wandb/
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir $CACHE_DIR \
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 24\
--sp_size 8\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=2000\
--learning_rate=5e-7\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir=$OUTPUT_DIR \
--tracker_project_name PCM \
--num_frames 139 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1
scripts/experiment_25-A100.sh
-49
View File
@@ -1,49 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=data
IP=172.23.30.16
torchrun --nnodes 4 --nproc_per_node 4\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/shift1_euler_50_0.75_phase1"\
--tracker_project_name PCM \
--num_frames 163 \
--shift 1 \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_26-Runlong-H100.sh
-43
View File
@@ -1,43 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
DATA_DIR=data
torchrun --nnodes 1 --nproc_per_node 8\
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 1\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase_ema0.95"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--use_ema \
--ema_decay 0.95
scripts/experiment_27-A100.sh
-49
View File
@@ -1,49 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
DATA_DIR=data
IP=172.23.30.16
torchrun --nnodes 4 --nproc_per_node 4\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase_ema0.95_cfg4.5"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--use_ema \
--ema_decay 0.95 \
--distill_cfg 4.5
scripts/experiment_28.sh
-53
View File
@@ -1,53 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps "4,8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thresh0.1_lrg_0.75_phase1_ema0.95"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--use_ema \
--ema_decay 0.95
scripts/experiment_29.sh
-54
View File
@@ -1,54 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps "4,8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase_ema0.95_cfg6.0"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--use_ema \
--ema_decay 0.95 \
--distill_cfg 6.0
scripts/experiment_3.sh
-47
View File
@@ -1,47 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--uncond_prompt_dir "data/validation_embeddings/uncond_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="data/outputs/shift8_euler_100"\
--tracker_project_name PCM \
--num_frames 163 \
--shift 8.0 \
--validation_guidance_scale 4.5
gsutil cp data/outputs/shift8_euler_100/checkpoint-4000 gs://vid_gen/runlong_temp_folder_for_pandas70m_debugging/fastvid/shift8_euler_100/checkpoint-4000
scripts/experiment_30.sh
-54
View File
@@ -1,54 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps "4,8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thresh0.1_lrg_0.75_phase1_ema0.98_cfg4.5"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--use_ema \
--ema_decay 0.98 \
--distill_cfg 4.5
scripts/experiment_31.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=3e-7\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps "4,8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thresh0.1_lrg_0.75_phase1_lr_3e-7"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_32.sh
-54
View File
@@ -1,54 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps "4,8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thresh0.15_lrg_0.75_phase1_ema0.95_cfg4.5"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.15 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--use_ema \
--ema_decay 0.95 \
--distill_cfg 4.5
scripts/experiment_33.sh
-53
View File
@@ -1,53 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.142.161
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_linear_range_0.75_cfg_7.0"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--distill_cfg 6.0 \
--multi_phased_distill_schedule "4000-8" \
scripts/experiment_34-A100.sh
-46
View File
@@ -1,46 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
DATA_DIR=data
IP=172.23.30.16
torchrun --nnodes 4 --nproc_per_node 4\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=125\
--validation_steps 125\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_reproduce"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-8" \
scripts/experiment_35.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=5e-6\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr5e-6"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_36.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase2_lr1e-6"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-2"
scripts/experiment_37.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=5e-6\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase2_lr5e-6"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-2"
scripts/experiment_38.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=5e-6\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase8_lr5e-6"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-8"
scripts/experiment_39.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-5\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase8_lr1e-5"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-8"
scripts/experiment_4.sh
-49
View File
@@ -1,49 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--uncond_prompt_dir "data/validation_embeddings/uncond_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="data/outputs/shift8_euler_50"\
--tracker_project_name PCM \
--num_frames 163 \
--shift 8.0 \
--validation_guidance_scale 4.5 \
--num_euler_timesteps 50
gsutil cp data/outputs/shift8_euler_50/checkpoint-4000 gs://vid_gen/runlong_temp_folder_for_pandas70m_debugging/fastvid/shift8_euler_50/checkpoint-4000
scripts/experiment_40.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_repro"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_41.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=125\
--validation_steps 125\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_reproduce"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-8" \
scripts/experiment_42.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "4,8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase4_lr1e-6"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-4"
scripts/experiment_43.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=64\
--validation_steps 64\
--validation_sampling_steps "4,8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_cfg6.0"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--distill_cfg 6.0
scripts/experiment_44-A100.sh
-46
View File
@@ -1,46 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
DATA_DIR=data
IP=172.23.30.16
torchrun --nnodes 4 --nproc_per_node 4\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=5e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr_5e-6_test_norm"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1"
scripts/experiment_45.sh
-47
View File
@@ -1,47 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
DATA_DIR=data
IP=172.23.30.16
torchrun --nnodes 4 --nproc_per_node 4\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 14\
--sp_size 2\
--train_sp_batch_size 1\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=5e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr_5e-6_pred_decay_0.1_latent14"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.1
scripts/experiment_46.sh
-136
View File
@@ -1,136 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.1"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.1
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.1_l2"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.1 \
--pred_decay_type "l2"
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.001"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.001
scripts/experiment_47.sh
-135
View File
@@ -1,135 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.05"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.05
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.05_l2"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.05 \
--pred_decay_type "l2"
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.001_l2"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.001 \
--pred_decay_type "l2"
scripts/experiment_48.sh
-136
View File
@@ -1,136 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.139.86
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=1000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.01"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.01
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.01_l2"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.01 \
--pred_decay_type "l2"
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=640\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=128\
--validation_steps 64\
--validation_sampling_steps "8" \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_lrg_0.75_phase1_lr1e-6_pred_decay0.005_l2"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "0.5,1.5,2.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75 \
--multi_phased_distill_schedule "4000-1" \
--pred_decay_weight 0.005 \
--pred_decay_type "l2"
scripts/experiment_5.sh
-49
View File
@@ -1,49 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
# This can avoid OOM
export PYTORCH_CUDA_ALLOC_CONF='max_split_size_mb:1024'
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill_adv.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--uncond_prompt_dir "data/validation_embeddings/uncond_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=250\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="data/outputs"\
--tracker_project_name PCM \
--num_frames 163 \
--shift 8.0 \
--validation_guidance_scale 4.5
gsutil cp data/outputs/adv_shift8_euler_100/checkpoint-4000 gs://vid_gen/runlong_temp_folder_for_pandas70m_debugging/fastvid/adv_shift8_euler_100/checkpoint-4000
scripts/experiment_6.sh
-52
View File
@@ -1,52 +0,0 @@
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
# This can avoid OOM
export PYTORCH_CUDA_ALLOC_CONF='max_split_size_mb:1024'
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export NCCL_DEBUG=INFO
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=[MASTER_NODE_IP_ADDRESS]:29500 \
fastvideo/distill_adv.py\
--seed 42\
--pretrained_model_name_or_path data/mochi\
--cache_dir "data/.cache"\
--data_json_path "data/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "data/validation_embeddings/validation_prompt_embed_mask"\
--uncond_prompt_dir "data/validation_embeddings/uncond_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=250\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="data/outputs/adv_lq_euler_50"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale 4.5 \
--num_euler_timesteps 50
gsutil cp data/outputs/adv_lq_euler_50/checkpoint-4000 gs://vid_gen/runlong_temp_folder_for_pandas70m_debugging/fastvid/adv_lq_euler_50/checkpoint-4000
scripts/experiment_7.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.131.23
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thresh0.05_multiphaseshedule_125-16,250-8,375-4,500-2"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "2.5,3.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.05 \
--multi_phased_distill_schedule "125-16,250-8,375-4,500-2"
scripts/experiment_8.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.142.161
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.1_linear_range_0.75"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "2.5,3.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.1 \
--linear_range 0.75
scripts/experiment_9.sh
-51
View File
@@ -1,51 +0,0 @@
export WANDB_BASE_URL="https://api.wandb.ai"
export WANDB_DIR="$HOME"
export WANDB_MODE=online
export WANDB_API_KEY=4f6de3765d6464f43e0506ec7d785641af645e73
export LD_LIBRARY_PATH=/opt/amazon/efa/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH
export FI_PROVIDER=efa
export FI_EFA_USE_DEVICE_RDMA=1
export NCCL_PROTO=simple
DATA_DIR=/data
IP=10.4.129.100
torchrun --nnodes 2 --nproc_per_node 8\
--node_rank=0 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=$IP:29500 \
fastvideo/distill.py\
--seed 42\
--pretrained_model_name_or_path $DATA_DIR/mochi\
--cache_dir "data/.cache"\
--data_json_path "$DATA_DIR/Merge-30k-Data/video2caption.json"\
--validation_prompt_dir "$DATA_DIR/validation_embeddings/validation_prompt_embed_mask"\
--gradient_checkpointing\
--train_batch_size=1\
--num_latent_t 28\
--sp_size 4\
--train_sp_batch_size 2\
--dataloader_num_workers 4\
--gradient_accumulation_steps=1\
--max_train_steps=4000\
--learning_rate=1e-6\
--mixed_precision="bf16"\
--checkpointing_steps=500\
--validation_steps 125\
--validation_sampling_steps 8 \
--checkpoints_total_limit 3\
--allow_tf32\
--ema_start_step 0\
--cfg 0.0\
--ema_decay 0.999\
--log_validation\
--output_dir="$DATA_DIR/outputs/lq_euler_50_thres0.05_linear_range_0.75"\
--tracker_project_name PCM \
--num_frames 163 \
--scheduler_type pcm_linear_quadratic \
--validation_guidance_scale "2.5,3.5,4.5" \
--num_euler_timesteps 50 \
--linear_quadratic_threshold 0.05 \
--linear_range 0.75
scripts/finetune_T_J.sh
-81
View File
@@ -34,84 +34,3 @@ torchrun --nnodes 4 --nproc_per_node 4 \
--num_frames 91 \
--selective_checkpointing 1.0
torchrun --nnodes 4 --nproc_per_node 4 \
--node_rank=3 \
--rdzv_id=456 \
--rdzv_backend=c10d \
--rdzv_endpoint=172.23.30.15:29500 \
fastvideo/train.py \
--seed 42 \
--pretrained_model_name_or_path data/mochi \
--cache_dir "data/.cache" \
--data_json_path "data/BLACK-MYTH-DREAM-BOOTH/videos2caption.json" \
--validation_prompt_dir "data/BLACK-MYTH-DREAM-BOOTH/validation_prompt_embed_mask" \
--uncond_prompt_dir "data/BLACK-MYTH-DREAM-BOOTH/uncond_prompt_embed_mask" \
--gradient_checkpointing \
--train_batch_size=1 \
--num_latent_t 16 \
--sp_size 4 \
--train_sp_batch_size 4 \
--dataloader_num_workers 4 \
--gradient_accumulation_steps=1 \
--max_train_steps=4000 \
--learning_rate=2e-5 \
--mixed_precision="bf16" \
--checkpointing_steps=100 \
--validation_steps=50 \
--validation_sampling_steps 64 \
--checkpoints_total_limit 3 \
--allow_tf32 \
--ema_start_step 0 \
--cfg 0.1 \
--ema_decay 0.999 \
--log_validation \
--output_dir="data/outputs/black_myth_correct_video_mask" \
--weighting_scheme "uniform" \
--num_frames 91 \
--selective_checkpointing 1.0
torchrun --nnodes 1 --nproc_per_node 4 \
fastvideo/train.py \
--seed 42 \
--pretrained_model_name_or_path data/mochi \
--cache_dir "data/.cache" \
--data_json_path "data/BLACK-MYTH-Finetune-Dataset/videos2caption.json" \
--validation_prompt_dir "data/BLACK-MYTH-Finetune-Dataset/validation_prompt_embed_mask" \
--uncond_prompt_dir "data/BLACK-MYTH-Finetune-Dataset/uncond_prompt_embed_mask" \
--gradient_checkpointing \
--train_batch_size=1 \
--num_latent_t 16 \
--sp_size 4 \
--train_sp_batch_size 4 \
--dataloader_num_workers 4 \
--gradient_accumulation_steps=1 \
--max_train_steps=2000 \
--learning_rate=2e-5 \
--mixed_precision="bf16" \
--checkpointing_steps=100 \
--validation_steps=100 \
--validation_sampling_steps 64 \
--checkpoints_total_limit 3 \
--allow_tf32 \
--ema_start_step 0 \
--cfg 0.0 \
--ema_decay 0.999 \
--log_validation \
--output_dir="data/outputs/black_myth_dream" \
--weighting_scheme "uniform" \
--num_frames 91 \
| 4/2000 [01:23<10:59:33, 19.83s/it, loss=0.74, grad_norm=2.6]
| 2/2000 [00:44<11:59:27, 21.61s/it, loss=0.873, grad_norm=1.57]
| 3/2000 [01:03<11:18:51, 20.40s/it, loss=0.813, grad_norm=1.48]
# FSDP Hybrid:
scripts/data_preprocess/finetune_data_gen.sh → scripts/finetune_data_gen.sh
View File
scripts/overfit.sh
-39
View File
@@ -1,39 +0,0 @@
accelerate launch \
--config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
fastvideo/train.py \
--seed 42 \
--pretrained_model_name_or_path data/mochi \
--cache_dir "data/.cache" \
--data_merge_path "data/Mochi-Synthetic-Data/merge.txt" \
--gradient_checkpointing \
--train_batch_size=1 \
--num_latent_t 14 \
--dataloader_num_workers 1 \
--gradient_accumulation_steps=4 \
--max_train_steps=200 \
--learning_rate=1e-5 \
--lr_scheduler="constant" \
--lr_warmup_steps=0 \
--mixed_precision="bf16" \
--checkpointing_steps=50 \
--allow_tf32 \
--tile_sample_stride 192 \
--ema_start_step 0 \
--cfg 0.1 \
--ema_decay 0.999 \
--enable_tiling \
--sp_size 1 \
--train_sp_batch_size 1 \
--output_dir="data/outputs/debug_1e-5"
python fastvideo/sample/sample_t2v_mochi.py \
--model_path data/mochi \
--prompt "A hand enters the frame, pulling a sheet of plastic wrap over three balls of dough placed on a wooden surface. The plastic wrap is stretched to cover the dough more securely. The hand adjusts the wrap, ensuring that it is tight and smooth over the dough. The scene focuses on the hand's movements as it secures the edges of the plastic wrap. No new objects appear, and the camera remains stationary, focusing on the action of covering the dough." \
--num_frames 79 \
--height 480 \
--width 848 \
--num_inference_steps 64 \
--guidance_scale 4.5 \
--seed 12346 \
--transformer_path data/outputs/debug_1e-5/checkpoint-50/transformer \
--output_path 123.mp4