From e84fdaedde0bed5698103b253e5f1e286a9335fd Mon Sep 17 00:00:00 2001 From: Matthew Noto Date: Mon, 1 Sep 2025 03:49:18 +0000 Subject: [PATCH] add single example preprocessing --- .../SFWan2.1-I2V/distill_dmd_i2v_1.3B.slurm | 28 ++++++------ .../SFWan2.1-I2V/download_single_example.sh | 43 +++++++++++++++++++ .../SFWan2.1-I2V/preprocess_single_example.sh | 29 +++++++++++++ 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 examples/distill/SFWan2.1-I2V/download_single_example.sh create mode 100644 examples/distill/SFWan2.1-I2V/preprocess_single_example.sh diff --git a/examples/distill/SFWan2.1-I2V/distill_dmd_i2v_1.3B.slurm b/examples/distill/SFWan2.1-I2V/distill_dmd_i2v_1.3B.slurm index 7de7ce9e5..f17682d56 100644 --- a/examples/distill/SFWan2.1-I2V/distill_dmd_i2v_1.3B.slurm +++ b/examples/distill/SFWan2.1-I2V/distill_dmd_i2v_1.3B.slurm @@ -44,8 +44,8 @@ GENERATOR_MODEL_PATH="wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers" REAL_SCORE_MODEL_PATH="Wan-AI/Wan2.1-T2V-14B-Diffusers" # Teacher model FAKE_SCORE_MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers" # Critic model -DATA_DIR="data/crush-smol_processed_t2v/combined_parquet_dataset/" -VALIDATION_DATASET_FILE="examples/distill/SFWan2.1-I2V/validation.json" +DATA_DIR="data/crush-smol-single_processed_t2v/combined_parquet_dataset/" +VALIDATION_DATASET_FILE="data/crush-smol-single_processed_t2v/validation.json" # export CUDA_VISIBLE_DEVICES=4,5 # IP=[MASTER NODE IP] @@ -53,7 +53,7 @@ VALIDATION_DATASET_FILE="examples/distill/SFWan2.1-I2V/validation.json" training_args=( --tracker_project_name SFwan_t2v_distill_self_forcing_dmd --output_dir "checkpoints/SFwan_t2v_finetune" - --max_train_steps 4000 + --max_train_steps 500 --train_batch_size 1 --train_sp_batch_size 1 --gradient_accumulation_steps 1 @@ -64,7 +64,7 @@ training_args=( --enable_gradient_checkpointing_type "full" --log_visualization --simulate_generator_forward - --num_frame_per_block 4 + --num_frame_per_block 3 --enable_gradient_masking --gradient_mask_last_n_frames 21 ) @@ -97,17 +97,17 @@ dataset_args=( validation_args=( --log_validation --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 200 - --validation_sampling_steps "3" + --validation_steps 10 + --validation_sampling_steps "1" --validation_guidance_scale "6.0" # not used for dmd inference ) # Optimizer arguments optimizer_args=( - --learning_rate 2e-6 + --learning_rate 1e-5 --mixed_precision "bf16" - --training_state_checkpointing_steps 500 - --weight_only_checkpointing_steps 500 + --training_state_checkpointing_steps 50 + --weight_only_checkpointing_steps 50 --weight_decay 0.01 --betas '0.0,0.999' --max_grad_norm 1.0 @@ -121,10 +121,10 @@ miscellaneous_args=( --dit_precision "fp32" --flow_shift 5 --seed 1000 - --use_ema True - --ema_decay 0.99 - --ema_start_step 200 - --init_weights_from_safetensors "/mnt/weka/home/hao.zhang/wl/Self-Forcing/diffusers_ode_init/model.safetensors" + # --use_ema True + # --ema_decay 0.99 + # --ema_start_step 200 + # --init_weights_from_safetensors "/mnt/weka/home/hao.zhang/wl/Self-Forcing/diffusers_ode_init/model.safetensors" ) # Self-forcing DMD arguments @@ -134,7 +134,7 @@ dmd_args=( --max_timestep_ratio 0.98 --dfake_gen_update_ratio 5 --real_score_guidance_scale 3.0 - --fake_score_learning_rate 4e-7 + --fake_score_learning_rate 8e-6 --fake_score_betas '0.0,0.999' ) diff --git a/examples/distill/SFWan2.1-I2V/download_single_example.sh b/examples/distill/SFWan2.1-I2V/download_single_example.sh new file mode 100644 index 000000000..cf41fd648 --- /dev/null +++ b/examples/distill/SFWan2.1-I2V/download_single_example.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Download the full dataset first +python scripts/huggingface/download_hf.py --repo_id "wlsaidhi/crush-smol-merged" --local_dir "data/crush-smol" --repo_type "dataset" + +# Create a single-example dataset for debugging +SINGLE_EXAMPLE_DIR="data/crush-smol-single" +mkdir -p "$SINGLE_EXAMPLE_DIR/videos" + +# Copy the specific video that matches the validation.json style (macaron crushing) +cp "data/crush-smol/videos/7P02AihYkCU-Scene-005.mp4" "$SINGLE_EXAMPLE_DIR/videos/" + +# Create a single-line videos.txt +echo "videos/7P02AihYkCU-Scene-005.mp4" > "$SINGLE_EXAMPLE_DIR/videos.txt" + +# Create a single-line prompt.txt with the macaron crushing prompt +echo "PIKA_CRUSH A large metal press is shown compressing a pile of colorful macarons, flattening them as if they were under a hydraulic press. The press moves down, crushing the macarons into a pile of crumbs and squishing the colorful filling out." > "$SINGLE_EXAMPLE_DIR/prompt.txt" + +# Generate the JSON file and merge.txt for the single example +python scripts/dataset_preparation/prepare_json_file.py --data_folder "$SINGLE_EXAMPLE_DIR" --output "videos2caption.json" + +# Create a validation.json that uses the same example for consistency +cat > "$SINGLE_EXAMPLE_DIR/validation.json" << 'EOF' +{ + "data": [ + { + "caption": "A large metal press is shown compressing a pile of colorful macarons, flattening them as if they were under a hydraulic press. The press moves down, crushing the macarons into a pile of crumbs and squishing the colorful filling out.", + "image_path": null, + "video_path": null, + "num_inference_steps": 50, + "height": 480, + "width": 832, + "num_frames": 77 + } + ] + } +EOF + +echo "Single example dataset created at $SINGLE_EXAMPLE_DIR" +echo "Contains:" +echo "- 1 video: $(cat $SINGLE_EXAMPLE_DIR/videos.txt)" +echo "- 1 prompt: $(cat $SINGLE_EXAMPLE_DIR/prompt.txt)" +echo "- Validation file created with the same example for consistency" \ No newline at end of file diff --git a/examples/distill/SFWan2.1-I2V/preprocess_single_example.sh b/examples/distill/SFWan2.1-I2V/preprocess_single_example.sh new file mode 100644 index 000000000..7b251401a --- /dev/null +++ b/examples/distill/SFWan2.1-I2V/preprocess_single_example.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +GPU_NUM=1 # 2,4,8 +MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +MODEL_TYPE="wan" +DATA_MERGE_PATH="data/crush-smol-single/merge.txt" +OUTPUT_DIR="data/crush-smol-single_processed_t2v/" + +torchrun --nproc_per_node=$GPU_NUM \ + fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 1 \ + --seed 42 \ + --max_height 480 \ + --max_width 832 \ + --num_frames 77 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --train_fps 16 \ + --samples_per_file 1 \ + --flush_frequency 1 \ + --video_length_tolerance_range 5 \ + --preprocess_task "t2v" + +# Copy the validation.json to the output directory for consistency +cp "data/crush-smol-single/validation.json" "$OUTPUT_DIR/" + +echo "Preprocessing completed. Validation file copied to $OUTPUT_DIR/" \ No newline at end of file