From fd236a06a77b77cd82dfeb710291c350355e7f41 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Fri, 24 Jul 2026 20:29:03 -0700 Subject: [PATCH] add config --- .../wantrack_causal_synth_stage2/README.md | 43 +++++++ .../wantrack_causal_synth_stage2/cd.yaml | 95 ++++++++++++++++ .../wantrack_causal_synth_stage2/sf.yaml | 106 ++++++++++++++++++ .../wantrack_causal_synth_stage2/tf.yaml | 79 +++++++++++++ 4 files changed, 323 insertions(+) create mode 100644 examples/train/configs/ablation/wantrack_causal_synth_stage2/README.md create mode 100644 examples/train/configs/ablation/wantrack_causal_synth_stage2/cd.yaml create mode 100644 examples/train/configs/ablation/wantrack_causal_synth_stage2/sf.yaml create mode 100644 examples/train/configs/ablation/wantrack_causal_synth_stage2/tf.yaml diff --git a/examples/train/configs/ablation/wantrack_causal_synth_stage2/README.md b/examples/train/configs/ablation/wantrack_causal_synth_stage2/README.md new file mode 100644 index 000000000..01881182b --- /dev/null +++ b/examples/train/configs/ablation/wantrack_causal_synth_stage2/README.md @@ -0,0 +1,43 @@ +# WanTrack causal synth stage-2 run + +This directory archives the exact configuration and launch artifacts used by +the completed WanTrack causal run on 2026-07-23/24. + +- Run root: + `/mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32` +- Initial checkpoint: + `/mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias` +- Dataset: + `/mnt/lustre/vlm-s4duan/data/combined_synth_parquets` +- Validation dataset: + `/mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset` +- Topology: 4 rack-3 nodes, 4 GPUs per node, micro-batch 2, global batch 32 +- Attention: 3 latent frames per block, sink 1, local window 6, + relativistic RoPE +- Schedule: TF 3000, CD 2000, SF 1000 optimizer steps +- Validation: every 250 steps; TF uses 30 denoising steps while CD and SF use + 4 denoising steps +- Checkpoints: every 500 optimizer steps + +The TF job resumed from checkpoint 2000 after its validation configuration was +corrected to multi-step sampling. The YAML files in this directory are the +final files used by the run, including their absolute cluster paths. + +`cluster/run_pipeline_node.sh` launches the resumable TF -> CD -> SF pipeline +and exports TF `student`, CD `ema`, and SF `student_ema`. It requires +`WANDB_API_KEY` to be injected through the process environment. The gallery +upload script similarly requires an in-memory `HF_TOKEN`; no credentials are +stored here. + +The original Kubernetes manifest retains the historical workload label +`wantrack-causal-framewise-gbs32`. That label is stale metadata: the training +authority is the three YAML files, all of which use +`num_frames_per_block: 3`. + +The SF EMA export intentionally contains only the trainable checkpoint role. +For standalone inference, four frozen track-encoder parameters were restored +from the initialization checkpoint. See `receipts/sf-full-export.json` for the +full bundle provenance and hashes. + +The `gallery/` scripts produced 64 generated examples plus their 64 matching +ground-truth videos from the full SF EMA bundle. diff --git a/examples/train/configs/ablation/wantrack_causal_synth_stage2/cd.yaml b/examples/train/configs/ablation/wantrack_causal_synth_stage2/cd.yaml new file mode 100644 index 000000000..c77463ee9 --- /dev/null +++ b/examples/train/configs/ablation/wantrack_causal_synth_stage2/cd.yaml @@ -0,0 +1,95 @@ +models: + student: + _target_: fastvideo.train.models.wantrack.WanTrackCausalModel + init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias + transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/tf/transformer/model.safetensors + trainable: true + num_frames_per_block: 3 + freeze_track_encoder: true + track_augmentation: + enabled: true + sparse_object_sampling: true + extra_points: 20 + extra_point_sampling: random + track_dropout_probability: 0.5 + temporal_mask_probability: 0.2 + temporal_mask_chunk_size: 8 + motion_dropout_probability: 0.3 + text_dropout_probability: 0.0 + teacher: + _target_: fastvideo.train.models.wantrack.WanTrackCausalModel + init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias + transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/tf/transformer/model.safetensors + trainable: false + num_frames_per_block: 3 + freeze_track_encoder: true + ema: + _target_: fastvideo.train.models.wantrack.WanTrackCausalModel + init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias + transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/tf/transformer/model.safetensors + trainable: false + num_frames_per_block: 3 + freeze_track_encoder: true +method: + _target_: fastvideo.train.methods.consistency_model.causal_cd.CausalConsistencyDistillationMethod + discrete_cd_N: 48 + guidance_scale: 3.0 + ema_decay: 0.99 + ema_start_step: 200 +training: + distributed: + num_gpus: 16 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 4 + hsdp_shard_dim: 4 + data: + data_path: /mnt/lustre/vlm-s4duan/data/combined_synth_parquets + preprocessed_data_type: i2v_track + dataloader_num_workers: 4 + train_batch_size: 2 + training_cfg_rate: 0.0 + seed: 1000 + num_latent_t: 31 + num_height: 480 + num_width: 832 + num_frames: 121 + optimizer: + learning_rate: 2.0e-6 + betas: [0.0, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + loop: + max_train_steps: 2000 + gradient_accumulation_steps: 1 + checkpoint: + output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/cd/checkpoints + training_state_checkpointing_steps: 500 + checkpoints_total_limit: 4 + tracker: + trackers: [wandb] + entity: kaiqin_kong_ucsd + project_name: wantrack_causal_synth_stage2 + run_name: wantrack_ckpt600_block3_sink1_local6_relative_cd2k_gbs32 + model: + enable_gradient_checkpointing_type: full +callbacks: + grad_clip: + max_grad_norm: 1.0 + wantrack_validation: + _target_: fastvideo.train.callbacks.track_validation.TrackValidationCallback + every_steps: 250 + val_data_path: /mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset + num_val_samples: 4 + num_inference_steps: 4 + output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/cd/validation + seed: 1000 + validate_at_start: false +pipeline: + flow_shift: 6 + dit_config: + local_attn_size: 6 + sink_size: 1 + rope_cache_policy: relativistic + causal_train_attention: triton diff --git a/examples/train/configs/ablation/wantrack_causal_synth_stage2/sf.yaml b/examples/train/configs/ablation/wantrack_causal_synth_stage2/sf.yaml new file mode 100644 index 000000000..36b6c08fb --- /dev/null +++ b/examples/train/configs/ablation/wantrack_causal_synth_stage2/sf.yaml @@ -0,0 +1,106 @@ +models: + student: + _target_: fastvideo.train.models.wantrack.WanTrackCausalModel + init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias + transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/cd/transformer/model.safetensors + trainable: true + num_frames_per_block: 3 + freeze_track_encoder: true + track_augmentation: + enabled: true + sparse_object_sampling: true + extra_points: 20 + extra_point_sampling: random + track_dropout_probability: 0.5 + temporal_mask_probability: 0.2 + temporal_mask_chunk_size: 8 + motion_dropout_probability: 0.3 + text_dropout_probability: 0.0 + teacher: + _target_: fastvideo.train.models.wantrack.WanTrackModel + init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias + trainable: false + freeze_track_encoder: true + disable_custom_init_weights: true + critic: + _target_: fastvideo.train.models.wantrack.WanTrackModel + init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias + trainable: true + freeze_track_encoder: true + disable_custom_init_weights: true +method: + _target_: fastvideo.train.methods.distribution_matching.self_forcing.SelfForcingMethod + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 4.0 + dmd_denoising_steps: [1000, 750, 500, 250] + warp_denoising_step: true + chunk_size: 3 + student_sample_type: sde + context_noise: 0.0 + same_step_across_blocks: true + enable_gradient_in_rollout: true + start_gradient_frame: 0 + fake_score_learning_rate: 4.0e-7 + fake_score_betas: [0.0, 0.999] + fake_score_lr_scheduler: constant +training: + distributed: + num_gpus: 16 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 4 + hsdp_shard_dim: 4 + data: + data_path: /mnt/lustre/vlm-s4duan/data/combined_synth_parquets + preprocessed_data_type: i2v_track + dataloader_num_workers: 4 + train_batch_size: 2 + training_cfg_rate: 0.0 + seed: 1000 + num_latent_t: 31 + num_height: 480 + num_width: 832 + num_frames: 121 + optimizer: + learning_rate: 2.0e-6 + betas: [0.0, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + loop: + max_train_steps: 1000 + gradient_accumulation_steps: 1 + checkpoint: + output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/sf/checkpoints + training_state_checkpointing_steps: 500 + checkpoints_total_limit: 2 + tracker: + trackers: [wandb] + entity: kaiqin_kong_ucsd + project_name: wantrack_causal_synth_stage2 + run_name: wantrack_ckpt600_block3_sink1_local6_relative_sf1k_gbs32 + model: + enable_gradient_checkpointing_type: full +callbacks: + ema: + decay: 0.99 + start_iter: 200 + grad_clip: + max_grad_norm: 1.0 + wantrack_validation: + _target_: fastvideo.train.callbacks.track_validation.TrackValidationCallback + every_steps: 250 + val_data_path: /mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset + num_val_samples: 4 + num_inference_steps: 4 + output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/sf/validation + seed: 1000 + validate_at_start: false +pipeline: + flow_shift: 6 + dit_config: + local_attn_size: 6 + sink_size: 1 + rope_cache_policy: relativistic + causal_train_attention: triton diff --git a/examples/train/configs/ablation/wantrack_causal_synth_stage2/tf.yaml b/examples/train/configs/ablation/wantrack_causal_synth_stage2/tf.yaml new file mode 100644 index 000000000..29283a2e6 --- /dev/null +++ b/examples/train/configs/ablation/wantrack_causal_synth_stage2/tf.yaml @@ -0,0 +1,79 @@ +models: + student: + _target_: fastvideo.train.models.wantrack.WanTrackCausalModel + init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias + trainable: true + num_frames_per_block: 3 + freeze_track_encoder: true + track_augmentation: + enabled: true + sparse_object_sampling: true + extra_points: 20 + extra_point_sampling: random + track_dropout_probability: 0.5 + temporal_mask_probability: 0.2 + temporal_mask_chunk_size: 8 + motion_dropout_probability: 0.3 + text_dropout_probability: 0.0 +method: + _target_: fastvideo.train.methods.fine_tuning.tfsft.TeacherForcingSFTMethod + chunk_size: 3 +training: + distributed: + num_gpus: 16 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 4 + hsdp_shard_dim: 4 + data: + data_path: /mnt/lustre/vlm-s4duan/data/combined_synth_parquets + preprocessed_data_type: i2v_track + dataloader_num_workers: 4 + train_batch_size: 2 + training_cfg_rate: 0.0 + seed: 1000 + num_latent_t: 31 + num_height: 480 + num_width: 832 + num_frames: 121 + optimizer: + learning_rate: 2.0e-6 + betas: [0.0, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + loop: + max_train_steps: 3000 + gradient_accumulation_steps: 1 + checkpoint: + output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/tf/checkpoints + training_state_checkpointing_steps: 500 + checkpoints_total_limit: 6 + tracker: + trackers: [wandb] + entity: kaiqin_kong_ucsd + project_name: wantrack_causal_synth_stage2 + run_name: wantrack_ckpt600_block3_sink1_local6_relative_tf3k_gbs32_resume2000_multistep + model: + enable_gradient_checkpointing_type: full +callbacks: + grad_clip: + max_grad_norm: 1.0 + wantrack_validation: + _target_: fastvideo.train.callbacks.track_validation.TrackValidationCallback + every_steps: 250 + val_data_path: /mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset + num_val_samples: 4 + num_inference_steps: 30 + guidance_scale: 3.0 + motion_guidance_scale: 1.5 + output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/tf/validation + seed: 1000 + validate_at_start: false +pipeline: + flow_shift: 6 + dit_config: + local_attn_size: 6 + sink_size: 1 + rope_cache_policy: relativistic + causal_train_attention: triton