add config

examples/train/configs/ablation/wantrack_causal_synth_stage2/README.md

@@ -0,0 +1,43 @@
+# WanTrack causal synth stage-2 run
+
+This directory archives the exact configuration and launch artifacts used by
+the completed WanTrack causal run on 2026-07-23/24.
+
+- Run root:
+  `/mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32`
+- Initial checkpoint:
+  `/mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias`
+- Dataset:
+  `/mnt/lustre/vlm-s4duan/data/combined_synth_parquets`
+- Validation dataset:
+  `/mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset`
+- Topology: 4 rack-3 nodes, 4 GPUs per node, micro-batch 2, global batch 32
+- Attention: 3 latent frames per block, sink 1, local window 6,
+  relativistic RoPE
+- Schedule: TF 3000, CD 2000, SF 1000 optimizer steps
+- Validation: every 250 steps; TF uses 30 denoising steps while CD and SF use
+  4 denoising steps
+- Checkpoints: every 500 optimizer steps
+
+The TF job resumed from checkpoint 2000 after its validation configuration was
+corrected to multi-step sampling. The YAML files in this directory are the
+final files used by the run, including their absolute cluster paths.
+
+`cluster/run_pipeline_node.sh` launches the resumable TF -> CD -> SF pipeline
+and exports TF `student`, CD `ema`, and SF `student_ema`. It requires
+`WANDB_API_KEY` to be injected through the process environment. The gallery
+upload script similarly requires an in-memory `HF_TOKEN`; no credentials are
+stored here.
+
+The original Kubernetes manifest retains the historical workload label
+`wantrack-causal-framewise-gbs32`. That label is stale metadata: the training
+authority is the three YAML files, all of which use
+`num_frames_per_block: 3`.
+
+The SF EMA export intentionally contains only the trainable checkpoint role.
+For standalone inference, four frozen track-encoder parameters were restored
+from the initialization checkpoint. See `receipts/sf-full-export.json` for the
+full bundle provenance and hashes.
+
+The `gallery/` scripts produced 64 generated examples plus their 64 matching
+ground-truth videos from the full SF EMA bundle.

examples/train/configs/ablation/wantrack_causal_synth_stage2/cd.yaml

@@ -0,0 +1,95 @@
+models:
+  student:
+    _target_: fastvideo.train.models.wantrack.WanTrackCausalModel
+    init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias
+    transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/tf/transformer/model.safetensors
+    trainable: true
+    num_frames_per_block: 3
+    freeze_track_encoder: true
+    track_augmentation:
+      enabled: true
+      sparse_object_sampling: true
+      extra_points: 20
+      extra_point_sampling: random
+      track_dropout_probability: 0.5
+      temporal_mask_probability: 0.2
+      temporal_mask_chunk_size: 8
+      motion_dropout_probability: 0.3
+      text_dropout_probability: 0.0
+  teacher:
+    _target_: fastvideo.train.models.wantrack.WanTrackCausalModel
+    init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias
+    transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/tf/transformer/model.safetensors
+    trainable: false
+    num_frames_per_block: 3
+    freeze_track_encoder: true
+  ema:
+    _target_: fastvideo.train.models.wantrack.WanTrackCausalModel
+    init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias
+    transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/tf/transformer/model.safetensors
+    trainable: false
+    num_frames_per_block: 3
+    freeze_track_encoder: true
+method:
+  _target_: fastvideo.train.methods.consistency_model.causal_cd.CausalConsistencyDistillationMethod
+  discrete_cd_N: 48
+  guidance_scale: 3.0
+  ema_decay: 0.99
+  ema_start_step: 200
+training:
+  distributed:
+    num_gpus: 16
+    sp_size: 1
+    tp_size: 1
+    hsdp_replicate_dim: 4
+    hsdp_shard_dim: 4
+  data:
+    data_path: /mnt/lustre/vlm-s4duan/data/combined_synth_parquets
+    preprocessed_data_type: i2v_track
+    dataloader_num_workers: 4
+    train_batch_size: 2
+    training_cfg_rate: 0.0
+    seed: 1000
+    num_latent_t: 31
+    num_height: 480
+    num_width: 832
+    num_frames: 121
+  optimizer:
+    learning_rate: 2.0e-6
+    betas: [0.0, 0.999]
+    weight_decay: 0.01
+    lr_scheduler: constant
+    lr_warmup_steps: 0
+  loop:
+    max_train_steps: 2000
+    gradient_accumulation_steps: 1
+  checkpoint:
+    output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/cd/checkpoints
+    training_state_checkpointing_steps: 500
+    checkpoints_total_limit: 4
+  tracker:
+    trackers: [wandb]
+    entity: kaiqin_kong_ucsd
+    project_name: wantrack_causal_synth_stage2
+    run_name: wantrack_ckpt600_block3_sink1_local6_relative_cd2k_gbs32
+  model:
+    enable_gradient_checkpointing_type: full
+callbacks:
+  grad_clip:
+    max_grad_norm: 1.0
+  wantrack_validation:
+    _target_: fastvideo.train.callbacks.track_validation.TrackValidationCallback
+    every_steps: 250
+    val_data_path: /mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset
+    num_val_samples: 4
+    num_inference_steps: 4
+    output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/cd/validation
+    seed: 1000
+    validate_at_start: false
+pipeline:
+  flow_shift: 6
+  dit_config:
+    local_attn_size: 6
+    sink_size: 1
+    rope_cache_policy: relativistic
+    causal_train_attention: triton

examples/train/configs/ablation/wantrack_causal_synth_stage2/sf.yaml

@@ -0,0 +1,106 @@
+models:
+  student:
+    _target_: fastvideo.train.models.wantrack.WanTrackCausalModel
+    init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias
+    transformer_override_safetensor: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/export/cd/transformer/model.safetensors
+    trainable: true
+    num_frames_per_block: 3
+    freeze_track_encoder: true
+    track_augmentation:
+      enabled: true
+      sparse_object_sampling: true
+      extra_points: 20
+      extra_point_sampling: random
+      track_dropout_probability: 0.5
+      temporal_mask_probability: 0.2
+      temporal_mask_chunk_size: 8
+      motion_dropout_probability: 0.3
+      text_dropout_probability: 0.0
+  teacher:
+    _target_: fastvideo.train.models.wantrack.WanTrackModel
+    init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias
+    trainable: false
+    freeze_track_encoder: true
+    disable_custom_init_weights: true
+  critic:
+    _target_: fastvideo.train.models.wantrack.WanTrackModel
+    init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias
+    trainable: true
+    freeze_track_encoder: true
+    disable_custom_init_weights: true
+method:
+  _target_: fastvideo.train.methods.distribution_matching.self_forcing.SelfForcingMethod
+  rollout_mode: simulate
+  generator_update_interval: 5
+  real_score_guidance_scale: 4.0
+  dmd_denoising_steps: [1000, 750, 500, 250]
+  warp_denoising_step: true
+  chunk_size: 3
+  student_sample_type: sde
+  context_noise: 0.0
+  same_step_across_blocks: true
+  enable_gradient_in_rollout: true
+  start_gradient_frame: 0
+  fake_score_learning_rate: 4.0e-7
+  fake_score_betas: [0.0, 0.999]
+  fake_score_lr_scheduler: constant
+training:
+  distributed:
+    num_gpus: 16
+    sp_size: 1
+    tp_size: 1
+    hsdp_replicate_dim: 4
+    hsdp_shard_dim: 4
+  data:
+    data_path: /mnt/lustre/vlm-s4duan/data/combined_synth_parquets
+    preprocessed_data_type: i2v_track
+    dataloader_num_workers: 4
+    train_batch_size: 2
+    training_cfg_rate: 0.0
+    seed: 1000
+    num_latent_t: 31
+    num_height: 480
+    num_width: 832
+    num_frames: 121
+  optimizer:
+    learning_rate: 2.0e-6
+    betas: [0.0, 0.999]
+    weight_decay: 0.01
+    lr_scheduler: constant
+    lr_warmup_steps: 0
+  loop:
+    max_train_steps: 1000
+    gradient_accumulation_steps: 1
+  checkpoint:
+    output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/sf/checkpoints
+    training_state_checkpointing_steps: 500
+    checkpoints_total_limit: 2
+  tracker:
+    trackers: [wandb]
+    entity: kaiqin_kong_ucsd
+    project_name: wantrack_causal_synth_stage2
+    run_name: wantrack_ckpt600_block3_sink1_local6_relative_sf1k_gbs32
+  model:
+    enable_gradient_checkpointing_type: full
+callbacks:
+  ema:
+    decay: 0.99
+    start_iter: 200
+  grad_clip:
+    max_grad_norm: 1.0
+  wantrack_validation:
+    _target_: fastvideo.train.callbacks.track_validation.TrackValidationCallback
+    every_steps: 250
+    val_data_path: /mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset
+    num_val_samples: 4
+    num_inference_steps: 4
+    output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/sf/validation
+    seed: 1000
+    validate_at_start: false
+pipeline:
+  flow_shift: 6
+  dit_config:
+    local_attn_size: 6
+    sink_size: 1
+    rope_cache_policy: relativistic
+    causal_train_attention: triton

examples/train/configs/ablation/wantrack_causal_synth_stage2/tf.yaml

@@ -0,0 +1,79 @@
+models:
+  student:
+    _target_: fastvideo.train.models.wantrack.WanTrackCausalModel
+    init_from: /mnt/lustre/vlm-k1kong/models/wantrack-synth-stage2-ckpt600-bias
+    trainable: true
+    num_frames_per_block: 3
+    freeze_track_encoder: true
+    track_augmentation:
+      enabled: true
+      sparse_object_sampling: true
+      extra_points: 20
+      extra_point_sampling: random
+      track_dropout_probability: 0.5
+      temporal_mask_probability: 0.2
+      temporal_mask_chunk_size: 8
+      motion_dropout_probability: 0.3
+      text_dropout_probability: 0.0
+method:
+  _target_: fastvideo.train.methods.fine_tuning.tfsft.TeacherForcingSFTMethod
+  chunk_size: 3
+training:
+  distributed:
+    num_gpus: 16
+    sp_size: 1
+    tp_size: 1
+    hsdp_replicate_dim: 4
+    hsdp_shard_dim: 4
+  data:
+    data_path: /mnt/lustre/vlm-s4duan/data/combined_synth_parquets
+    preprocessed_data_type: i2v_track
+    dataloader_num_workers: 4
+    train_batch_size: 2
+    training_cfg_rate: 0.0
+    seed: 1000
+    num_latent_t: 31
+    num_height: 480
+    num_width: 832
+    num_frames: 121
+  optimizer:
+    learning_rate: 2.0e-6
+    betas: [0.0, 0.999]
+    weight_decay: 0.01
+    lr_scheduler: constant
+    lr_warmup_steps: 0
+  loop:
+    max_train_steps: 3000
+    gradient_accumulation_steps: 1
+  checkpoint:
+    output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/tf/checkpoints
+    training_state_checkpointing_steps: 500
+    checkpoints_total_limit: 6
+  tracker:
+    trackers: [wandb]
+    entity: kaiqin_kong_ucsd
+    project_name: wantrack_causal_synth_stage2
+    run_name: wantrack_ckpt600_block3_sink1_local6_relative_tf3k_gbs32_resume2000_multistep
+  model:
+    enable_gradient_checkpointing_type: full
+callbacks:
+  grad_clip:
+    max_grad_norm: 1.0
+  wantrack_validation:
+    _target_: fastvideo.train.callbacks.track_validation.TrackValidationCallback
+    every_steps: 250
+    val_data_path: /mnt/lustre/vlm-s4duan/val_examples_mixed/combined_parquet_dataset
+    num_val_samples: 4
+    num_inference_steps: 30
+    guidance_scale: 3.0
+    motion_guidance_scale: 1.5
+    output_dir: /mnt/lustre/vlm-k1kong/experiments/wantrack_causal/20260723_070728/wantrack-ckpt600-block3-local6-sink1-relative-gbs32/tf/validation
+    seed: 1000
+    validate_at_start: false
+pipeline:
+  flow_shift: 6
+  dit_config:
+    local_attn_size: 6
+    sink_size: 1
+    rope_cache_policy: relativistic
+    causal_train_attention: triton