[misc]: Pika H3 Flex Ref2VA QAD run kit, pinned calibration table, final recipe config

2026-10-30 13:57:47 -07:00
parent 72424d0692
commit e6979289ca
18 changed files with 6159 additions and 7 deletions
@@ -0,0 +1,15 @@
+# Pika H3 Flex Ref2VA NVFP4 QAD: Slinky (GB200) run kit
+
+Cluster scripts and resolved configs exactly as used for the first Flex Ref2VA QAD run (2026-10-30). Paths are the
+cluster's (`/mnt/nfs/vlm-aryan/pika-flex-qad`); the code there is a checkout of this branch. Jobs attach to held
+trays with `srun --overlap`.
+
+| Step | Script | Notes |
+|---|---|---|
+| Teachers | `merge.sh` | Downloads both adapters and runs `merge_pika_adapter.py` for ref2va and fl2va. |
+| Coverage rows | `build_rows.sh plan / encode / finalize` | `build_flex_ref2va_rows.py`: 480p rows at Flex reference sizing plus audio-reference, mixed image+video, 6-9 image and two-video rows, from the stored v4 latents. 1,186 rows. |
+| Calibration | `calib_pipeline.sh` (uses `calib.sh`) | collect (1,133 rows, 8 GPUs) -> candidates -> MSE -> choose -> step-0 PTQ evals. Step-0 scores: max 0.33112, MSE 0.32987, balanced 0.32595 (pinned, `examples/train/calibration/pika_h3_flex/amax-ref2va-balanced.json`). |
+| Warm-up trial | `theseus_trial.sh` | 8 steps each: baseline 0.34472 -> 0.32510, Theseus (4 warm-up + 4) -> 0.32943. Not adopted. |
+| QAD | `qad_node.sh` (4 nodes x 4 GPUs) with `ref2va-qad.yaml` | Needs `FASTVIDEO_PG_TIMEOUT_MIN=120` (the default 10-minute collective timeout fires during the model load). |
+
+`env.sh` sources the cluster's `secrets.env` by path; no credentials are stored here.
@@ -0,0 +1,10 @@
+#!/bin/bash
+# usage: build_rows.sh plan | encode <num_shards> <shard_base> | finalize   (runs under env.sh; encode = one process per GPU)
+O=/mnt/nfs/vlm-aryan/pika-flex-qad/data/flex-ref2va; V4=/mnt/lustre/vlm-wlsaidhi/fastvideo/data/omniref-v4-processed/20260919T001000Z
+EVAL=/mnt/nfs/vlm-aryan/h3-nvfp4-decoder/gen-latents/omniref-eval/manifest.json
+case $1 in
+  plan) $PY /mnt/nfs/vlm-aryan/pika-flex-qad/code/scripts/distill/pika_h3_flex_qad/build_flex_ref2va_rows.py plan --output-dir $O --v4-manifest $V4/encoded/full/MANIFEST.jsonl --plan-config /mnt/nfs/vlm-aryan/pika-flex-qad/ref2va-qad.yaml --models.student.data.max_frames 345 --models.student.data.per_group.480p 160 --models.student.data.per_group.768p 40 --eval-manifest $EVAL --extra-per-group 480p=80 768p=20 ;;
+  encode) for g in 0 1 2 3; do CUDA_VISIBLE_DEVICES=$g MASTER_PORT=$((29700 + g)) $PY /mnt/nfs/vlm-aryan/pika-flex-qad/code/scripts/distill/pika_h3_flex_qad/build_flex_ref2va_rows.py encode --output-dir $O --model-path /mnt/nfs/vlm-aryan/pika-flex-qad/models/pika-flex-ref2va-merged --shard $(($3 + g)) --num-shards $2 ${@:4} >> $O/encode-$(($3 + g)).log 2>&1 & done; wait ;;
+  finalize) $PY /mnt/nfs/vlm-aryan/pika-flex-qad/code/scripts/distill/pika_h3_flex_qad/build_flex_ref2va_rows.py finalize --output-dir $O --eval-manifest $EVAL ;;
+esac
+echo BUILD_DONE $1
@@ -0,0 +1,15 @@
+#!/bin/bash
+# usage: calib.sh <collect|mse|candidates|choose> <config> <out_dir> [shard num_shards] [--dotted.key value ...]  (runs under env.sh)
+set -uo pipefail
+STAGE=$1; CFG=$2; OUT=$3; shift 3
+S=/mnt/nfs/vlm-aryan/pika-flex-qad/code/scripts/distill/pika_h3_flex_qad/calibrate_flex_nvfp4.py
+mkdir -p $OUT
+case $STAGE in
+  collect|mse)
+    SH=$1; NS=$2; shift 2
+    FASTVIDEO_PG_TIMEOUT_MIN=120 $PY -m torch.distributed.run --nproc_per_node=${NPROC:-4} --master_port=$((29600 + SH)) \
+      --log-dir $OUT/torchlogs-$STAGE-$SH --redirects 3 --tee 3 $S $STAGE --config $CFG --output-dir $OUT \
+      --shard $SH --num-shards $NS --training.distributed.num_gpus ${NPROC:-4} --training.distributed.hsdp_shard_dim ${NPROC:-4} --training.distributed.sp_size ${NPROC:-4} "$@" ;;
+  *) $PY $S $STAGE --output-dir $OUT "$@" ;;
+esac
+echo "CALIB_STAGE_DONE $STAGE"
@@ -0,0 +1,31 @@
+#!/bin/bash
+# Overnight Flex Ref2VA calibration on trays 13509 + 13483 (login node, in tmux): finalize coverage rows,
+# collect, candidates, MSE, choose, then step-0 PTQ evals of each table. Log: logs/calib-pipeline.log
+R=/mnt/nfs/vlm-aryan/pika-flex-qad; C=$R/calib/ref2va; CFG=$R/ref2va-qad.yaml; L=$R/logs
+T0=13509; T1=13483
+run() {  # run <tray> <log> <cmd...>: one container step under env.sh, foreground
+  local t=$1 log=$2; shift 2
+  srun --overlap --jobid=$t --nodes=1 --ntasks=1 --kill-on-bad-exit=1 --container-image=nvcr.io/nvidia/pytorch:25.06-py3 \
+    --container-mounts=/mnt/nfs/vlm-aryan:/mnt/nfs/vlm-aryan,/mnt/lustre:/mnt/lustre:ro bash $R/env.sh "$@" > $log 2>&1
+}
+say() { echo "$(date -u +%FT%TZ) $*"; }
+say "start: coverage rows final (retry stopped)"
+run $T0 $L/build-finalize.log bash $R/jobs/build_rows.sh finalize; tail -3 $L/build-finalize.log
+A="--models.student.data.per_group.480p 120 --models.student.data.per_group.768p 30"
+say "collect"
+run $T0 $L/calib-collect-0.log bash $R/jobs/calib.sh collect $CFG $C 0 2 $A & run $T1 $L/calib-collect-1.log bash $R/jobs/calib.sh collect $CFG $C 1 2 $A & wait
+grep -h "CALIB_STAGE_DONE\|Traceback" $L/calib-collect-*.log; wc -l $C/collect-*.jsonl
+run $T0 $L/calib-candidates.log bash $R/jobs/calib.sh candidates - $C --outlier-rows 8; tail -2 $L/calib-candidates.log
+say "mse"
+run $T0 $L/calib-mse-0.log bash $R/jobs/calib.sh mse $CFG $C 0 2 --mse-rows-per-group 3 & run $T1 $L/calib-mse-1.log bash $R/jobs/calib.sh mse $CFG $C 1 2 --mse-rows-per-group 3 & wait
+run $T0 $L/calib-choose.log bash $R/jobs/calib.sh choose - $C; tail -3 $L/calib-choose.log
+say "step-0 PTQ evals"
+ptq() {  # ptq <tray> <name> <table>
+  run $1 $L/ptq-$2.log bash $R/jobs/smoke.sh $CFG $R/runs/ptq-$2 --models.student.qad.linears.amax_json $3 \
+    --training.distributed.num_gpus 4 --training.distributed.hsdp_shard_dim 4 --training.loop.max_train_steps 1 \
+    --training.optimizer.learning_rate 0.0 --method.noise_floor_eval false --training.tracker.trackers "[]"
+  grep -o "QAD eval @0: score [0-9.]* (PTQ [0-9.]*)" $L/ptq-$2.log | head -1 | sed "s/^/$2: /"
+}
+ptq $T0 max $C/amax-max.json & ptq $T1 mse $C/amax-mse.json & wait
+ptq $T0 balanced $C/amax-mse-balanced.json
+say "CALIB_PIPELINE_DONE"
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Calibration smoke on one tray: 2 collect rows, candidates, 2 MSE rows + outliers, choose.
+R=/mnt/nfs/vlm-aryan/pika-flex-qad; O=$R/runs/calib-smoke; rm -rf $O
+bash $R/jobs/calib.sh collect $R/smoke-ref2va.yaml $O 0 1 --max-rows 2 || exit 1
+bash $R/jobs/calib.sh candidates - $O --outlier-rows 1 || exit 1
+bash $R/jobs/calib.sh mse $R/smoke-ref2va.yaml $O 0 1 --mse-rows-per-group 1 || exit 1
+bash $R/jobs/calib.sh choose - $O || exit 1
+ls -la $O
@@ -0,0 +1,23 @@
+# Pika H3 Flex QAD environment (OmniRef QAD v2 env with this project code). Usage: env.sh <cmd...>
+set -a
+Q=/mnt/nfs/vlm-aryan/h3-omniref-qad
+source /mnt/nfs/vlm-aryan/fasth3-33b-20260806/secrets.env >/dev/null 2>&1
+EXP=/mnt/lustre/vlm-wlsaidhi/fastvideo/experiments/main-hf-step3500-gb200-20261005T203006Z
+KERNEL=$EXP/kernel/install
+PY=/mnt/nfs/vlm-aryan/h3-nvfp4-decoder/venv/bin/python
+CODE=/mnt/nfs/vlm-aryan/pika-flex-qad/code
+PYTHONPATH=$KERNEL:$CODE
+HF_HOME=/mnt/nfs/vlm-aryan/hf-cache HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_HUB_DISABLE_TELEMETRY=1
+WANDB_MODE=${WANDB_MODE:-online} WANDB_DIR=/mnt/nfs/vlm-aryan/pika-flex-qad/wandb WANDB_PROJECT=pika-h3-flex-nvfp4 WANDB_ENTITY=aryan5v-san-jose-state-university
+PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=8
+SCRATCH=/tmp/pika-flex-${SLURM_JOB_ID:-nojob}
+HOME=$SCRATCH/home XDG_CACHE_HOME=$SCRATCH/cache XDG_CONFIG_HOME=$SCRATCH/config TMPDIR=$SCRATCH/tmp
+TRITON_CACHE_DIR=/tmp/triton-pika-flex TORCHINDUCTOR_CACHE_DIR=/tmp/inductor-pika-flex TORCH_EXTENSIONS_DIR=/tmp/torchext-pika-flex
+NUMBA_CACHE_DIR=$SCRATCH/numba FLASHINFER_WORKSPACE_BASE=/mnt/nfs/vlm-aryan/h3-omniref-qad/cache/flashinfer
+mkdir -p $HOME $XDG_CACHE_HOME $XDG_CONFIG_HOME $TMPDIR $NUMBA_CACHE_DIR $TRITON_CACHE_DIR $TORCHINDUCTOR_CACHE_DIR $TORCH_EXTENSIONS_DIR $WANDB_DIR
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+TORCH_NCCL_ENABLE_MONITORING=0 NCCL_CUMEM_ENABLE=0 NCCL_MNNVL_ENABLE=0 NCCL_NVLS_ENABLE=0 NCCL_NET=IB NCCL_IB_HCA==mlx5_0,mlx5_1,mlx5_3,mlx5_4 NCCL_SOCKET_IFNAME=eth0 NCCL_SOCKET_FAMILY=AF_INET NCCL_DEBUG=WARN
+FASTVIDEO_MINIMAX_H3_FUSIONS=0 FASTVIDEO_H3_VSA_FP4=0 FASTVIDEO_VSA_SM100A=0
+set +a
+cd $CODE
+exec "$@"
@@ -0,0 +1,8 @@
+#!/bin/bash
+# usage: launch_o.sh <tray jobid> <timeout seconds> <log> <cmd...>  (Pika Flex env; detached on the login node)
+J=$1; T=$2; LOG=$3; shift 3
+nohup timeout $((T+120)) srun --overlap --jobid=$J --nodes=1 --ntasks=1 --kill-on-bad-exit=1 \
+  --container-image=nvcr.io/nvidia/pytorch:25.06-py3 \
+  --container-mounts=/mnt/nfs/vlm-aryan:/mnt/nfs/vlm-aryan,/mnt/lustre:/mnt/lustre:ro \
+  timeout $T bash /mnt/nfs/vlm-aryan/pika-flex-qad/env.sh "$@" > $LOG 2>&1 &
+echo "launched pid $! on tray $J -> $LOG"
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Download the two Pika H3 Flex adapters and merge each into its MiniMax-H3 trunk (runs under env.sh).
+set -euo pipefail
+R=/mnt/nfs/vlm-aryan/pika-flex-qad
+BASE=$HF_HOME/hub/models--MiniMaxAI--MiniMax-H3/snapshots/42ed227ee7df40d41602854ae760620d6eb651fe
+ADAPTERS=$R/models/pika-h3-flex-adapters
+for task in ref2va fl2va; do
+  out=$R/models/pika-flex-$task-merged
+  if [ -f $out/pika_flex_qad.json ]; then echo "$out exists, skipping"; continue; fi
+  rm -rf $out
+  time $PY scripts/distill/pika_h3_flex_qad/merge_pika_adapter.py --base $BASE --adapter $ADAPTERS --task $task --output $out
+  cat $out/pika_flex_qad.json
+done
@@ -0,0 +1,13 @@
+#!/bin/bash
+# One-tray QAD pilot (SP 4): pilot.sh <tray> <name> <amax table> [--dotted.key value ...]; logs/pilot-<name>.log, runs/pilot-<name>
+R=/mnt/nfs/vlm-aryan/pika-flex-qad; T=$1; N=$2; A=$3; shift 3
+nohup srun --overlap --jobid=$T --nodes=1 --ntasks=1 --kill-on-bad-exit=1 --container-image=nvcr.io/nvidia/pytorch:25.06-py3 \
+  --container-mounts=/mnt/nfs/vlm-aryan:/mnt/nfs/vlm-aryan,/mnt/lustre:/mnt/lustre:ro \
+  bash $R/env.sh bash $R/jobs/smoke.sh $R/ref2va-qad.yaml $R/runs/pilot-$N \
+  --models.student.qad.linears.amax_json $A --training.distributed.num_gpus 4 --training.distributed.hsdp_shard_dim 4 \
+  --training.loop.max_train_steps 60 --method.eval_every 15 --method.full_eval_every 30 --method.patience 99 \
+  --method.noise_floor_eval false --method.max_rollout_rows 4 \
+  --models.student.data.eval_per_group.480p 1 --models.student.data.eval_per_group.768p 1 \
+  --training.tracker.run_name pilot-$N --training.checkpoint.training_state_checkpointing_steps 1000 "$@" \
+  > $R/logs/pilot-$N.log 2>&1 &
+echo "pilot $N on tray $T (pid $!) -> $R/logs/pilot-$N.log"
@@ -0,0 +1,6 @@
+#!/bin/bash
+# usage: qad_node.sh <node_rank> <master_ip> <out_dir> <config> [overrides]  (4 nodes x 4 GPUs; runs under env.sh)
+NR=$1; MASTER=$2; OUT=$3; CFG=$4; shift 4
+mkdir -p $OUT
+FASTVIDEO_PG_TIMEOUT_MIN=120 $PY -m torch.distributed.run --nnodes=4 --nproc_per_node=4 --node_rank=$NR --master_addr=$MASTER \
+  --master_port=29650 --log-dir $OUT/torchlogs-n$NR --redirects 3 --tee 3 -m fastvideo.train.entrypoint.train --config $CFG "$@"
@@ -0,0 +1,136 @@
+# Pika H3 Flex Ref2VA NVFP4 QAD (the FastH3 QAD recipe on Flex's six-forward ladder).
+#
+# Teacher: Pika H3 Flex Ref2VA in bf16, i.e. MiniMax-H3 `transformer_ref` with the
+# released `ref2va` adapter merged by
+# scripts/distill/pika_h3_flex_qad/merge_pika_adapter.py. Student: the same
+# weights with NVFP4 W4A4 q/k/v/out plus FFN fc_in/fc_out in all 50 blocks
+# active from step 0, static FFN activation scales from the Flex calibration
+# table. Attention stays BF16 dense: Flex has no VSA gate, so there is no FP4
+# sparse-attention stage and no gate to freeze. Norms, AdaLN, the token refiner
+# and the FP32 boundary modules stay frozen.
+#
+# The objective, evaluation, guards and checkpoint contract are the shared
+# recipe's (docs/training/minimax_h3_fp4_qad.md). Rows are OmniRef Ref2VA
+# parquet rows; the teacher computes every target on the fly, so the OmniRef
+# held-out eval manifest is reused as is.
+#
+# Paths are placeholders: the merged model, the calibration table (pin its
+# sha256 once calibrated), the row manifests and the output dir.
+models:
+  student:
+    _target_: fastvideo.train.models.minimax_h3.flex.MiniMaxH3FlexRef2VAModel
+    init_from: /mnt/nfs/vlm-aryan/pika-flex-qad/models/pika-flex-ref2va-merged
+    trainable: true
+    enable_gradient_checkpointing_type: full
+    qad:
+      enabled: true
+      linears:
+        enabled: true
+        amax_json: /mnt/nfs/vlm-aryan/pika-flex-qad/calib/amax-ref2va-balanced-ad60b45fea00.json
+        amax_sha256: ad60b45fea00906c55e370a0dec82638eb413d6b68537d1ceb38fc52da90b373
+        # Attention linears use the unit activation scale; only the FFN
+        # linears read their calibrated amax from this table.
+        attention: true
+        ffn: true
+        skip_blocks: []
+      fp4_attention:
+        enabled: false
+    data:
+      manifests:
+      # Flex-tier rows first: they replace the v4 rows with the same id and resolution.
+      - /mnt/nfs/vlm-aryan/pika-flex-qad/data/flex-ref2va/MANIFEST.jsonl
+      - /mnt/lustre/vlm-wlsaidhi/fastvideo/data/omniref-v4-processed/20260919T001000Z/encoded/full/MANIFEST.jsonl
+      cases:
+      - first_frame
+      - first_last_frame
+      - storyboard
+      - continue_scene
+      - continue_shot
+      - audio_reference
+      - mixed_image_video
+      - many_images
+      - multi_video
+      per_group:
+        480p: 160
+        768p: 40
+      seed: 20261007
+      max_frames: 345
+      eval_manifest: /mnt/nfs/vlm-aryan/pika-flex-qad/data/flex-ref2va/eval-manifest.json
+      eval_resolutions:
+      - 480p
+      - 768p
+      eval_per_group:
+        480p: 2
+        768p: 1
+      eval_max_frames: 0
+  teacher:
+    _target_: fastvideo.train.models.minimax_h3.flex.MiniMaxH3FlexRef2VAModel
+    init_from: /mnt/nfs/vlm-aryan/pika-flex-qad/models/pika-flex-ref2va-merged
+    trainable: false
+    frozen_bf16: true
+
+method:
+  _target_: fastvideo.train.methods.knowledge_distillation.pdd_qad.OmniRefPDDQADMethod
+  video_weight: 1.0
+  audio_weight: 1.0
+  rollout_fraction: 0.25
+  rollout_start_step: 20
+  # Half of the six forwards train per row, as OmniRef trains 4 of 8.
+  rungs_per_row: 3
+  eval_every: 15
+  full_eval_every: 30
+  patience: 2
+  fidelity_tolerance: 0.05
+  require_bitwise_init: true
+  save_best: true
+  select_metric: score
+  eval_save_dir: /mnt/nfs/vlm-aryan/pika-flex-qad/runs/ref2va-qad/samples
+  objective: output+recon
+  recon_lambda: 1.0
+  recon_normalize: step0
+  max_rollout_rows: 10
+  noise_floor_eval: true
+  step0_cache: /mnt/nfs/vlm-aryan/pika-flex-qad/runs/ref2va-qad/step0_cache.json
+
+training:
+  distributed:
+    num_gpus: 16
+    sp_size: 4
+    tp_size: 1
+    hsdp_replicate_dim: 1
+    hsdp_shard_dim: 16
+    pin_cpu_memory: false
+  data:
+    train_batch_size: 1
+    seed: 1234
+  optimizer:
+    learning_rate: 5.0e-06
+    betas: [0.9, 0.999]
+    eps: 1.0e-08
+    weight_decay: 0.0
+    lr_scheduler: cosine_with_min_lr
+    lr_warmup_steps: 10
+    min_lr_ratio: 0.3
+  loop:
+    max_train_steps: 200
+    gradient_accumulation_steps: 1
+  checkpoint:
+    output_dir: /mnt/nfs/vlm-aryan/pika-flex-qad/runs/ref2va-qad
+    training_state_checkpointing_steps: 25
+    checkpoints_total_limit: 2
+    resume_from_checkpoint: ''
+  tracker:
+    trackers: [wandb]
+    project_name: pika-h3-flex-nvfp4
+    run_name: pika-flex-ref2va-nvfp4-qad
+  model:
+    enable_gradient_checkpointing_type: full
+  dit_precision: fp32
+  vsa_sparsity: 0.0
+
+callbacks:
+  grad_clip:
+    _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
+    max_grad_norm: 1.0
+
+pipeline: {}
@@ -0,0 +1,126 @@
+# Pika H3 Flex Ref2VA NVFP4 QAD (the FastH3 QAD recipe on Flex's six-forward ladder).
+#
+# Teacher: Pika H3 Flex Ref2VA in bf16, i.e. MiniMax-H3 `transformer_ref` with the
+# released `ref2va` adapter merged by
+# scripts/distill/pika_h3_flex_qad/merge_pika_adapter.py. Student: the same
+# weights with NVFP4 W4A4 q/k/v/out plus FFN fc_in/fc_out in all 50 blocks
+# active from step 0, static FFN activation scales from the Flex calibration
+# table. Attention stays BF16 dense: Flex has no VSA gate, so there is no FP4
+# sparse-attention stage and no gate to freeze. Norms, AdaLN, the token refiner
+# and the FP32 boundary modules stay frozen.
+#
+# The objective, evaluation, guards and checkpoint contract are the shared
+# recipe's (docs/training/minimax_h3_fp4_qad.md). Rows are OmniRef Ref2VA
+# parquet rows; the teacher computes every target on the fly, so the OmniRef
+# held-out eval manifest is reused as is.
+#
+# Paths are placeholders: the merged model, the calibration table (pin its
+# sha256 once calibrated), the row manifests and the output dir.
+models:
+  student:
+    _target_: fastvideo.train.models.minimax_h3.flex.MiniMaxH3FlexRef2VAModel
+    init_from: /mnt/nfs/vlm-aryan/pika-flex-qad/models/pika-flex-ref2va-merged
+    trainable: true
+    enable_gradient_checkpointing_type: full
+    qad:
+      enabled: true
+      linears:
+        enabled: true
+        amax_json: null
+        provisional_calibration_rows: 2
+        amax_sha256: null
+        # Attention linears use the unit activation scale; only the FFN
+        # linears read their calibrated amax from this table.
+        attention: true
+        ffn: true
+        skip_blocks: []
+      fp4_attention:
+        enabled: false
+    data:
+      manifests:
+      - /mnt/lustre/vlm-wlsaidhi/fastvideo/data/omniref-v4-processed/20260919T001000Z/encoded/full/MANIFEST.jsonl
+      cases:
+      - first_frame
+      - full_scene_audio_image_reference
+      per_group:
+        480p: 400
+        768p: 50
+      seed: 20261007
+      max_frames: 124
+      max_train_rows: 4
+      eval_manifest: /mnt/nfs/vlm-aryan/h3-nvfp4-decoder/gen-latents/omniref-eval/manifest.json
+      eval_resolutions:
+      - 480p
+      eval_per_group: 1
+      eval_max_frames: 0
+  teacher:
+    _target_: fastvideo.train.models.minimax_h3.flex.MiniMaxH3FlexRef2VAModel
+    init_from: /mnt/nfs/vlm-aryan/pika-flex-qad/models/pika-flex-ref2va-merged
+    trainable: false
+    frozen_bf16: true
+
+method:
+  _target_: fastvideo.train.methods.knowledge_distillation.pdd_qad.OmniRefPDDQADMethod
+  video_weight: 1.0
+  audio_weight: 1.0
+  rollout_fraction: 0.25
+  rollout_start_step: 20
+  # Half of the six forwards train per row, as OmniRef trains 4 of 8.
+  rungs_per_row: 3
+  eval_every: 2
+  full_eval_every: 2
+  patience: 2
+  fidelity_tolerance: 0.05
+  require_bitwise_init: true
+  save_best: true
+  select_metric: score
+  eval_save_dir: /mnt/nfs/vlm-aryan/pika-flex-qad/runs/smoke-ref2va/samples
+  objective: output+recon
+  recon_lambda: 1.0
+  recon_normalize: step0
+  max_rollout_rows: 2
+  noise_floor_eval: false
+  step0_cache: /mnt/nfs/vlm-aryan/pika-flex-qad/runs/smoke-ref2va/step0_cache.json
+
+training:
+  distributed:
+    num_gpus: 4
+    sp_size: 4
+    tp_size: 1
+    hsdp_replicate_dim: 1
+    hsdp_shard_dim: 4
+    pin_cpu_memory: false
+  data:
+    train_batch_size: 1
+    seed: 1234
+  optimizer:
+    learning_rate: 5.0e-06
+    betas: [0.9, 0.999]
+    eps: 1.0e-08
+    weight_decay: 0.0
+    lr_scheduler: cosine_with_min_lr
+    lr_warmup_steps: 10
+    min_lr_ratio: 0.3
+  loop:
+    max_train_steps: 2
+    gradient_accumulation_steps: 1
+  checkpoint:
+    output_dir: /mnt/nfs/vlm-aryan/pika-flex-qad/runs/smoke-ref2va
+    training_state_checkpointing_steps: 25
+    checkpoints_total_limit: 2
+    resume_from_checkpoint: ''
+  tracker:
+    trackers: []
+    project_name: pika-h3-flex-nvfp4
+    run_name: pika-flex-ref2va-nvfp4-qad
+  model:
+    enable_gradient_checkpointing_type: full
+  dit_precision: fp32
+  vsa_sparsity: 0.0
+
+callbacks:
+  grad_clip:
+    _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
+    max_grad_norm: 1.0
+
+pipeline: {}
@@ -0,0 +1,4 @@
+#!/bin/bash
+# 1-node QAD smoke: usage smoke.sh <config> <out>   (runs under env.sh)
+CFG=$1; OUT=$2; shift 2; rm -rf $OUT; mkdir -p $OUT
+FASTVIDEO_PG_TIMEOUT_MIN=120 $PY -m torch.distributed.run --nproc_per_node=4 --master_port=29571 --log-dir $OUT/torchlogs --redirects 3 --tee 3 -m fastvideo.train.entrypoint.train --config $CFG --training.checkpoint.output_dir $OUT --method.eval_save_dir $OUT/samples --method.step0_cache $OUT/step0.json "$@"
@@ -0,0 +1,9 @@
+#!/bin/bash
+# GPU smoke of the Theseus warm-up: 7 warm-up steps (one per width 1..50) + 2 normal steps, provisional scales.
+R=/mnt/nfs/vlm-aryan/pika-flex-qad; T=$1
+nohup srun --overlap --jobid=$T --nodes=1 --ntasks=1 --kill-on-bad-exit=1 --container-image=nvcr.io/nvidia/pytorch:25.06-py3 \
+  --container-mounts=/mnt/nfs/vlm-aryan:/mnt/nfs/vlm-aryan,/mnt/lustre:/mnt/lustre:ro \
+  bash $R/env.sh bash $R/jobs/smoke.sh $R/smoke-ref2va.yaml $R/runs/theseus-smoke \
+  --method.theseus_steps 7 --training.loop.max_train_steps 9 --method.eval_every 9 --method.full_eval_every 9 \
+  --models.student.data.max_train_rows 9 > $R/logs/theseus-smoke.log 2>&1 &
+echo "theseus smoke on tray $T (pid $!)"
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Bounded Theseus warm-up trial on tray 13510 once it holds a node (login node, tmux). Two arms, 30 min each, sequential.
+R=/mnt/nfs/vlm-aryan/pika-flex-qad; T=13510; S=$R/logs/theseus-trial-summary.txt
+say() { echo "$(date -u +%FT%TZ) $*" | tee -a $S; }
+say "waiting for tray $T"
+for i in $(seq 1 600); do [ "$(squeue -h -j $T -o %T)" = RUNNING ] && break; sleep 60; done
+[ "$(squeue -h -j $T -o %T)" = RUNNING ] || { say "tray $T never started; giving up"; exit 0; }
+say "tray $T running on $(squeue -h -j $T -o %N)"
+COMMON="--models.student.qad.linears.amax_json null --models.student.qad.linears.provisional_calibration_rows 2 \
+ --models.student.data.max_frames 124 --models.student.data.max_train_rows 8 --models.student.data.eval_resolutions [480p] \
+ --models.student.data.eval_per_group 1 --training.distributed.num_gpus 4 --training.distributed.hsdp_shard_dim 4 \
+ --training.loop.max_train_steps 8 --training.optimizer.learning_rate 2e-5 --training.optimizer.lr_warmup_steps 0 \
+ --training.optimizer.lr_scheduler constant --method.eval_every 8 --method.full_eval_every 8 --method.patience 99 \
+ --method.noise_floor_eval false --method.rollout_fraction 0 --training.tracker.trackers []"
+arm() {  # arm <name> [extra overrides]
+  local name=$1; shift
+  timeout 1800 srun --overlap --jobid=$T --nodes=1 --ntasks=1 --kill-on-bad-exit=1 \
+    --container-image=nvcr.io/nvidia/pytorch:25.06-py3 --container-mounts=/mnt/nfs/vlm-aryan:/mnt/nfs/vlm-aryan,/mnt/lustre:/mnt/lustre:ro \
+    bash $R/env.sh bash $R/jobs/smoke.sh $R/ref2va-qad.yaml $R/runs/trial-$name $COMMON "$@" > $R/logs/trial-$name.log 2>&1
+  say "$name exit $?: $(grep -o "QAD eval @[0-9]*: score [0-9.]*" $R/logs/trial-$name.log | tr "\n" " ") $(grep -c Traceback $R/logs/trial-$name.log) tracebacks"
+  grep -o "Theseus warm-up step [0-9]* (width [0-9]*)[^:]*: rel [0-9.]*" $R/logs/trial-$name.log | sed "s/^/  /" >> $S
+  rm -rf $R/runs/trial-$name/checkpoint-* $R/runs/trial-$name/best
+}
+arm baseline
+arm theseus --method.theseus_steps 4 --method.theseus_widths [1,4,16,50] --method.theseus_gamma 1.0
+say "TRIAL_DONE"