shaoxiongduan
ac919b1c87
- data_pipeline: SAM+CoTracker preprocessing, sharded Slurm launchers, OpenVid download/filter, and a data-parallel Wan2.2-T2V-A14B synthetic video generator with the first-frame over-brightness fix (over-generate 129 -> drop first 8 -> keep 121). - examples/train: OpenVid bidir/stage2/stage3 launchers + finetune YAMLs (sparse, merged, merged_bias, stage2v2 hi-lr, stage3A/B/C). - research_log: first-frame saturation root-cause writeup (DiT position-0 latent, not VAE decode). - fastvideo: trackwan model / track-encoder / preprocessing updates, dcp_to_diffusers export, FA4.
27 lines
1.5 KiB
Bash
27 lines
1.5 KiB
Bash
#!/usr/bin/env bash
|
|
# Overlap tracking with the slow self-healing download: idempotent multi-node passes
|
|
# over the growing videos/ dir until download done (98 parts) AND ~all tracked.
|
|
# MUST run inside srun on a compute node (login-node `sleep` gets killed by the harness).
|
|
# Waits for any in-flight tracking pass first, so it's safe to relaunch (no double-submit).
|
|
set +e
|
|
WORK=/mnt/lustre/vlm-s4duan
|
|
cd "$WORK/FastVideo"
|
|
DC(){ ls "$WORK"/openvid_1m/_extracted/*.done 2>/dev/null | wc -l; }
|
|
RUNNING(){ squeue -u vlm-s4duan -h -n cotracker_dp -o %i 2>/dev/null; }
|
|
NODES=${NODES:-8}
|
|
while true; do
|
|
while [ -n "$(RUNNING)" ]; do sleep 90; done # let in-flight pass finish
|
|
ls "$WORK"/openvid_1m/videos/*.mp4 > "$WORK"/openvid_1m/videos_all.txt 2>/dev/null
|
|
nv=$(wc -l < "$WORK"/openvid_1m/videos_all.txt 2>/dev/null); nv=${nv:-0}
|
|
nt=$(ls "$WORK"/openvid_1m/tracks/*.npz 2>/dev/null | wc -l)
|
|
echo "[$(date +%H:%M)] parts=$(DC)/98 videos=$nv tracks=$nt"
|
|
if [ "$(DC)" -ge 98 ] && [ "$nt" -ge "$((nv - nv/40 - 200))" ]; then
|
|
echo "TRACK_LOOP_DONE videos=$nv tracks=$nt"; break; fi
|
|
if [ "$nv" -le "$((nt + 300))" ]; then echo " little new; wait 300s"; sleep 300; continue; fi
|
|
jid=$(CLIPS_DIR="$WORK"/openvid_1m/clips VIDEO_LIST="$WORK"/openvid_1m/videos_all.txt \
|
|
OUT_DIR="$WORK"/openvid_1m/tracks NODES=$NODES PROCS_PER_GPU=2 FPS=24 NUM_FRAMES=121 GRID=50 \
|
|
bash data_pipeline/run_tracks_slurm.sh 2>&1 | grep -oE "[0-9]+$" | tail -1)
|
|
echo " submitted track job $jid over $nv videos"
|
|
[ -z "$jid" ] && sleep 180
|
|
done
|