trackwan: 14B teacher pipeline, layer-wise track LR, and overfit tooling
Training:
- finetune.py: optional track/base param groups (WANTRACK_TRACK_GROUP) so the
track pathway can take a different LR (WANTRACK_TRACK_LR_MULT); patch_embedding
gradient masked to the track slot [:, 36:]. Raises if nothing matches.
- track_warmup.py callback: hold the pretrained DiT at lr 0 while the track
pathway warms up, then ramp it back in.
14B configs + held-alloc launchers:
- finetune_wantrack_{synth_sparse_fixed,synth_sparse_random,openvid_sparse_merged_bias,
synth_stage2}_14b + openvid_14b_{warmup,lwlr} + 1.3B stage2 configs.
- run_wan14b_held.sh (held alloc + auto-restart + rank cleanup, per-stage WANTRACK
env), acquire_nodes.sh, chain_stepAB.sh, run_stepB_seed.sh, run_stepC_merge.sh,
watch_gate.sh, run_synth_stage2_slurm.sh.
Init / diagnostics:
- convert_trackwan_init_v2.py: --pe-src to lift patch_embedding[:, 36:] and
--track-src to lift track_encoder together (co-adapted merge); sharded-source support.
- check_gate_growth.py, gradient_flow_analysis.py, gradient_flow_summarize.py.
Data pipeline: segment_tracks.py sharding; decode/segment/tracks slurm scripts;
precompute_null_text, two_track_val, cfg_ablation, lwlr_experiment.
Inference: app_multi_trace.py updates.
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Read /mnt/lustre/vlm-s4duan/gradient_analysis/summary.json and print a compact table
|
||||
plus a "magnitude ratio" analysis (track/base) at bs=N averaging.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
path = Path(sys.argv[1] if len(sys.argv) > 1 else "/mnt/lustre/vlm-s4duan/gradient_analysis/summary.json")
|
||||
with open(path) as f:
|
||||
stats = json.load(f)
|
||||
|
||||
layers = [
|
||||
"patch_embedding.proj.weight[:,:36]",
|
||||
"patch_embedding.proj.weight[:,36:]",
|
||||
"track_encoder.temporal_conv.weight",
|
||||
"track_encoder.proj.weight",
|
||||
"blocks.0.to_q.weight",
|
||||
"blocks.14.ffn.fc_in.weight",
|
||||
"blocks.29.to_out.weight",
|
||||
]
|
||||
|
||||
print(f"{'model':<14} {'layer':<44} {'per_sample':>12} {'mean_grad':>12} {'snr':>8}")
|
||||
print("-" * 96)
|
||||
for model, s in stats.items():
|
||||
for lay in layers:
|
||||
if lay not in s:
|
||||
continue
|
||||
v = s[lay]
|
||||
print(f"{model:<14} {lay:<44} {v['avg_per_sample_norm']:>12.4e} "
|
||||
f"{v['mean_grad_norm']:>12.4e} {v['snr']:>8.4f}")
|
||||
print("-" * 96)
|
||||
|
||||
# Magnitude ratio: how big is bs=N averaged grad on the track slice vs base slice?
|
||||
print("\nMagnitude ratio (mean_grad_norm(track slice) / mean_grad_norm(base slice))")
|
||||
print(f"{'model':<14} {'track_slice / base_slice (patch_embed)':>44}")
|
||||
for model, s in stats.items():
|
||||
try:
|
||||
base = s["patch_embedding.proj.weight[:,:36]"]["mean_grad_norm"]
|
||||
trk = s["patch_embedding.proj.weight[:,36:]"]["mean_grad_norm"]
|
||||
ratio = trk / max(base, 1e-30)
|
||||
except KeyError:
|
||||
ratio = float("nan")
|
||||
print(f"{model:<14} {ratio:>44.6f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user