Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
83ad0f4d6c | ||
|
|
3b7e51a886 | ||
|
|
f91958c40f | ||
|
|
e69a034046 |
@@ -56,6 +56,11 @@ def test_performance_lane_classifies_pull_requests_before_main():
|
||||
"BUILDKITE_PULL_REQUEST") < function_strings.index("BUILDKITE_BRANCH")
|
||||
|
||||
|
||||
def test_vsa_training_lane_uses_strict_h100_pair():
|
||||
function_strings = _function_strings(PR_TEST, "run_training_tests_VSA")
|
||||
assert "H100!:2" in function_strings.splitlines()
|
||||
|
||||
|
||||
def test_pr_model_load_and_training_lanes_disable_fa4():
|
||||
lanes = {
|
||||
"run_transformer_tests": "pytest ./fastvideo/tests/transformers -vs",
|
||||
|
||||
@@ -229,7 +229,7 @@ def run_training_lora_tests():
|
||||
)
|
||||
|
||||
|
||||
@app.function(gpu="H100:2",
|
||||
@app.function(gpu="H100!:2",
|
||||
image=image,
|
||||
timeout=900,
|
||||
secrets=[wandb_secret, ci_env_secret])
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"step_time": 0.8274769484996796,
|
||||
"avg_step_time": 1.4095150619745254,
|
||||
"timing/optimizer_step": 0.02941332757472992,
|
||||
"grad_norm": 0.24477368593215942,
|
||||
"timing/forward_backward": 0.693624597042799,
|
||||
"_timestamp": 1765590605.595408,
|
||||
"timing/prepare_dit_inputs": 0.009604107588529587,
|
||||
"learning_rate": 1e-05,
|
||||
"train_loss": 0.30774877965450287,
|
||||
"_step": 5,
|
||||
"timing/normalize_input": 0.0003825277090072632,
|
||||
"vsa_sparsity": 0.05,
|
||||
"timing/clip_grad_norm": 0.025074530392885208,
|
||||
"_runtime": 90.44591733,
|
||||
"timing/get_next_batch": 0.0055067278444767,
|
||||
"timing/reduce_loss": 0.00021916255354881287
|
||||
}
|
||||
@@ -18,8 +18,7 @@ from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs
|
||||
from fastvideo.utils import FlexibleArgumentParser
|
||||
|
||||
wandb_name = "test_training_loss_VSA"
|
||||
h100_reference_wandb_summary_file = "fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json"
|
||||
h200_reference_wandb_summary_file = "fastvideo/tests/training/VSA/h200_reference_wandb_summary_VSA.json"
|
||||
reference_wandb_summary_file = "fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json"
|
||||
|
||||
NUM_NODES = "1"
|
||||
NUM_GPUS_PER_NODE = "2"
|
||||
@@ -110,12 +109,9 @@ def test_distributed_training():
|
||||
summary_file = 'data/wan_finetune_test_VSA/tracker/wandb/latest-run/files/wandb-summary.json'
|
||||
|
||||
device_name = torch.cuda.get_device_name()
|
||||
if "H100" in device_name:
|
||||
reference_wandb_summary_file = h100_reference_wandb_summary_file
|
||||
elif "H200" in device_name:
|
||||
reference_wandb_summary_file = h200_reference_wandb_summary_file
|
||||
else:
|
||||
raise ValueError(f"Unknown device: {device_name}")
|
||||
print(f"INFO: device: {device_name}")
|
||||
if "H100" not in device_name:
|
||||
raise ValueError(f"VSA training regression requires H100 GPUs, got: {device_name}")
|
||||
|
||||
reference_wandb_summary = json.load(open(reference_wandb_summary_file))
|
||||
wandb_summary = json.load(open(summary_file))
|
||||
|
||||
Reference in New Issue
Block a user