Compare commits

...
4 changed files with 10 additions and 27 deletions
@@ -56,6 +56,11 @@ def test_performance_lane_classifies_pull_requests_before_main():
"BUILDKITE_PULL_REQUEST") < function_strings.index("BUILDKITE_BRANCH")
def test_vsa_training_lane_uses_strict_h100_pair():
function_strings = _function_strings(PR_TEST, "run_training_tests_VSA")
assert "H100!:2" in function_strings.splitlines()
def test_pr_model_load_and_training_lanes_disable_fa4():
lanes = {
"run_transformer_tests": "pytest ./fastvideo/tests/transformers -vs",
+1 -1
View File
@@ -229,7 +229,7 @@ def run_training_lora_tests():
)
@app.function(gpu="H100:2",
@app.function(gpu="H100!:2",
image=image,
timeout=900,
secrets=[wandb_secret, ci_env_secret])
@@ -1,18 +0,0 @@
{
"step_time": 0.8274769484996796,
"avg_step_time": 1.4095150619745254,
"timing/optimizer_step": 0.02941332757472992,
"grad_norm": 0.24477368593215942,
"timing/forward_backward": 0.693624597042799,
"_timestamp": 1765590605.595408,
"timing/prepare_dit_inputs": 0.009604107588529587,
"learning_rate": 1e-05,
"train_loss": 0.30774877965450287,
"_step": 5,
"timing/normalize_input": 0.0003825277090072632,
"vsa_sparsity": 0.05,
"timing/clip_grad_norm": 0.025074530392885208,
"_runtime": 90.44591733,
"timing/get_next_batch": 0.0055067278444767,
"timing/reduce_loss": 0.00021916255354881287
}
@@ -18,8 +18,7 @@ from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs
from fastvideo.utils import FlexibleArgumentParser
wandb_name = "test_training_loss_VSA"
h100_reference_wandb_summary_file = "fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json"
h200_reference_wandb_summary_file = "fastvideo/tests/training/VSA/h200_reference_wandb_summary_VSA.json"
reference_wandb_summary_file = "fastvideo/tests/training/VSA/h100_reference_wandb_summary_VSA.json"
NUM_NODES = "1"
NUM_GPUS_PER_NODE = "2"
@@ -110,12 +109,9 @@ def test_distributed_training():
summary_file = 'data/wan_finetune_test_VSA/tracker/wandb/latest-run/files/wandb-summary.json'
device_name = torch.cuda.get_device_name()
if "H100" in device_name:
reference_wandb_summary_file = h100_reference_wandb_summary_file
elif "H200" in device_name:
reference_wandb_summary_file = h200_reference_wandb_summary_file
else:
raise ValueError(f"Unknown device: {device_name}")
print(f"INFO: device: {device_name}")
if "H100" not in device_name:
raise ValueError(f"VSA training regression requires H100 GPUs, got: {device_name}")
reference_wandb_summary = json.load(open(reference_wandb_summary_file))
wandb_summary = json.load(open(summary_file))