230 lines
7.7 KiB
Python
230 lines
7.7 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
from fastvideo.tests.performance import compare_baseline
|
|
from fastvideo.performance.metric_policy import resolve_metric_policies
|
|
|
|
|
|
def _raw_result():
|
|
return {
|
|
"benchmark_id": "wan-t2v-1.3b-2gpu",
|
|
"device": "NVIDIA L40S",
|
|
"avg_generation_time_s": 10.0,
|
|
"throughput_fps": 4.5,
|
|
"max_peak_memory_mb": 10000.0,
|
|
"commit": "a" * 40,
|
|
"timestamp": "2026-06-16T00:00:00+00:00",
|
|
"pr_number": "123",
|
|
}
|
|
|
|
|
|
def test_detect_run_source_prefers_explicit_env(monkeypatch):
|
|
monkeypatch.setenv("PERF_RUN_SOURCE", "local")
|
|
monkeypatch.setenv("BUILDKITE_PULL_REQUEST", "123")
|
|
|
|
assert compare_baseline._detect_run_source() == "local"
|
|
|
|
|
|
def test_detect_run_source_infers_pr(monkeypatch):
|
|
monkeypatch.delenv("PERF_RUN_SOURCE", raising=False)
|
|
monkeypatch.setenv("BUILDKITE_PULL_REQUEST", "123")
|
|
|
|
assert compare_baseline._detect_run_source() == "pr"
|
|
|
|
|
|
def test_detect_run_source_infers_scheduled_main(monkeypatch):
|
|
monkeypatch.delenv("PERF_RUN_SOURCE", raising=False)
|
|
monkeypatch.setenv("BUILDKITE_PULL_REQUEST", "false")
|
|
monkeypatch.setenv("BUILDKITE_BRANCH", "main")
|
|
monkeypatch.setenv("TEST_SCOPE", "full")
|
|
|
|
assert compare_baseline._detect_run_source() == "scheduled_main"
|
|
|
|
|
|
def test_upload_policy_pass_requires_success(monkeypatch):
|
|
monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "pass")
|
|
|
|
assert compare_baseline._upload_allowed({"success": True}) is True
|
|
assert compare_baseline._upload_allowed({"success": False}) is False
|
|
|
|
|
|
def test_upload_policy_always_uploads_failures(monkeypatch):
|
|
monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "always")
|
|
|
|
assert compare_baseline._upload_allowed({"success": False}) is True
|
|
|
|
|
|
def test_normalized_record_includes_source_metadata(monkeypatch):
|
|
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
|
|
monkeypatch.setenv("BUILDKITE_BRANCH", "feature/perf")
|
|
monkeypatch.setenv("TEST_SCOPE", "direct")
|
|
monkeypatch.setenv("BUILDKITE_BUILD_URL", "https://buildkite.example/build")
|
|
monkeypatch.setenv("BUILDKITE_BUILD_ID", "build-1")
|
|
monkeypatch.setenv("BUILDKITE_JOB_ID", "job-1")
|
|
|
|
record = compare_baseline.normalize_performance_result(_raw_result())
|
|
|
|
assert record["run_source"] == "pr"
|
|
assert record["baseline_eligible"] is False
|
|
assert record["branch"] == "feature/perf"
|
|
assert record["pr_number"] == "123"
|
|
assert record["test_scope"] == "direct"
|
|
assert record["build_url"] == "https://buildkite.example/build"
|
|
assert record["build_id"] == "build-1"
|
|
assert record["job_id"] == "job-1"
|
|
|
|
|
|
def test_normalized_record_includes_effective_regression_thresholds(monkeypatch):
|
|
monkeypatch.setenv("PERF_RUN_SOURCE", "pr")
|
|
raw = _raw_result()
|
|
raw["regression_thresholds"] = {
|
|
"latency": {
|
|
"threshold_percent": 0.09,
|
|
"threshold_absolute": 0.75,
|
|
"gated": True,
|
|
},
|
|
"throughput": {
|
|
"gated": False,
|
|
},
|
|
}
|
|
|
|
record = compare_baseline.normalize_performance_result(raw)
|
|
|
|
assert record["regression_thresholds"]["latency"] == {
|
|
"threshold_percent": 0.09,
|
|
"threshold_absolute": 0.75,
|
|
"gated": True,
|
|
}
|
|
assert record["regression_thresholds"]["throughput"]["gated"] is False
|
|
|
|
|
|
def test_invalid_regression_threshold_container_uses_defaults():
|
|
policies = resolve_metric_policies(["not", "a", "mapping"])
|
|
|
|
latency = next(policy for policy in policies if policy.key == "latency")
|
|
assert latency.threshold_percent == 0.08
|
|
assert latency.threshold_absolute == 0.5
|
|
assert latency.gated is True
|
|
|
|
|
|
def test_boolean_regression_threshold_values_are_ignored():
|
|
policies = resolve_metric_policies({
|
|
"latency": {
|
|
"threshold_percent": True,
|
|
"threshold_absolute": False,
|
|
"gated": "false",
|
|
}
|
|
})
|
|
|
|
latency = next(policy for policy in policies if policy.key == "latency")
|
|
assert latency.threshold_percent == 0.08
|
|
assert latency.threshold_absolute == 0.5
|
|
assert latency.gated is False
|
|
|
|
|
|
def test_baseline_eligibility_only_for_successful_scheduled_main():
|
|
assert compare_baseline._is_baseline_eligible("scheduled_main", True) is True
|
|
assert compare_baseline._is_baseline_eligible("scheduled_main", False) is False
|
|
assert compare_baseline._is_baseline_eligible("pr", True) is False
|
|
assert compare_baseline._is_baseline_eligible("local", True) is False
|
|
|
|
|
|
def test_latency_regression_requires_percent_and_absolute_floors():
|
|
baseline = [{"latency": 10.0}]
|
|
current = {"model_id": "wan", "latency": 10.6}
|
|
|
|
percent_only = resolve_metric_policies({
|
|
"latency": {
|
|
"threshold_percent": 0.05,
|
|
"threshold_absolute": 0.75,
|
|
}
|
|
})
|
|
absolute_only = resolve_metric_policies({
|
|
"latency": {
|
|
"threshold_percent": 0.10,
|
|
"threshold_absolute": 0.5,
|
|
}
|
|
})
|
|
both = resolve_metric_policies({
|
|
"latency": {
|
|
"threshold_percent": 0.05,
|
|
"threshold_absolute": 0.5,
|
|
}
|
|
})
|
|
|
|
assert compare_baseline._check_regressions(current, baseline, percent_only) == []
|
|
assert compare_baseline._check_regressions(current, baseline, absolute_only) == []
|
|
|
|
failures = compare_baseline._check_regressions(current, baseline, both)
|
|
assert len(failures) == 1
|
|
assert "latency regressed by 6.0% and 0.600" in failures[0]
|
|
|
|
|
|
def test_throughput_regression_uses_higher_is_better_direction():
|
|
baseline = [{"throughput": 10.0}]
|
|
current = {"model_id": "wan", "throughput": 9.0}
|
|
policies = resolve_metric_policies({
|
|
"throughput": {
|
|
"threshold_percent": 0.05,
|
|
"threshold_absolute": 0.5,
|
|
}
|
|
})
|
|
|
|
failures = compare_baseline._check_regressions(current, baseline, policies)
|
|
|
|
assert len(failures) == 1
|
|
assert "throughput regressed by 10.0% and 1.000" in failures[0]
|
|
|
|
|
|
def test_memory_regression_uses_metric_specific_absolute_floor():
|
|
baseline = [{"memory": 10000.0}]
|
|
current = {"model_id": "wan", "memory": 10600.0}
|
|
policies = resolve_metric_policies({
|
|
"memory": {
|
|
"threshold_percent": 0.05,
|
|
"threshold_absolute": 256.0,
|
|
}
|
|
})
|
|
|
|
failures = compare_baseline._check_regressions(current, baseline, policies)
|
|
|
|
assert len(failures) == 1
|
|
assert "memory regressed by 6.0% and 600.0" in failures[0]
|
|
|
|
|
|
def test_component_metric_can_gate_independently():
|
|
baseline = [{"dit_time_s": 8.0}]
|
|
current = {"model_id": "wan", "dit_time_s": 8.6}
|
|
policies = resolve_metric_policies({
|
|
"dit_time_s": {
|
|
"threshold_percent": 0.05,
|
|
"threshold_absolute": 0.25,
|
|
}
|
|
})
|
|
|
|
failures = compare_baseline._check_regressions(current, baseline, policies)
|
|
|
|
assert len(failures) == 1
|
|
assert "dit_time_s regressed by 7.5% and 0.600" in failures[0]
|
|
|
|
|
|
def test_informational_metric_remains_visible_without_failing():
|
|
baseline = [{"throughput": 10.0}]
|
|
current = {"model_id": "wan", "gpu_type": "NVIDIA L40S", "throughput": 8.0}
|
|
policies = resolve_metric_policies({
|
|
"throughput": {
|
|
"threshold_percent": 0.01,
|
|
"threshold_absolute": 0.01,
|
|
"gated": False,
|
|
}
|
|
})
|
|
|
|
row = compare_baseline._build_summary_row(current, baseline, policies, False)
|
|
|
|
assert compare_baseline._check_regressions(current, baseline, policies) == []
|
|
assert row["metrics"]["throughput"]["regression_pct"] == 20.0
|
|
assert row["metrics"]["throughput"]["gated"] is False
|
|
assert row["metrics"]["throughput"]["threshold_exceeded"] is True
|
|
assert row["metrics"]["throughput"]["regressed"] is False
|
|
assert row["threshold_exceeded_metrics"] == ["throughput"]
|
|
assert row["failing_metrics"] == []
|