From da9cbe3da956d9415efc62672efd3f139d072b73 Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Tue, 7 Jul 2026 04:54:40 -0700 Subject: [PATCH] [ci]: skip perf gating on packed hosts via host CPU profile guard The wan-t2v perf lane intermittently runs ~1.7x slow on packed Modal hosts (TE pinned at 3.595s vs the 2.03s healthy cluster, DiT +58-80%, GPU clocks at full boost): host CPU contention that cpu=8/16 reservations cannot isolate. 9 of the last ~11 full suites redded only on this lane. A perf gate must never fail a PR on a measurement taken on a degraded host (review r30, option a). The harness now runs a ~1.5s host CPU probe (single-thread pure-Python loop, gated axis; 8-thread torch matmul, informational) once per pytest process and stores host_cpu_score on every record. The comparator emits HOST_BELOW_PROFILE when the score is under the calibrated floor (PERF_HOST_CPU_MIN_SCORE, default 0.75): gating is skipped loudly, CI passes, and the record can never seed a baseline. Records without a score gate normally. Calibrated against five fresh perf-lane containers (L40S:2 cpu=8): healthy hosts score 0.97-1.17, packed-host projections 0.54-0.73; sampled data and recalibration procedure documented in host_probe.py. Stacked on maint/perf-comparator-statuses (f46c9e61). --- docs/contributing/performance_benchmarks.md | 9 ++ .../tests/performance/compare_baseline.py | 47 +++++++++ fastvideo/tests/performance/host_probe.py | 96 +++++++++++++++++++ .../test_compare_baseline_policy.py | 51 ++++++++++ .../tests/performance/test_host_probe.py | 18 ++++ .../performance/test_inference_performance.py | 23 +++++ 6 files changed, 244 insertions(+) create mode 100644 fastvideo/tests/performance/host_probe.py create mode 100644 fastvideo/tests/performance/test_host_probe.py diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index 7e72bb327..41de0ce67 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -247,6 +247,7 @@ record and in the Markdown summary: | `CALIBRATION_NEEDED` | No comparable baseline exists. Gating is inactive and the record does **not** seed a baseline — seed new cohorts explicitly via the reseed workflow. The record also carries `baseline_status: "initialized_new_cohort"`. | passes | | `RECIPE_MISMATCH` | Baseline history exists for the same variant/hardware/software cohort but under a different `recipe_fingerprint`. Represent recipe changes as a new `variant_id`, or reseed. | fails | | `INFRA_ERROR` | The comparison itself failed (for example, baseline history could not be loaded). | fails | +| `HOST_BELOW_PROFILE` | The harness's ~2s host CPU probe (`host_probe.py`) scored below the healthy-host floor (`PERF_HOST_CPU_MIN_SCORE`, default 0.75): the shared runner was packed, so the measurements are not comparable. Gating is skipped and the record never seeds a baseline. | passes | | `QUALITY_BLOCKED` | Reserved for the promoted-baseline workflow; never emitted by the comparator. | n/a | Metric-specific threshold policies and promoted baselines remain separate @@ -532,6 +533,14 @@ The run passes but does not seed a baseline; seed it explicitly with the `variant_id`. Either bump `variant_id` to open a new cohort, or reseed the baseline if the existing variant should adopt the new recipe. +**`HOST_BELOW_PROFILE: host below profile — measurements not comparable`** — +the shared Modal host was packed during the run (host CPU contention; GPU +telemetry typically still looks healthy). Not a property of the PR: the gate +is skipped and CI passes. Retry the lane if you need a comparable measurement. +If every host suddenly scores low after an image/python/torch bump, the probe +reference is stale — recalibrate `PERF_HOST_CPU_ST_REF_KOPS` (see +`host_probe.py`, which documents the calibration procedure and sampled data). + **Persistent failure right after a torch / kernel / image upgrade** — genuine regression *or* baseline drift. Compare the failing normalized record with recent successful records in the HF dataset. If the shift is expected and diff --git a/fastvideo/tests/performance/compare_baseline.py b/fastvideo/tests/performance/compare_baseline.py index 002c77397..3839f46c9 100644 --- a/fastvideo/tests/performance/compare_baseline.py +++ b/fastvideo/tests/performance/compare_baseline.py @@ -95,9 +95,29 @@ STATUS_REGRESSION = "REGRESSION" STATUS_CALIBRATION_NEEDED = "CALIBRATION_NEEDED" STATUS_RECIPE_MISMATCH = "RECIPE_MISMATCH" STATUS_INFRA_ERROR = "INFRA_ERROR" +STATUS_HOST_BELOW_PROFILE = "HOST_BELOW_PROFILE" # Reserved for the promoted-baseline workflow; never emitted by this comparator. STATUS_QUALITY_BLOCKED = "QUALITY_BLOCKED" +# Floor for host_cpu_score (host_probe.py: ~1.0 on a healthy perf-lane host). +# Calibration basis (see host_probe.py for the sampled data): healthy lane +# hosts scored 0.97-1.17; the packed-host signature (review r30; TE 3.595s vs +# the 2.03s healthy cluster, DiT +58-80%) inflates CPU-bound wall time by +# 1.58-1.77x, i.e. a packed host projects to 0.54-0.73. 0.75 sits above every +# packed projection and below every healthy sample. +DEFAULT_HOST_CPU_MIN_SCORE = 0.75 + + +def _host_cpu_min_score() -> float: + raw = os.environ.get("PERF_HOST_CPU_MIN_SCORE", "").strip() + if raw: + try: + return float(raw) + except ValueError: + print(f"Invalid PERF_HOST_CPU_MIN_SCORE={raw!r}; " + f"using {DEFAULT_HOST_CPU_MIN_SCORE}") + return DEFAULT_HOST_CPU_MIN_SCORE + def _should_persist_tracking() -> bool: return _normalized_upload_policy() != "never" @@ -268,6 +288,13 @@ def normalize_performance_result(result: dict[str, Any]) -> dict[str, Any]: "success": True, **_record_metadata(_detect_run_source(), result), } + # Host CPU probe fields (host_probe.py); absent on records measured + # before the probe existed or when the probe failed. + for key in ("host_cpu_score", "host_cpu_single_thread_kops", + "host_cpu_multi_thread_gflops"): + value = safe_float(result.get(key)) + if value is not None: + record[key] = value record.update(_identity_metadata(result)) return record @@ -386,6 +413,26 @@ def _compare_record( for cohort bookkeeping) and returns ``(failures, baseline_records)``. """ model = record.get("model_id", "unknown") + + # Host-profile guard (r30 option a): a measurement taken on a packed host + # is not comparable to anything, so skip gating loudly instead of failing + # the PR. The record keeps its non-PASS status, so it can never advance + # the rolling baseline. Records without a score gate normally (fail-open). + host_cpu_score = safe_float(record.get("host_cpu_score")) + min_score = _host_cpu_min_score() + if host_cpu_score is not None and host_cpu_score < min_score: + record["comparator_status"] = STATUS_HOST_BELOW_PROFILE + print("=" * 72) + print(f"HOST_BELOW_PROFILE: host below profile — measurements not " + f"comparable (host_cpu_score={host_cpu_score:.3f} < floor " + f"{min_score:.2f}) for {model}.") + print("Regression gating is SKIPPED for this record and it will NOT " + "seed a baseline. This is host CPU contention on the shared " + "runner, not a property of the PR; retry the lane to land on a " + "healthy host.") + print("=" * 72) + return [], [] + try: identity_filters = _comparison_identity_filters(record) baseline_records = load_records_for_model( diff --git a/fastvideo/tests/performance/host_probe.py b/fastvideo/tests/performance/host_probe.py new file mode 100644 index 000000000..cdd1f1870 --- /dev/null +++ b/fastvideo/tests/performance/host_probe.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Host CPU health probe for the performance lane. + +The perf lane runs on shared Modal hosts. A packed host slows CPU-bound +pipeline work by ~1.6-1.8x (observed: text encoder pinned at 3.595s vs the +2.03s healthy cluster, DiT +58-80%) while GPU telemetry stays healthy, which +produces false REGRESSION verdicts against PRs. CPU reservations are a floor, +not isolation, so the comparator instead refuses to gate on measurements +taken on a degraded host (review r30, option a). + +Two fixed-work, wall-clock-timed sub-probes (~1.5s total), run once per +pytest process before the first benchmark: + +- single-thread: a pure-Python arithmetic loop. Captures per-core slowdown + (CPU steal, scheduling delay, frequency/cache pressure). This is the axis + ``host_cpu_score`` gates on: it directly mirrors the packed-host signature + (single-stream stage wall time), and healthy lane hosts cluster tightly. +- multi-thread: torch float32 matmuls pinned to the lane's CPU reservation + (cpu=8 in fastvideo/tests/modal/pr_test.py). Recorded as informational + telemetry only: healthy-pool throughput spans 582-1010 GFLOP/s across host + SKUs (1.74x), wider than the packed-host signal, so it cannot gate. + +Calibration basis (2026-07-07, fastvideo-dev:latest image /opt/venv +python 3.10.19 + torch 2.9.1, five Modal L40S:2 cpu=8 memory=32GiB +containers — the exact perf-lane reservation): four hosts (cpu_count=21) +scored 19.0-20.1k kops/s single-thread, one faster host (cpu_count=24) +scored 22.9k. Reference = 19700 (common-SKU median), so healthy samples +score 0.97-1.17. The packed-host signature (r30) inflates CPU-bound wall +time 1.58-1.77x, i.e. throughput drops to 0.56-0.63x: 0.54-0.61 on the +common SKU, at most ~0.73 on the fast SKU. The comparator's default floor +of 0.75 (PERF_HOST_CPU_MIN_SCORE) sits above every packed projection and +below every healthy sample. Recalibrate the reference after an image / +python / torch bump via PERF_HOST_CPU_ST_REF_KOPS (a stale reference can +only skip the gate loudly, never fail a PR). +""" + +import os +import time + +_DEFAULT_ST_REF_KOPS = 19700.0 + +_PROBE_THREADS = 8 # matches the perf lane's cpu=8.0 Modal reservation +_ST_ITERS = 20_000_000 # ~1.0s single-threaded on a healthy lane host +_MT_N = 2048 +_MT_REPS = 6 + + +def _single_thread_kops(iters: int = _ST_ITERS) -> float: + """Fixed pure-Python workload; returns kilo-iterations per second.""" + start = time.perf_counter() + acc = 0 + for i in range(iters): + acc += i * i + elapsed = time.perf_counter() - start + return iters / elapsed / 1e3 + + +def _multi_thread_gflops(n: int = _MT_N, reps: int = _MT_REPS) -> float: + """Fixed torch float32 matmul workload on the lane's CPU reservation.""" + import torch + old_threads = torch.get_num_threads() + torch.set_num_threads(min(_PROBE_THREADS, os.cpu_count() or _PROBE_THREADS)) + try: + a = torch.rand(n, n, dtype=torch.float32) + b = torch.rand(n, n, dtype=torch.float32) + a @ b # warmup: thread-pool spin-up and first-touch page faults + start = time.perf_counter() + for _ in range(reps): + a @ b + elapsed = time.perf_counter() - start + finally: + torch.set_num_threads(old_threads) + return reps * 2 * n**3 / elapsed / 1e9 + + +def measure_host_cpu_profile() -> dict[str, float]: + """Run both sub-probes and return the normalized host CPU profile. + + ``host_cpu_score`` is ~1.0 on a healthy perf-lane host (single-thread + axis); the raw sub-probe throughputs are kept alongside for auditing + and recalibration. + """ + st_ref = float(os.environ.get("PERF_HOST_CPU_ST_REF_KOPS", _DEFAULT_ST_REF_KOPS)) + st = _single_thread_kops() + mt = _multi_thread_gflops() + return { + "host_cpu_score": round(st / st_ref, 3), + "host_cpu_single_thread_kops": round(st, 1), + "host_cpu_multi_thread_gflops": round(mt, 1), + } + + +if __name__ == "__main__": + # Recalibration helper: run this on a perf-lane container (Modal L40S:2, + # cpu=8) a few times, take healthy-cluster medians, update the refs. + print(measure_host_cpu_profile()) diff --git a/fastvideo/tests/performance/test_compare_baseline_policy.py b/fastvideo/tests/performance/test_compare_baseline_policy.py index 7b1bfeaa1..d7ce49e4e 100644 --- a/fastvideo/tests/performance/test_compare_baseline_policy.py +++ b/fastvideo/tests/performance/test_compare_baseline_policy.py @@ -614,3 +614,54 @@ def test_main_baseline_load_failure_is_infra_error_and_fails_ci(monkeypatch, tmp assert record["success"] is False assert record["baseline_eligible"] is False assert "| INFRA_ERROR |" in markdown + + +def test_main_host_below_profile_skips_gate_and_never_seeds_baseline( + monkeypatch, tmp_path, capsys): + # A 2x latency "regression" measured on a packed host must not fail CI. + exit_code, record, markdown = _run_compare( + monkeypatch, tmp_path, + _v2_raw_result(avg_generation_time_s=20.0, host_cpu_score=0.55), + [_v2_baseline_record()]) + + assert exit_code == 0 + assert record["comparator_status"] == "HOST_BELOW_PROFILE" + assert record["success"] is True + assert record["baseline_eligible"] is False + assert "| HOST_BELOW_PROFILE |" in markdown + assert ("host below profile — measurements not comparable" + in capsys.readouterr().out) + + +def test_main_healthy_host_score_gates_normally(monkeypatch, tmp_path): + exit_code, record, _ = _run_compare( + monkeypatch, tmp_path, + _v2_raw_result(avg_generation_time_s=20.0, host_cpu_score=0.97), + [_v2_baseline_record()]) + + assert exit_code == 1 + assert record["comparator_status"] == "REGRESSION" + + +def test_main_healthy_host_pass_keeps_baseline_eligibility_and_score( + monkeypatch, tmp_path): + exit_code, record, _ = _run_compare( + monkeypatch, tmp_path, _v2_raw_result(host_cpu_score=1.02), + [_v2_baseline_record()]) + + assert exit_code == 0 + assert record["comparator_status"] == "PASS" + assert record["baseline_eligible"] is True + assert record["host_cpu_score"] == 1.02 + + +def test_host_cpu_min_score_env_override(monkeypatch, tmp_path): + monkeypatch.setenv("PERF_HOST_CPU_MIN_SCORE", "0.5") + + exit_code, record, _ = _run_compare( + monkeypatch, tmp_path, + _v2_raw_result(avg_generation_time_s=20.0, host_cpu_score=0.55), + [_v2_baseline_record()]) + + assert exit_code == 1 + assert record["comparator_status"] == "REGRESSION" diff --git a/fastvideo/tests/performance/test_host_probe.py b/fastvideo/tests/performance/test_host_probe.py new file mode 100644 index 000000000..3e7f449d2 --- /dev/null +++ b/fastvideo/tests/performance/test_host_probe.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU-runnable smoke test for the host CPU health probe (~2s).""" + +import pytest + +from fastvideo.tests.performance import host_probe + + +def test_probe_reports_positive_normalized_profile(monkeypatch): + monkeypatch.setenv("PERF_HOST_CPU_ST_REF_KOPS", "1000") + + profile = host_probe.measure_host_cpu_profile() + + assert profile["host_cpu_single_thread_kops"] > 0 + assert profile["host_cpu_multi_thread_gflops"] > 0 + # The gated score is the single-thread axis normalized by the reference. + assert profile["host_cpu_score"] == pytest.approx( + profile["host_cpu_single_thread_kops"] / 1000.0, rel=0.05) diff --git a/fastvideo/tests/performance/test_inference_performance.py b/fastvideo/tests/performance/test_inference_performance.py index 8b90c3db3..2a581a3c6 100644 --- a/fastvideo/tests/performance/test_inference_performance.py +++ b/fastvideo/tests/performance/test_inference_performance.py @@ -19,6 +19,7 @@ import pytest from fastvideo import VideoGenerator from fastvideo.logger import init_logger +from fastvideo.tests.performance.host_probe import measure_host_cpu_profile from fastvideo.tests.performance.identity import ( benchmark_identity_from_config, build_recipe_from_benchmark_config, @@ -153,6 +154,24 @@ _BENCHMARK_CONFIGS = _discover_benchmarks() # -- Helpers ---------------------------------------------------------------- +# Probed once per pytest process (before the first benchmark's GPU work) and +# attached to every result: the comparator skips gating when the host is +# below the healthy-host CPU profile. Fail-open: a probe error just omits the +# fields, and the comparator gates normally. +_HOST_CPU_PROFILE: dict[str, float] | None = None + + +def _host_cpu_profile() -> dict[str, float]: + global _HOST_CPU_PROFILE + if _HOST_CPU_PROFILE is None: + try: + _HOST_CPU_PROFILE = measure_host_cpu_profile() + logger.info("Host CPU probe: %s", _HOST_CPU_PROFILE) + except Exception as e: + logger.warning("Host CPU probe failed (%s); records will carry no host_cpu_score", e) + _HOST_CPU_PROFILE = {} + return _HOST_CPU_PROFILE + def _get_thresholds(cfg): """Return thresholds dict for the current GPU from config.""" @@ -477,6 +496,7 @@ def _run_benchmark(cfg): num_warmup, num_measure = _validate_run_counts(run_config, cfg["benchmark_id"]) thresholds = _get_thresholds(cfg) + host_cpu_profile = _host_cpu_profile() # Remap JSON keys to VideoGenerator kwargs text_enc_prec = init_kwargs.pop("text_encoder_precisions", None) @@ -537,6 +557,9 @@ def _run_benchmark(cfg): runtime_identity=runtime_identity, device_name=device_name, ) + # Attach the host CPU probe so the comparator can skip gating on + # below-profile (packed) hosts. Fail-open: an empty probe adds nothing. + results.update(host_cpu_profile) logger.info( "Performance results: avg_time=%.2fs, "