From 652659a502cfaebbeb7fa6a44aca5a830ecbf35a Mon Sep 17 00:00:00 2001 From: Mac Lee Date: Sun, 5 Jul 2026 16:31:17 +0000 Subject: [PATCH] [ci]: add v2 performance baseline seeding --- .../reseed-performance-baseline/SKILL.md | 177 +++++++++++++---- docs/contributing/performance_benchmarks.md | 8 +- fastvideo/tests/performance/seed_baseline.py | 188 ++++++++++++++++++ .../test_compare_baseline_policy.py | 89 +++++++++ 4 files changed, 423 insertions(+), 39 deletions(-) create mode 100644 fastvideo/tests/performance/seed_baseline.py diff --git a/.agents/skills/reseed-performance-baseline/SKILL.md b/.agents/skills/reseed-performance-baseline/SKILL.md index ed3ddc06f..a32631167 100644 --- a/.agents/skills/reseed-performance-baseline/SKILL.md +++ b/.agents/skills/reseed-performance-baseline/SKILL.md @@ -1,20 +1,23 @@ --- name: reseed-performance-baseline -description: Re-seed the HF performance-tracking baseline for an intentional runtime, dependency, or environment-caused benchmark shift using one or more reviewed normalized performance JSONs. Use when performance CI fails because metrics such as latency, throughput, component time, or peak memory changed for an accepted reason and the rolling median baseline in FastVideo/performance-tracking must be advanced from a consistent batch of reviewed source results. The workflow backs up existing history under /tmp, validates all source JSONs for the same (model_id, gpu_type), rejects internally inconsistent source batches, uploads one success=true reseed record per accepted source JSON, and offers to clean local temp state after a successful upload. +description: Re-seed the HF performance-tracking baseline for an intentional runtime, dependency, environment-caused benchmark shift, or reviewed v2 calibration using one or more reviewed normalized performance JSONs. Use when performance CI fails because metrics such as latency, throughput, component time, or peak memory changed for an accepted reason and the rolling median baseline in FastVideo/performance-tracking must be advanced, or when a new v2 exact comparable identity needs its first approved baseline. The workflow backs up existing history under /tmp, validates all source JSONs for the same legacy (model_id, gpu_type) target or the same v2 exact identity, rejects internally inconsistent source batches, uploads one success=true baseline record per accepted source JSON, and offers to clean local temp state after a successful upload. --- # Re-seed Performance Baseline ## Purpose -Replace or advance the rolling performance baseline for a single -`(model_id, gpu_type)` pair in the HF dataset -`FastVideo/performance-tracking`. +Replace or advance the rolling performance baseline in the HF dataset +`FastVideo/performance-tracking`. Legacy targets are scoped by +`(model_id, gpu_type)`. V2 targets are scoped by exact comparable identity: +`workload_id`, `variant_id`, `benchmark_version`, `hardware_profile_id`, +`software_profile_id`, and `recipe_fingerprint`. -Performance comparison uses the median of up to the last 5 successful records -for the same model and GPU. Failed records are useful audit history, but they -do not move the future baseline because `compare_baseline.py` loads records -with `successful_only=True`. +Performance comparison uses the median of up to the last 5 successful, +baseline-eligible records for the same target. Failed or calibration-only +records are useful audit history, but they do not move the future baseline +because `compare_baseline.py` loads records with `successful_only=True` and +`baseline_eligible_only=True`. This skill now reseeds from a reviewed batch of one or more source performance JSONs. It uploads one new `success=true` record per accepted source JSON; it @@ -22,11 +25,13 @@ does not blindly replicate one measurement into 3 or 5 records. The effective reseed size is therefore dynamic and equals the number of provided, validated, internally consistent source JSONs. -If the operator provides fewer than 3 records, call out that the last-5 rolling -median may not move immediately. If the operator provides 3 consistent shifted -records, the rolling median usually moves immediately. If the operator provides -5 consistent shifted records, the last-5 window is effectively reset to the new -runtime profile. +For baseline shifts with existing history, if the operator provides fewer than +3 records, call out that the last-5 rolling median may not move immediately. If +the operator provides 3 consistent shifted records, the rolling median usually +moves immediately. If the operator provides 5 consistent shifted records, the +last-5 window is effectively reset to the new runtime profile. For the first +approved v2 baseline of a new exact identity, one reviewed calibration seed is +enough for the next comparable run to leave `CALIBRATION_NEEDED`. These records are intentional operator-approved baseline resets, not ordinary independent main-branch persistence. Mark them clearly with provenance fields @@ -66,8 +71,8 @@ approval, then upload reviewed accepted baseline records. | Parameter | Required | Description | |-----------|----------|-------------| -| `model_id` | Yes | Benchmark id, e.g. `wan-t2v-1.3b-2gpu`. This maps to the HF subdirectory after `sanitize(model_id)`. | -| `gpu_type` | Yes | Exact GPU device string from the performance record, e.g. the L40S device name emitted by CI. Baselines are GPU-specific. | +| `model_id` | Legacy required; v2 inferred | Benchmark id, e.g. `wan-t2v-1.3b-2gpu`. This maps to the HF subdirectory after `sanitize(model_id)`. For v2 records, use the `model_id` from each source artifact only as the upload directory; comparison is by exact identity. | +| `gpu_type` | Legacy required; v2 inferred | Exact GPU device string from the performance record, e.g. the L40S device name emitted by CI. V2 hardware matching uses `hardware_profile_id`; preserve `gpu_type` as display metadata. | | `source_results` | Yes | One or more local paths or Buildkite artifact URLs for accepted shifted performance JSONs. Prefer normalized `normalized_perf_*.json` artifacts emitted by `compare_baseline.py`. Accept `source_result` as an alias only for a single JSON. | | `max_intra_batch_regression` | No | Maximum allowed regression of any source JSON against the source batch median. Default: `0.05` (5%). | | `intent_rationale` | Yes | One-line explanation for why the baseline shift is legitimate. This is written into provenance and should be reused in the PR. | @@ -80,8 +85,9 @@ Hardcoded defaults: is supported). - Backup root: `/tmp/performance_reseed_backup`. - Download scratch root for source artifact URLs: `/tmp/performance_reseed_source`. -- Baseline window: last 5 `success=true` records for the same - `(model_id, gpu_type)`. +- Baseline window: last 5 `success=true`, `baseline_eligible=true` records + for the same legacy `(model_id, gpu_type)` target or the same v2 exact + comparable identity. - Reseed count: dynamic. Upload exactly one accepted seed record per validated source JSON. @@ -115,12 +121,22 @@ with open(source_result, encoding="utf-8") as f: record = json.load(f) ``` -Stop if any normalized record's `model_id` or `gpu_type` does not match the -requested `model_id` and `gpu_type`. +Classify the source batch before continuing: + +- **Legacy source records** have no v2 exact identity fields. Stop if any + normalized record's `model_id` or `gpu_type` does not match the requested + `model_id` and `gpu_type`. +- **V2 source records** have exact identity fields. Stop unless every source + record has all six comparable identity fields and they are identical across + the batch: `workload_id`, `variant_id`, `benchmark_version`, + `hardware_profile_id`, `software_profile_id`, and `recipe_fingerprint`. + Do not fall back to legacy `(model_id, gpu_type)` matching for v2 records. The source records may have `success: false` when they came from failed rolling baseline comparisons. That is expected; only the reviewed reseed records become new `success: true` baseline records after explicit approval. +For a first v2 baseline seed, the source records must instead be successful +`CALIBRATION_NEEDED` normalized artifacts. Sort validated source records by their original `timestamp` ascending before preparing the seed records. If a source timestamp is missing or unparsable, @@ -194,7 +210,7 @@ export HF_REPO_ID="${HF_REPO_ID:-FastVideo/performance-tracking}" python -c 'from fastvideo.performance.hf_store import sync_from_hf; import os; sync_from_hf(os.environ["PERFORMANCE_TRACKING_ROOT"], strict=True)' ``` -Then back up only the sanitized model directory under `/tmp`: +For legacy records, back up the sanitized model directory under `/tmp`: ```bash SHORT_COMMIT=$(git rev-parse --short=12 HEAD) @@ -209,6 +225,16 @@ mkdir -p "$BACKUP_DIR" cp -R "${PERFORMANCE_TRACKING_ROOT}/${MODEL_SAFE}" "$BACKUP_DIR/" 2>/dev/null || true ``` +For v2 records, back up the full local tracking root after sync. Exact identity +lookup scans across model directories, so a benchmark rename may have relevant +history outside the current source artifact's `model_id` directory: + +```bash +BACKUP_DIR="/tmp/performance_reseed_backup/${TIMESTAMP}_${SHORT_COMMIT}_v2_exact_identity" +mkdir -p "$BACKUP_DIR" +cp -R "${PERFORMANCE_TRACKING_ROOT}" "$BACKUP_DIR/tracking-root" +``` + Write provenance next to the backup: ```bash @@ -231,7 +257,9 @@ first baseline seed. Continue, but report that baseline history was empty. ### 3. Compute old baseline and candidate shift -Load the last 5 successful records for the target: +Load the last 5 successful baseline records for the target. + +For legacy targets: ```python from fastvideo.performance.hf_store import load_records_for_model @@ -242,6 +270,28 @@ records = load_records_for_model( "", last_n=5, successful_only=True, + baseline_eligible_only=True, +) +``` + +For v2 exact-identity targets: + +```python +from hf_store import load_records_for_identity + +records = load_records_for_identity( + "/tmp/perf-tracking", + { + "workload_id": "", + "variant_id": "", + "benchmark_version": "", + "hardware_profile_id": "", + "software_profile_id": "", + "recipe_fingerprint": "", + }, + last_n=5, + successful_only=True, + baseline_eligible_only=True, ) ``` @@ -257,7 +307,8 @@ medians after appending the proposed seed records, and source batch spread for: Also print how many successful old records exist. Make clear: -- 1 seed record usually does not move a last-5 median by itself. +- 1 seed record usually does not move an existing last-5 median by itself, but + it is enough to establish the first v2 baseline for a new exact identity. - 3 consistent seed records usually move the last-5 median immediately. - 5 consistent seed records effectively reset the last-5 window. - The records are intentional approved baseline resets and must be labeled @@ -267,10 +318,10 @@ Also print how many successful old records exist. Make clear: Require an explicit confirmation phrase before preparing the upload: -> About to RE-SEED performance baseline for `` on ``. +> About to RE-SEED performance baseline for ``. > This will upload `` new `success=true` records to -> `FastVideo/performance-tracking//`, one per accepted -> source JSON. +> `FastVideo/performance-tracking//` or the source +> artifact's v2 model directory, one per accepted source JSON. > > Reason: `` > Source results: `` @@ -288,25 +339,47 @@ Do not continue unless the user types exactly `confirm performance reseed`. ### 5. Create the accepted seed records -Create one seed record from each normalized source result. Do not copy the +Create one seed record from each normalized source result. + +For first v2 baseline seeds, use the scoped utility. It validates exact +identity, requires successful `CALIBRATION_NEEDED` source artifacts, preserves +the normalized v2 identity and metadata fields, and writes seed records with +`success=true`, `baseline_eligible=true`, and `comparison_status=PASS`: + +```bash +python fastvideo/tests/performance/seed_baseline.py \ + --source-result \ + --source-result \ + --intent-rationale "" \ + --tracking-root "${PERFORMANCE_TRACKING_ROOT}" +``` + +If the prepared seed records look correct, upload only those scoped records in +step 7. Do not rerun the utility with a different source list after approval. + +For legacy reseeds or accepted v2 baseline shifts from regression artifacts, +create one seed record from each normalized source result. Do not copy the source JSON wholesale. Infer the baseline field allowlist from all existing HF records for the target -`(model_id, gpu_type)` after syncing, including both `success=true` and -`success=false` records. Use the union of non-provenance keys present in those -target records, preserving only fields that also exist in the normalized -source record or are explicitly set by the reseed workflow. Always include -`model_id`, `timestamp`, and `success` because the upload path and baseline -loader depend on them. Always set `timestamp` to a fresh reseed timestamp and -`success` to `true`. Do not include unrelated source-only fields that are -absent from existing HF records. +after syncing, including both `success=true` and `success=false` records. For +legacy targets the target is `(model_id, gpu_type)`. For v2 baseline-shift +reseeds the target is the exact comparable identity. Use the union of +non-provenance keys present in those target records, preserving only fields +that also exist in the normalized source record or are explicitly set by the +reseed workflow. Always include `model_id`, `timestamp`, `success`, +`baseline_eligible`, and `comparison_status` because the upload path and +baseline loader depend on them. Always set `timestamp` to a fresh reseed +timestamp, `success` to `true`, `baseline_eligible` to `true`, and +`comparison_status` to `PASS`. Do not include unrelated source-only fields +that are absent from existing HF records. Exclude existing provenance or operator metadata from the inferred baseline field allowlist. At minimum, exclude keys prefixed with `baseline_reseed` and any fields known to be local-only audit metadata. -If there are no previous HF records for the target model/GPU, fall back to this -default baseline field list: +If there are no previous HF records for the target, fall back to this default +baseline field list: - `model_id` - `timestamp` @@ -319,6 +392,21 @@ default baseline field list: - `dit_time_s` - `vae_decode_time_s` - `success` +- `baseline_eligible` +- `comparison_status` + +For v2 baseline-shift reseeds with no previous HF records for the exact +identity, also preserve: + +- `workload_id` +- `variant_id` +- `benchmark_version` +- `recipe_fingerprint` +- `hardware_profile_id` +- `software_profile_id` +- `recipe` +- `hardware_profile` +- `software_profile` Do not upload extra fields from the source artifact. @@ -334,6 +422,19 @@ Optional provenance fields are allowed and useful: - `baseline_reseed_operator` - `baseline_reseed_max_intra_batch_regression` +The v2 calibration seed utility writes analogous first-seed provenance: + +- `baseline_seed: true` +- `baseline_seed_reason` +- `baseline_seed_source_result` +- `baseline_seed_source_status` +- `baseline_seed_source_timestamp` +- `baseline_seed_source_success` +- `baseline_seed_source_run_source` +- `baseline_seed_batch_size` +- `baseline_seed_batch_index` +- `baseline_seed_operator` + Use a fresh reseed timestamp for each seed record, not the original source result timestamp. This is required because `load_records_for_model(..., last_n=5)` keeps the last records after loading @@ -436,7 +537,9 @@ directories created for this reseed. Never remove unrelated `/tmp` contents. against the source batch median by more than `max_intra_batch_regression`. Ask for cleaner sources or a reviewed explanation before continuing. - **Too few source records to move the median.** Continue only after making - clear that one or two records may not immediately move the last-5 median. + clear that one or two records may not immediately move an existing last-5 + median. This warning does not block a first v2 calibration seed for an exact + identity with no eligible baseline yet. - **The source results are noisy or suspicious.** Stop. Reseeding amplifies those measurements into the baseline, so they must be reviewed first. - **HF sync fails.** Stop for destructive reseeds. A stale or empty sync can diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index 739546bdb..17f9113f5 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -210,7 +210,11 @@ visible but do not authorize future CI-gating recipe mismatch failures. When the baseline shifts for a legitimate reason (torch upgrade, kernel change, etc.) and CI starts failing, use the [`reseed-performance-baseline`](https://github.com/hao-ai-lab/FastVideo/blob/main/.agents/skills/reseed-performance-baseline/SKILL.md) -agent skill to advance the rolling median. +agent skill to advance the rolling median. To approve the first baseline for +a new v2 exact identity, use `fastvideo/tests/performance/seed_baseline.py` +from a reviewed `CALIBRATION_NEEDED` normalized artifact; it writes an +explicit `baseline_eligible=true` seed record with the same six identity +fields. ## Schemas @@ -516,7 +520,7 @@ When the rolling-baseline phase runs, it emits: baseline. V2 benchmarks with no exact comparable baseline report `CALIBRATION_NEEDED`; the record remains visible but does not become baseline eligible until a comparable scheduled-main run is reviewed and - seeded through the baseline workflow. + seeded with `fastvideo/tests/performance/seed_baseline.py`. 4. If the benchmark targets a GPU not currently in `thresholds`, either add that GPU as a key or rely on the `default` block. Note that `default` is diff --git a/fastvideo/tests/performance/seed_baseline.py b/fastvideo/tests/performance/seed_baseline.py new file mode 100644 index 000000000..b21392495 --- /dev/null +++ b/fastvideo/tests/performance/seed_baseline.py @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Prepare reviewed v2 calibration artifacts as baseline seed records.""" + +import argparse +import json +import os +import sys +from datetime import datetime, timezone +from typing import Any + +try: + from fastvideo.tests.performance.compare_baseline import ( + STATUS_CALIBRATION_NEEDED, + STATUS_PASS, + _comparison_identity_filters, + _record_uses_v2_identity, + ) + from fastvideo.performance.hf_store import sanitize, upload_record +except ImportError: + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) + if repo_root not in sys.path: + sys.path.insert(0, repo_root) + from fastvideo.tests.performance.compare_baseline import ( + STATUS_CALIBRATION_NEEDED, + STATUS_PASS, + _comparison_identity_filters, + _record_uses_v2_identity, + ) + from fastvideo.performance.hf_store import sanitize, upload_record + +TRACKING_ROOT = os.environ.get("PERFORMANCE_TRACKING_ROOT", "/tmp/perf-tracking") + + +def _now_utc_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _load_json(path: str) -> dict[str, Any]: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _write_json(path: str, payload: dict[str, Any]) -> None: + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + +def _source_identity(record: dict[str, Any]) -> dict[str, str]: + if not _record_uses_v2_identity(record): + raise ValueError("baseline seeds require a v2 record with exact comparable identity fields") + return _comparison_identity_filters(record) + + +def _validate_calibration_source(record: dict[str, Any]) -> None: + _source_identity(record) + if record.get("comparison_status") != STATUS_CALIBRATION_NEEDED: + raise ValueError("baseline seeds require a CALIBRATION_NEEDED source artifact") + if record.get("success") is not True: + raise ValueError("baseline seeds require a successful source artifact") + + +def build_baseline_seed_record( + source_record: dict[str, Any], + *, + reason: str, + source_result: str | None = None, + operator: str | None = None, + timestamp: str | None = None, + batch_size: int = 1, + batch_index: int = 1, +) -> dict[str, Any]: + """Return an approved v2 baseline seed derived from a calibration artifact.""" + if not reason.strip(): + raise ValueError("baseline seed reason must be a non-empty string") + _validate_calibration_source(source_record) + + seed = dict(source_record) + seed.update({ + "timestamp": timestamp or _now_utc_iso(), + "success": True, + "baseline_eligible": True, + "comparison_status": STATUS_PASS, + "comparison_status_reason": "Approved baseline seed from CALIBRATION_NEEDED source artifact", + "baseline_seed": True, + "baseline_seed_reason": reason, + "baseline_seed_source_status": source_record.get("comparison_status"), + "baseline_seed_source_timestamp": source_record.get("timestamp"), + "baseline_seed_source_success": source_record.get("success"), + "baseline_seed_source_run_source": source_record.get("run_source"), + "baseline_seed_batch_size": batch_size, + "baseline_seed_batch_index": batch_index, + }) + if source_result: + seed["baseline_seed_source_result"] = source_result + if operator: + seed["baseline_seed_operator"] = operator + return seed + + +def write_seed_record( + local_dir: str, + record: dict[str, Any], + *, + suffix: str | None = None, +) -> str: + """Write *record* under the tracking root and return the local JSON path.""" + model_dir = os.path.join(local_dir, sanitize(record["model_id"])) + os.makedirs(model_dir, exist_ok=True) + + timestamp = sanitize(record["timestamp"]) + commit = sanitize(record.get("commit_sha") or "unknown") + suffix_part = f"_{sanitize(suffix)}" if suffix else "" + out_path = os.path.join(model_dir, f"{timestamp}_{commit}_seed{suffix_part}.json") + _write_json(out_path, record) + return out_path + + +def _validate_same_identity(records: list[dict[str, Any]]) -> dict[str, str]: + first = _source_identity(records[0]) + for index, record in enumerate(records[1:], start=2): + identity = _source_identity(record) + if identity != first: + raise ValueError(f"source artifact {index} does not match the first exact comparable identity") + return first + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Create reviewed baseline seed records from v2 CALIBRATION_NEEDED artifacts.", + ) + parser.add_argument( + "--source-result", + dest="source_results", + action="append", + required=True, + help="Path to a normalized_perf_*.json artifact. Repeat for multiple reviewed source artifacts.", + ) + parser.add_argument( + "--intent-rationale", + required=True, + help="Reviewed reason this calibration artifact should seed the baseline.", + ) + parser.add_argument( + "--tracking-root", + default=TRACKING_ROOT, + help="Local performance tracking root to write seed records under.", + ) + parser.add_argument( + "--operator", + default=os.environ.get("USER"), + help="Operator name recorded in seed provenance.", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload prepared seed records to the configured HF performance tracking dataset.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + source_records = [_load_json(path) for path in args.source_results] + identity = _validate_same_identity(source_records) + print("Seeding exact comparable identity:") + for key, value in identity.items(): + print(f" {key}: {value}") + + for index, (source_path, source_record) in enumerate(zip(args.source_results, source_records), start=1): + seed_record = build_baseline_seed_record( + source_record, + reason=args.intent_rationale, + source_result=source_path, + operator=args.operator, + batch_size=len(source_records), + batch_index=index, + ) + suffix = f"{index:02d}" if len(source_records) > 1 else None + seed_path = write_seed_record(args.tracking_root, seed_record, suffix=suffix) + print(f"Prepared baseline seed: {seed_path}") + if args.upload: + upload_record(seed_path, seed_record, strict=True) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fastvideo/tests/performance/test_compare_baseline_policy.py b/fastvideo/tests/performance/test_compare_baseline_policy.py index e1509af34..0b12979c2 100644 --- a/fastvideo/tests/performance/test_compare_baseline_policy.py +++ b/fastvideo/tests/performance/test_compare_baseline_policy.py @@ -5,6 +5,7 @@ import json import pytest from fastvideo.tests.performance import compare_baseline +from fastvideo.tests.performance import seed_baseline from fastvideo.performance.hf_store import ( load_records_for_identity, load_records_for_model, @@ -554,6 +555,94 @@ def test_missing_exact_v2_baseline_reports_calibration_needed(monkeypatch): assert "No baseline found for exact comparable identity" in reason +def test_seeded_v2_calibration_artifact_enables_next_compare_pass( + monkeypatch, + tmp_path, +): + tracking_root = tmp_path / "tracking" + results_dir = tmp_path / "results" + reports_dir = tmp_path / "reports" + results_dir.mkdir() + + calibration_record = _v2_record() + calibration_record.update({ + "comparison_status": compare_baseline.STATUS_CALIBRATION_NEEDED, + "comparison_status_reason": "No baseline found for exact comparable identity", + "success": True, + "baseline_eligible": False, + "run_source": "scheduled_main", + }) + seed_record = seed_baseline.build_baseline_seed_record( + calibration_record, + reason="reviewed first v2 baseline", + timestamp="2026-06-17T00:00:00+00:00", + operator="test", + ) + seed_baseline.write_seed_record(str(tracking_root), seed_record) + + assert seed_record["baseline_eligible"] is True + assert seed_record["comparison_status"] == compare_baseline.STATUS_PASS + assert seed_record["baseline_seed_source_status"] == compare_baseline.STATUS_CALIBRATION_NEEDED + assert load_records_for_identity( + str(tracking_root), + compare_baseline._comparison_identity_filters(calibration_record), + baseline_eligible_only=True, + ) == [seed_record] + + with open(results_dir / "perf_current.json", "w", encoding="utf-8") as f: + json.dump(_v2_raw_result( + avg_generation_time_s=10.1, + timestamp="2026-06-18T00:00:00+00:00", + ), f) + + monkeypatch.setenv("PERF_RUN_SOURCE", "pr") + monkeypatch.setattr(compare_baseline, "TRACKING_ROOT", str(tracking_root)) + monkeypatch.setattr(compare_baseline, "RESULTS_DIR", str(results_dir)) + monkeypatch.setattr(compare_baseline, "PERF_REPORTS_DIR", str(reports_dir)) + monkeypatch.setattr(compare_baseline, "UPLOAD_POLICY", "never") + monkeypatch.setattr(compare_baseline, "sync_from_hf", lambda local_dir, strict=False: local_dir) + monkeypatch.delenv("PERF_PYTEST_RC", raising=False) + + assert compare_baseline.main() == 0 + + artifacts = list((reports_dir / "results").glob("normalized_perf_*.json")) + assert len(artifacts) == 1 + with open(artifacts[0], encoding="utf-8") as f: + normalized = json.load(f) + + assert normalized["comparison_status"] == compare_baseline.STATUS_PASS + assert normalized["baseline_eligible"] is False + + +def test_baseline_seed_rejects_non_calibration_sources(monkeypatch): + monkeypatch.setenv("PERF_RUN_SOURCE", "scheduled_main") + record = _v2_record() + record.update({ + "comparison_status": compare_baseline.STATUS_PASS, + "success": True, + }) + + with pytest.raises(ValueError, match="CALIBRATION_NEEDED"): + seed_baseline.build_baseline_seed_record( + record, + reason="not a calibration", + ) + + +def test_baseline_seed_rejects_mixed_exact_identities(monkeypatch): + monkeypatch.setenv("PERF_RUN_SOURCE", "scheduled_main") + first = _v2_record() + second = _v2_record(variant_id="different-variant") + for record in (first, second): + record.update({ + "comparison_status": compare_baseline.STATUS_CALIBRATION_NEEDED, + "success": True, + }) + + with pytest.raises(ValueError, match="exact comparable identity"): + seed_baseline._validate_same_identity([first, second]) + + def test_same_variant_changed_recipe_reports_recipe_mismatch(monkeypatch): monkeypatch.setenv("PERF_RUN_SOURCE", "pr") record = _v2_record(recipe_fingerprint="recipe-2")