diff --git a/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json b/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json index 7893ddde2..8a568ef6b 100644 --- a/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json +++ b/.buildkite/performance-benchmarks/tests/wan-t2v-1.3b.json @@ -1,5 +1,9 @@ { "benchmark_id": "wan-t2v-1.3b-2gpu", + "config_schema_version": 2, + "workload_id": "wan-t2v-1.3b", + "variant_id": "canonical", + "benchmark_version": 1, "description": "Wan2.1 T2V 1.3B inference performance", "model": { "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", diff --git a/docs/contributing/performance_benchmarks.md b/docs/contributing/performance_benchmarks.md index dbb64b45e..0d5d6f919 100644 --- a/docs/contributing/performance_benchmarks.md +++ b/docs/contributing/performance_benchmarks.md @@ -172,6 +172,45 @@ agent skill to advance the rolling median. ## Schemas +### Benchmark config (`.buildkite/performance-benchmarks/tests/*.json`) + +Benchmark configs without `config_schema_version` are treated as legacy v1 +configs and remain loadable. New or migrated configs should use +`config_schema_version: 2` and include explicit comparable identity fields: + +```jsonc +{ + "benchmark_id": "wan-t2v-1.3b-2gpu", + "config_schema_version": 2, + "workload_id": "wan-t2v-1.3b", + "variant_id": "canonical", + "benchmark_version": 1 +} +``` + +`benchmark_id` is still required in this phase because raw artifact names, +generated-video directories, normalized record paths, and the current rolling +baseline comparator still depend on it. The v2 identity fields are config +metadata that make the measured workload explicit: + +| Field | Purpose | +|---|---| +| `workload_id` | Stable benchmark family, such as `wan-t2v-1.3b`. | +| `variant_id` | Intentional recipe family, such as `canonical`. | +| `benchmark_version` | Version of the measurement protocol and comparison policy. | + +If a config declares `config_schema_version: 2`, loading fails clearly when any +required v2 identity field is missing. If v2 identity or metadata fields are +added without `config_schema_version: 2`, loading also fails so partial +migrations do not silently run as v1 configs. Optional v2 metadata fields +reserved for follow-up work, such as `recipe`, `metric_threshold_policy`, and +`quality_metadata`, must be JSON objects when present. + +Recipe fingerprinting, hardware/software profile IDs, exact-identity +comparison, metric-specific threshold policy behavior, promoted baselines, and +dashboard regrouping are separate follow-up changes. Until those land, rolling +baseline comparison remains keyed by `(model_id, gpu_type)`. + ### Raw record (`results/perf_*.json`) Written by `test_inference_performance.py`. One file per benchmark run. @@ -179,6 +218,10 @@ Written by `test_inference_performance.py`. One file per benchmark run. ```jsonc { "benchmark_id": "wan-t2v-1.3b-2gpu", + "config_schema_version": 2, + "workload_id": "wan-t2v-1.3b", + "variant_id": "canonical", + "benchmark_version": 1, "model_short_name": "Wan2.1-T2V-1.3B-Diffusers", "device": "NVIDIA L40S", "num_gpus": 2, @@ -279,11 +322,16 @@ observability. When pytest passes, the rolling-baseline phase emits: ## Adding a new benchmark 1. Drop a new JSON config into - `.buildkite/performance-benchmarks/tests/.json`. Required keys: + `.buildkite/performance-benchmarks/tests/.json`. New configs should + use v2 identity fields: ```json { "benchmark_id": "", + "config_schema_version": 2, + "workload_id": "", + "variant_id": "canonical", + "benchmark_version": 1, "model": { "model_path": "...", "model_short_name": "..." }, "init_kwargs": { "num_gpus": 1, ... }, "generation_kwargs": { "num_frames": 45, ... }, @@ -303,6 +351,10 @@ observability. When pytest passes, the rolling-baseline phase emits: } ``` + Legacy v1 configs without `config_schema_version` still load, but should not + gain v2 identity or metadata fields until they are migrated to + `config_schema_version: 2`. + 2. The pytest test auto-discovers all configs — no test code needed. CI picks it up on the next `/test performance` run. diff --git a/fastvideo/tests/performance/test_benchmark_config_validation.py b/fastvideo/tests/performance/test_benchmark_config_validation.py new file mode 100644 index 000000000..7b27f0235 --- /dev/null +++ b/fastvideo/tests/performance/test_benchmark_config_validation.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from fastvideo.tests.performance.test_inference_performance import ( + _benchmark_display_id, + _config_identity_metadata, + _is_v2_config, + _validate_benchmark_config, +) + + +def _v2_config(): + return { + "benchmark_id": "wan-t2v-1.3b-2gpu", + "config_schema_version": 2, + "workload_id": "wan-t2v-1.3b", + "variant_id": "canonical", + "benchmark_version": 1, + } + + +def test_v1_benchmark_config_without_schema_version_validates(): + cfg = { + "benchmark_id": "legacy-benchmark", + } + + _validate_benchmark_config(cfg, "legacy.json") + + assert _is_v2_config(cfg) is False + assert _config_identity_metadata(cfg) == {} + assert _benchmark_display_id(cfg) == "legacy-benchmark" + + +def test_v2_benchmark_config_identity_validates_and_is_preserved(): + cfg = _v2_config() + cfg["quality_metadata"] = {"some": "data"} + + _validate_benchmark_config(cfg, "wan.json") + + assert _is_v2_config(cfg) is True + assert _config_identity_metadata(cfg) == { + "config_schema_version": 2, + "workload_id": "wan-t2v-1.3b", + "variant_id": "canonical", + "benchmark_version": 1, + "quality_metadata": {"some": "data"}, + } + + +def test_v2_benchmark_config_missing_identity_fields_fails_clearly(): + cfg = _v2_config() + del cfg["variant_id"] + del cfg["benchmark_version"] + + expected = "wan.json: missing required v2 identity fields: variant_id, benchmark_version" + with pytest.raises(ValueError, match=expected): + _validate_benchmark_config(cfg, "wan.json") + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("workload_id", {}), + ("workload_id", ""), + ("workload_id", " "), + ("variant_id", []), + ("variant_id", ""), + ("variant_id", " "), + ], +) +def test_v2_benchmark_config_rejects_invalid_string_identity_values(field, value): + cfg = _v2_config() + cfg[field] = value + + expected = f"wan.json: v2 identity field {field!r} must be a non-empty string" + with pytest.raises(ValueError, match=expected): + _validate_benchmark_config(cfg, "wan.json") + + +@pytest.mark.parametrize("value", [None, "1", 1.5, True]) +def test_v2_benchmark_config_rejects_invalid_benchmark_version_values(value): + cfg = _v2_config() + cfg["benchmark_version"] = value + + expected = "wan.json: v2 identity field 'benchmark_version' must be an integer" + with pytest.raises(ValueError, match=expected): + _validate_benchmark_config(cfg, "wan.json") + + +def test_partial_v2_identity_requires_schema_version(): + cfg = { + "benchmark_id": "wan-t2v-1.3b-2gpu", + "workload_id": "wan-t2v-1.3b", + "variant_id": "canonical", + "benchmark_version": 1, + } + + expected = "wan.json: v2 benchmark identity fields require config_schema_version=2" + with pytest.raises(ValueError, match=expected): + _validate_benchmark_config(cfg, "wan.json") + + +def test_optional_v2_metadata_fields_must_be_objects(): + cfg = { + "benchmark_id": "wan-t2v-1.3b-2gpu", + "config_schema_version": 2, + "workload_id": "wan-t2v-1.3b", + "variant_id": "canonical", + "benchmark_version": 1, + "quality_metadata": ["not", "an", "object"], + } + + expected = "wan.json: optional v2 metadata field 'quality_metadata' must be an object" + with pytest.raises(ValueError, match=expected): + _validate_benchmark_config(cfg, "wan.json") diff --git a/fastvideo/tests/performance/test_inference_performance.py b/fastvideo/tests/performance/test_inference_performance.py index 2406bebb8..e4ee84b47 100644 --- a/fastvideo/tests/performance/test_inference_performance.py +++ b/fastvideo/tests/performance/test_inference_performance.py @@ -28,6 +28,17 @@ STAGE_METRIC_MAP: dict[str, str] = { "DmdDenoisingStage": "dit_time_s", "DecodingStage": "vae_decode_time_s", } +V2_CONFIG_SCHEMA_VERSION = 2 +V2_REQUIRED_IDENTITY_FIELDS = ( + "workload_id", + "variant_id", + "benchmark_version", +) +V2_OPTIONAL_METADATA_FIELDS = ( + "recipe", + "metric_threshold_policy", + "quality_metadata", +) # -- Config discovery ------------------------------------------------------- @@ -42,6 +53,71 @@ _BENCHMARKS_DIR = os.path.join( ) +def _has_v2_fields(cfg): + v2_fields = V2_REQUIRED_IDENTITY_FIELDS + V2_OPTIONAL_METADATA_FIELDS + return any(field in cfg for field in v2_fields) + + +def _is_v2_config(cfg): + return cfg.get("config_schema_version") == V2_CONFIG_SCHEMA_VERSION + + +def _validate_non_empty_string(value, field, path): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{path}: v2 identity field {field!r} must be a non-empty string") + + +def _validate_integer(value, field, path): + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{path}: v2 identity field {field!r} must be an integer") + + +def _validate_benchmark_config(cfg, path=""): + missing_common = [field for field in ("benchmark_id",) if field not in cfg] + if missing_common: + raise ValueError(f"{path}: missing required benchmark config fields: {', '.join(missing_common)}") + + schema_version = cfg.get("config_schema_version") + if schema_version is None: + if _has_v2_fields(cfg): + raise ValueError(f"{path}: v2 benchmark identity fields require config_schema_version=2") + return + + if schema_version != V2_CONFIG_SCHEMA_VERSION: + raise ValueError(f"{path}: unsupported benchmark config_schema_version={schema_version!r}") + + missing_v2 = [field for field in V2_REQUIRED_IDENTITY_FIELDS if field not in cfg] + if missing_v2: + raise ValueError(f"{path}: missing required v2 identity fields: {', '.join(missing_v2)}") + + _validate_non_empty_string(cfg["workload_id"], "workload_id", path) + _validate_non_empty_string(cfg["variant_id"], "variant_id", path) + _validate_integer(cfg["benchmark_version"], "benchmark_version", path) + + for field in V2_OPTIONAL_METADATA_FIELDS: + if field in cfg and not isinstance(cfg[field], Mapping): + raise ValueError(f"{path}: optional v2 metadata field {field!r} must be an object") + + +def _config_identity_metadata(cfg): + if not _is_v2_config(cfg): + return {} + metadata = { + "config_schema_version": cfg["config_schema_version"], + "workload_id": cfg["workload_id"], + "variant_id": cfg["variant_id"], + "benchmark_version": cfg["benchmark_version"], + } + for field in V2_OPTIONAL_METADATA_FIELDS: + if field in cfg: + metadata[field] = cfg[field] + return metadata + + +def _benchmark_display_id(cfg): + return cfg["benchmark_id"] + + def _discover_benchmarks(): """Glob benchmark JSON configs and return list of (id, config) tuples.""" pattern = os.path.join(_BENCHMARKS_DIR, "*.json") @@ -49,6 +125,7 @@ def _discover_benchmarks(): for path in sorted(glob.glob(pattern)): with open(path) as f: cfg = json.load(f) + _validate_benchmark_config(cfg, path) configs.append(cfg) return configs @@ -219,6 +296,7 @@ def _run_benchmark(cfg): results = { "benchmark_id": cfg["benchmark_id"], + **_config_identity_metadata(cfg), "model_short_name": model_info.get("model_short_name", ""), "device": device_name, "num_gpus": init_kwargs.get("num_gpus", 1), @@ -275,7 +353,7 @@ def _run_benchmark(cfg): @pytest.mark.parametrize( "cfg", _BENCHMARK_CONFIGS, - ids=[c["benchmark_id"] for c in _BENCHMARK_CONFIGS], + ids=[_benchmark_display_id(c) for c in _BENCHMARK_CONFIGS], ) def test_inference_performance(cfg): """Measure generation latency, peak GPU memory, and component-level timings