130 lines
3.7 KiB
Python
130 lines
3.7 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
import pytest
|
|
|
|
from fastvideo.tests.performance.test_inference_performance import (
|
|
_benchmark_display_id,
|
|
_config_identity_metadata,
|
|
_is_v2_config,
|
|
_validate_benchmark_config,
|
|
)
|
|
|
|
|
|
def _v2_config():
|
|
return {
|
|
"benchmark_id": "wan-t2v-1.3b-2gpu",
|
|
"config_schema_version": 2,
|
|
"workload_id": "wan-t2v",
|
|
"variant_id": "1.3b-sp2",
|
|
"benchmark_version": 2,
|
|
}
|
|
|
|
|
|
def test_v1_benchmark_config_without_schema_version_validates():
|
|
cfg = {
|
|
"benchmark_id": "legacy-benchmark",
|
|
}
|
|
|
|
_validate_benchmark_config(cfg, "legacy.json")
|
|
|
|
assert _is_v2_config(cfg) is False
|
|
assert _config_identity_metadata(cfg) == {}
|
|
assert _benchmark_display_id(cfg) == "legacy-benchmark"
|
|
|
|
|
|
def test_v2_benchmark_config_identity_validates_and_is_preserved():
|
|
cfg = _v2_config()
|
|
cfg["quality_metadata"] = {"some": "data"}
|
|
|
|
_validate_benchmark_config(cfg, "wan.json")
|
|
|
|
assert _is_v2_config(cfg) is True
|
|
assert _config_identity_metadata(cfg) == {
|
|
"config_schema_version": 2,
|
|
"workload_id": "wan-t2v",
|
|
"variant_id": "1.3b-sp2",
|
|
"benchmark_version": 2,
|
|
"quality_metadata": {
|
|
"some": "data"
|
|
},
|
|
}
|
|
|
|
|
|
def test_v2_benchmark_config_missing_identity_fields_fails_clearly():
|
|
cfg = _v2_config()
|
|
del cfg["variant_id"]
|
|
del cfg["benchmark_version"]
|
|
|
|
expected = "wan.json: missing required v2 identity fields: variant_id, benchmark_version"
|
|
with pytest.raises(ValueError, match=expected):
|
|
_validate_benchmark_config(cfg, "wan.json")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field", "value"),
|
|
[
|
|
("workload_id", {}),
|
|
("workload_id", ""),
|
|
("workload_id", " "),
|
|
("variant_id", []),
|
|
("variant_id", ""),
|
|
("variant_id", " "),
|
|
],
|
|
)
|
|
def test_v2_benchmark_config_rejects_invalid_string_identity_values(field, value):
|
|
cfg = _v2_config()
|
|
cfg[field] = value
|
|
|
|
expected = f"wan.json: v2 identity field {field!r} must be a non-empty string"
|
|
with pytest.raises(ValueError, match=expected):
|
|
_validate_benchmark_config(cfg, "wan.json")
|
|
|
|
|
|
@pytest.mark.parametrize("value", [None, "1", 1.5, True])
|
|
def test_v2_benchmark_config_rejects_invalid_benchmark_version_values(value):
|
|
cfg = _v2_config()
|
|
cfg["benchmark_version"] = value
|
|
|
|
expected = "wan.json: v2 identity field 'benchmark_version' must be an integer"
|
|
with pytest.raises(ValueError, match=expected):
|
|
_validate_benchmark_config(cfg, "wan.json")
|
|
|
|
|
|
def test_partial_v2_identity_requires_schema_version():
|
|
cfg = {
|
|
"benchmark_id": "wan-t2v-1.3b-2gpu",
|
|
"workload_id": "wan-t2v",
|
|
"variant_id": "1.3b-sp2",
|
|
"benchmark_version": 2,
|
|
}
|
|
|
|
expected = "wan.json: v2 benchmark identity fields require config_schema_version=2"
|
|
with pytest.raises(ValueError, match=expected):
|
|
_validate_benchmark_config(cfg, "wan.json")
|
|
|
|
|
|
def test_optional_v2_metadata_fields_must_be_objects():
|
|
cfg = _v2_config()
|
|
cfg["quality_metadata"] = ["not", "an", "object"]
|
|
|
|
expected = "wan.json: optional v2 metadata field 'quality_metadata' must be an object"
|
|
with pytest.raises(ValueError, match=expected):
|
|
_validate_benchmark_config(cfg, "wan.json")
|
|
|
|
|
|
def test_regression_thresholds_validate_without_forcing_v2_schema():
|
|
cfg = {
|
|
"benchmark_id": "legacy-benchmark",
|
|
"regression_thresholds": {
|
|
"latency": {
|
|
"threshold_percent": 0.1
|
|
}
|
|
},
|
|
}
|
|
|
|
_validate_benchmark_config(cfg, "legacy.json")
|
|
|
|
cfg["regression_thresholds"] = []
|
|
with pytest.raises(ValueError, match="benchmark config field 'regression_thresholds' must be an object"):
|
|
_validate_benchmark_config(cfg, "legacy.json")
|