441 lines
14 KiB
Python
441 lines
14 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
from copy import deepcopy
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from fastvideo.tests.performance import identity as identity_module
|
|
from fastvideo.tests.performance.identity import (
|
|
build_recipe_from_benchmark_config,
|
|
canonical_json,
|
|
environment_fingerprint,
|
|
environment_metadata,
|
|
hardware_profile,
|
|
hardware_profile_id,
|
|
recipe_fingerprint,
|
|
resolved_revision_from_model_path,
|
|
software_profile,
|
|
software_profile_id,
|
|
)
|
|
|
|
|
|
def _benchmark_config():
|
|
return {
|
|
"benchmark_id": "wan-t2v-1.3b-2gpu",
|
|
"workload_id": "wan-t2v",
|
|
"variant_id": "1.3b-sp2",
|
|
"benchmark_version": 2,
|
|
"model": {
|
|
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
|
"model_short_name": "Wan2.1-T2V-1.3B",
|
|
},
|
|
"init_kwargs": {
|
|
"num_gpus": 2,
|
|
"sp_size": 2,
|
|
"tp_size": 1,
|
|
"vae_sp": True,
|
|
"vae_tiling": True,
|
|
"text_encoder_precisions": ["fp32"],
|
|
},
|
|
"generation_kwargs": {
|
|
"height": 480,
|
|
"width": 832,
|
|
"num_frames": 45,
|
|
"num_inference_steps": 4,
|
|
"guidance_scale": 3,
|
|
"embedded_cfg_scale": 6,
|
|
"seed": 1024,
|
|
"fps": 24,
|
|
"neg_prompt": "low quality",
|
|
},
|
|
"test_prompts": ["A cinematic video."],
|
|
}
|
|
|
|
|
|
def _fingerprint(cfg, attention_backend="FLASH_ATTN"):
|
|
recipe = build_recipe_from_benchmark_config(cfg, attention_backend=attention_backend)
|
|
return recipe_fingerprint(recipe)
|
|
|
|
|
|
def test_canonical_json_is_stable_for_mapping_order():
|
|
left = {"b": [2, 1], "a": {"z": "last", "m": "middle"}}
|
|
right = {"a": {"m": "middle", "z": "last"}, "b": [2, 1]}
|
|
|
|
assert canonical_json(left) == canonical_json(right)
|
|
|
|
|
|
def test_canonical_json_handles_sets_deterministically():
|
|
assert canonical_json({"values": {"b", "a"}}) == '{"values":["a","b"]}'
|
|
assert canonical_json({"values": frozenset(("b", "a"))}) == '{"values":["a","b"]}'
|
|
|
|
|
|
def test_same_config_produces_same_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
|
|
assert _fingerprint(cfg) == _fingerprint(deepcopy(cfg))
|
|
|
|
|
|
def test_recipe_includes_first_class_benchmark_identity():
|
|
recipe = build_recipe_from_benchmark_config(_benchmark_config())
|
|
|
|
assert recipe["recipe_schema_version"] == identity_module.RECIPE_SCHEMA_VERSION == 2
|
|
assert recipe["benchmark"] == {
|
|
"benchmark_id": "wan-t2v-1.3b-2gpu",
|
|
"workload_id": "wan-t2v",
|
|
"variant_id": "1.3b-sp2",
|
|
"benchmark_version": 2,
|
|
}
|
|
|
|
|
|
def test_recipe_requires_first_class_benchmark_identity():
|
|
cfg = _benchmark_config()
|
|
del cfg["workload_id"]
|
|
|
|
with pytest.raises(ValueError, match="workload_id"):
|
|
build_recipe_from_benchmark_config(cfg)
|
|
|
|
|
|
def test_semantically_equivalent_sequence_forms_match():
|
|
cfg = _benchmark_config()
|
|
equivalent = deepcopy(cfg)
|
|
equivalent["init_kwargs"]["text_encoder_precisions"] = tuple(equivalent["init_kwargs"]["text_encoder_precisions"])
|
|
|
|
assert _fingerprint(cfg) == _fingerprint(equivalent)
|
|
|
|
|
|
def test_output_path_does_not_change_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
with_output_path = deepcopy(cfg)
|
|
with_output_path["generation_kwargs"]["output_path"] = "/tmp/generated"
|
|
|
|
assert _fingerprint(cfg) == _fingerprint(with_output_path)
|
|
|
|
|
|
def test_runtime_resolved_values_do_not_change_recipe_fingerprint():
|
|
# Runtime-resolved values are audit metadata: an upstream repo commit
|
|
# (even a README-only edit) or a silent attention-backend fallback must
|
|
# not churn the cohort — that would open a fresh ungated cohort and let
|
|
# degraded numbers seed a new baseline. Declared inputs (including
|
|
# requested_backend) stay in the hash.
|
|
cfg = _benchmark_config()
|
|
recipe = build_recipe_from_benchmark_config(cfg,
|
|
attention_backend="FLASH_ATTN",
|
|
resolved_attention_backend="FLASH_ATTN",
|
|
resolved_model_revision="aaaa1111")
|
|
changed = build_recipe_from_benchmark_config(cfg,
|
|
attention_backend="FLASH_ATTN",
|
|
resolved_attention_backend="TORCH_SDPA",
|
|
resolved_model_revision="bbbb2222")
|
|
|
|
assert recipe["model"]["resolved_revision"] == "aaaa1111" # kept for audit
|
|
assert recipe["attention"]["resolved_backend"] == "FLASH_ATTN" # kept for audit
|
|
assert recipe_fingerprint(recipe) == recipe_fingerprint(changed)
|
|
|
|
|
|
def test_measured_prompt_override_ignores_unused_configured_prompts():
|
|
cfg = _benchmark_config()
|
|
changed = deepcopy(cfg)
|
|
changed["test_prompts"] = [
|
|
"Unused prompt that should not affect this measured workload.",
|
|
cfg["test_prompts"][0],
|
|
]
|
|
|
|
recipe = build_recipe_from_benchmark_config(cfg, measured_prompts=[cfg["test_prompts"][0]])
|
|
changed_recipe = build_recipe_from_benchmark_config(changed, measured_prompts=[cfg["test_prompts"][0]])
|
|
|
|
assert recipe_fingerprint(recipe) == recipe_fingerprint(changed_recipe)
|
|
|
|
|
|
def test_num_inference_steps_changes_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
changed = deepcopy(cfg)
|
|
changed["generation_kwargs"]["num_inference_steps"] = 8
|
|
|
|
assert _fingerprint(cfg) != _fingerprint(changed)
|
|
|
|
|
|
def test_benchmark_version_changes_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
changed = deepcopy(cfg)
|
|
changed["benchmark_version"] = 3
|
|
|
|
assert _fingerprint(cfg) != _fingerprint(changed)
|
|
|
|
|
|
def test_attention_backend_changes_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
|
|
assert _fingerprint(cfg, attention_backend="FLASH_ATTN") != _fingerprint(cfg, attention_backend="TORCH_SDPA")
|
|
|
|
|
|
def test_precision_changes_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
changed = deepcopy(cfg)
|
|
changed["init_kwargs"]["text_encoder_precisions"] = ["bf16"]
|
|
|
|
assert _fingerprint(cfg) != _fingerprint(changed)
|
|
|
|
|
|
def test_dimensions_and_seed_change_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
different_dimensions = deepcopy(cfg)
|
|
different_dimensions["generation_kwargs"]["width"] = 1024
|
|
different_seed = deepcopy(cfg)
|
|
different_seed["generation_kwargs"]["seed"] = 2048
|
|
|
|
assert _fingerprint(cfg) != _fingerprint(different_dimensions)
|
|
assert _fingerprint(cfg) != _fingerprint(different_seed)
|
|
|
|
|
|
def test_distributed_layout_changes_recipe_fingerprint():
|
|
cfg = _benchmark_config()
|
|
changed = deepcopy(cfg)
|
|
changed["init_kwargs"]["sp_size"] = 1
|
|
changed["init_kwargs"]["tp_size"] = 2
|
|
|
|
assert _fingerprint(cfg) != _fingerprint(changed)
|
|
|
|
|
|
def test_software_profile_uses_exact_attention_kernel_package_versions_for_id():
|
|
profile_a = software_profile(
|
|
python_version="3.12.4",
|
|
torch_version="2.12.0+cu130",
|
|
cuda_version="13.0",
|
|
package_versions={
|
|
"triton": "3.4.1",
|
|
"fastvideo_kernel": "0.3.2",
|
|
"flash_attn_4": "4.0.0.dev0",
|
|
"flashinfer": "0.2.11",
|
|
"nvidia_cutlass_dsl": "4.5.0",
|
|
},
|
|
)
|
|
profile_b = software_profile(
|
|
python_version="3.12.5",
|
|
torch_version="2.12.1+cu130",
|
|
cuda_version="13.0",
|
|
package_versions={
|
|
"triton": "3.4.9",
|
|
"fastvideo_kernel": "0.3.7",
|
|
"flash_attn_4": "4.0.0.dev1",
|
|
"flashinfer": "0.2.12",
|
|
"nvidia_cutlass_dsl": "4.5.1",
|
|
},
|
|
)
|
|
|
|
assert profile_a == {
|
|
"python": "3.12",
|
|
"pytorch": "2.12",
|
|
"cuda": "13.0",
|
|
"packages": {
|
|
"fastvideo_kernel": "0.3.2",
|
|
"flash_attn_4": "4.0.0.dev0",
|
|
"flashinfer": "0.2.11",
|
|
"nvidia_cutlass_dsl": "4.5.0",
|
|
"triton": "3.4.1",
|
|
},
|
|
}
|
|
assert profile_b["python"] == "3.12"
|
|
assert profile_b["pytorch"] == "2.12"
|
|
assert profile_b["cuda"] == "13.0"
|
|
assert software_profile_id(profile_a) != software_profile_id(profile_b)
|
|
|
|
|
|
def test_installed_package_versions_checks_attention_kernel_distributions(monkeypatch):
|
|
versions = {
|
|
"fastvideo-kernel": "0.3.2",
|
|
"flash-attn": "2.8.1",
|
|
"flash-attn-4": "4.0.0.dev0",
|
|
"flash-attention-fp4": "0.1.0",
|
|
"flashinfer-python": "0.2.11",
|
|
"nvidia-cutlass-dsl": "4.5.0",
|
|
}
|
|
|
|
def fake_version(name):
|
|
if name not in versions:
|
|
raise identity_module.importlib.metadata.PackageNotFoundError(name)
|
|
return versions[name]
|
|
|
|
monkeypatch.setattr(identity_module.importlib.metadata, "version", fake_version)
|
|
|
|
installed = identity_module._installed_package_versions()
|
|
|
|
assert installed["fastvideo_kernel"] == "0.3.2"
|
|
assert installed["flash_attn"] == "2.8.1"
|
|
assert installed["flash_attn_4"] == "4.0.0.dev0"
|
|
assert installed["flash_attention_fp4"] == "0.1.0"
|
|
assert installed["flashinfer"] == "0.2.11"
|
|
assert installed["nvidia_cutlass_dsl"] == "4.5.0"
|
|
|
|
|
|
def test_hardware_profile_id_uses_normalized_gpu_cohort():
|
|
profile = hardware_profile(
|
|
num_gpus=2,
|
|
gpu_devices=[
|
|
{
|
|
"name": "NVIDIA L40S",
|
|
"memory_bytes": 48 * 1024**3,
|
|
"compute_capability": "8.9",
|
|
},
|
|
{
|
|
"name": "NVIDIA L40S",
|
|
"memory_gb": 48,
|
|
"compute_capability": "8.9",
|
|
},
|
|
],
|
|
interconnect="none_or_partial",
|
|
)
|
|
|
|
assert profile == {
|
|
"device_type":
|
|
"cuda",
|
|
"gpu_count":
|
|
2,
|
|
"gpus": [
|
|
{
|
|
"name": "NVIDIA L40S",
|
|
"memory_gb": 48,
|
|
"compute_capability": "8.9",
|
|
},
|
|
{
|
|
"name": "NVIDIA L40S",
|
|
"memory_gb": 48,
|
|
"compute_capability": "8.9",
|
|
},
|
|
],
|
|
"interconnect":
|
|
"none_or_partial",
|
|
}
|
|
assert hardware_profile_id(profile).startswith("hw-")
|
|
|
|
|
|
def test_hardware_profile_pads_missing_requested_cuda_devices(monkeypatch):
|
|
|
|
class Props:
|
|
name = "NVIDIA L40S"
|
|
total_memory = 48 * 1024**3
|
|
|
|
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
|
|
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
|
|
monkeypatch.setattr(torch.cuda, "get_device_properties", lambda device_id: Props())
|
|
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device_id: (8, 9))
|
|
|
|
profile = hardware_profile(num_gpus=2, interconnect="unknown")
|
|
|
|
assert profile["gpu_count"] == 2
|
|
assert profile["gpus"] == [
|
|
{
|
|
"name": "NVIDIA L40S",
|
|
"memory_gb": 48,
|
|
"compute_capability": "8.9",
|
|
},
|
|
{
|
|
"name": "unknown",
|
|
"memory_gb": None,
|
|
"compute_capability": None,
|
|
},
|
|
]
|
|
|
|
|
|
def test_local_path_detection_checks_both_path_separators(monkeypatch):
|
|
monkeypatch.setattr(identity_module.os, "sep", "\\")
|
|
|
|
assert identity_module._looks_like_local_path("models/local/checkpoint") is True
|
|
assert identity_module._looks_like_local_path(r"C:\models\checkpoint") is True
|
|
assert identity_module._looks_like_local_path("Wan-AI/Wan2.1-T2V-1.3B-Diffusers") is False
|
|
|
|
|
|
def test_environment_metadata_is_separate_audit_fingerprint():
|
|
cfg = _benchmark_config()
|
|
recipe_hash = _fingerprint(cfg)
|
|
audit_a = environment_metadata(
|
|
env={"IMAGE_VERSION": "py3.12-cuda13.0.0"},
|
|
package_versions={"triton": "3.4.1"},
|
|
hardware={
|
|
"device_type": "cuda",
|
|
"gpu_count": 2
|
|
},
|
|
software={
|
|
"python": "3.12",
|
|
"pytorch": "2.12",
|
|
"cuda": "13.0"
|
|
},
|
|
)
|
|
audit_b = environment_metadata(
|
|
env={"IMAGE_VERSION": "py3.12-cuda13.0.1"},
|
|
package_versions={"triton": "3.4.1"},
|
|
hardware={
|
|
"device_type": "cuda",
|
|
"gpu_count": 2
|
|
},
|
|
software={
|
|
"python": "3.12",
|
|
"pytorch": "2.12",
|
|
"cuda": "13.0"
|
|
},
|
|
)
|
|
|
|
assert recipe_hash == _fingerprint(cfg)
|
|
assert environment_fingerprint(audit_a) != environment_fingerprint(audit_b)
|
|
|
|
|
|
def test_resolved_revision_from_hf_snapshot_path():
|
|
revision = "a" * 40
|
|
|
|
assert resolved_revision_from_model_path(
|
|
f"/root/.cache/huggingface/hub/models--org--repo/snapshots/{revision}") == revision
|
|
assert resolved_revision_from_model_path("/models/local-repo") is None
|
|
|
|
|
|
def test_runtime_identity_from_generator_summarizes_worker_records():
|
|
from fastvideo.tests.performance.test_inference_performance import _runtime_identity_from_generator
|
|
|
|
revision = "b" * 40
|
|
|
|
class FakeExecutor:
|
|
|
|
def collective_rpc(self, _method):
|
|
return [
|
|
{
|
|
"resolved_attention_backends": ["FLASH_ATTN"],
|
|
"resolved_model_path": f"/root/.cache/huggingface/hub/models--org--repo/snapshots/{revision}",
|
|
},
|
|
{
|
|
"resolved_attention_backends": ["FLASH_ATTN"],
|
|
"resolved_model_path": f"/root/.cache/huggingface/hub/models--org--repo/snapshots/{revision}",
|
|
},
|
|
]
|
|
|
|
class FakeGenerator:
|
|
executor = FakeExecutor()
|
|
|
|
assert _runtime_identity_from_generator(FakeGenerator()) == {
|
|
"resolved_attention_backend": "FLASH_ATTN",
|
|
"resolved_model_revision": revision,
|
|
}
|
|
|
|
|
|
def test_collect_worker_identity_handles_torch_module_pipeline():
|
|
from fastvideo.tests.performance.test_inference_performance import _collect_worker_identity
|
|
|
|
class Backend:
|
|
name = "FLASH_ATTN"
|
|
|
|
class Leaf(torch.nn.Module):
|
|
backend = Backend()
|
|
|
|
class Pipeline(torch.nn.Module):
|
|
model_path = "/models/local"
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.leaf = Leaf()
|
|
|
|
class Worker:
|
|
pipeline = Pipeline()
|
|
|
|
assert _collect_worker_identity(Worker()) == {
|
|
"resolved_attention_backends": ["FLASH_ATTN"],
|
|
"resolved_model_path": "/models/local",
|
|
}
|