76 lines
2.6 KiB
Python
76 lines
2.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""CPU-only contracts for how `fastvideo bench` measures request latency."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from types import SimpleNamespace
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from fastvideo.entrypoints.cli.bench_serving import (
|
|
RequestFuncInput,
|
|
RequestFuncOutput,
|
|
calculate_metrics,
|
|
limited_request,
|
|
)
|
|
|
|
|
|
def _output(latency: float, queue_wait: float, success: bool = True) -> RequestFuncOutput:
|
|
return RequestFuncOutput(success=success, latency=latency, queue_wait=queue_wait)
|
|
|
|
|
|
def test_e2e_latency_includes_the_wait_for_a_concurrency_slot():
|
|
outputs = [_output(latency=1.0, queue_wait=float(k)) for k in range(5)]
|
|
outputs.append(_output(latency=0.1, queue_wait=9.0, success=False))
|
|
|
|
metrics = calculate_metrics(outputs, total_duration=5.0)
|
|
|
|
assert metrics["latency_p99"] == pytest.approx(1.0)
|
|
assert metrics["e2e_latency_mean"] == pytest.approx(3.0)
|
|
assert metrics["e2e_latency_median"] == pytest.approx(3.0)
|
|
assert metrics["e2e_latency_p99"] == pytest.approx(np.percentile([1, 2, 3, 4, 5], 99))
|
|
assert metrics["failed_requests"] == 1
|
|
|
|
|
|
def test_e2e_latency_equals_latency_without_a_wait():
|
|
outputs = [_output(latency=x, queue_wait=0.0) for x in (0.5, 1.5, 2.5)]
|
|
|
|
metrics = calculate_metrics(outputs, total_duration=3.0)
|
|
|
|
for stat in ("mean", "median", "p99"):
|
|
assert metrics[f"e2e_latency_{stat}"] == pytest.approx(metrics[f"latency_{stat}"])
|
|
|
|
|
|
def test_limited_request_counts_the_wait_from_arrival():
|
|
clock = SimpleNamespace(now=0.0)
|
|
started = {"a": asyncio.Event(), "b": asyncio.Event()}
|
|
done = {"a": asyncio.Event(), "b": asyncio.Event()}
|
|
|
|
async def fake_request(req, session, pbar):
|
|
output = RequestFuncOutput(start_time=clock.now)
|
|
started[req.prompt].set()
|
|
await done[req.prompt].wait()
|
|
output.latency = clock.now - output.start_time
|
|
output.success = True
|
|
return output
|
|
|
|
async def run() -> list:
|
|
semaphore = asyncio.Semaphore(1)
|
|
reqs = [RequestFuncInput(prompt=p, api_url="http://unused", model="m") for p in ("a", "b")]
|
|
tasks = [asyncio.create_task(limited_request(fake_request, r, None, None, semaphore, 0.0)) for r in reqs]
|
|
clock.now = 0.5
|
|
await started["a"].wait()
|
|
clock.now = 3.0
|
|
done["a"].set()
|
|
await started["b"].wait()
|
|
clock.now = 5.0
|
|
done["b"].set()
|
|
return await asyncio.gather(*tasks)
|
|
|
|
a, b = asyncio.run(run())
|
|
|
|
assert (a.queue_wait, a.latency) == (0.5, 2.5)
|
|
assert (b.queue_wait, b.latency) == (3.0, 2.0)
|