From dcd2cba956857575dd4d45645ce4e98a533319e2 Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Fri, 10 Jul 2026 19:32:33 -0700 Subject: [PATCH] [bugfix]: coalesce in-flight arrivals when the batching deadline has expired (#1453) When batching_delay_ms > 0, the collect deadline anchors to the first job's enqueue time. Any job that waited behind an in-flight generation arrives with an already-expired deadline and was dispatched as a batch of 1, degrading dynamic batching to sequential under sustained load. Fall through to the same non-blocking greedy drain the delay=0 path uses so queued compatible jobs still coalesce up to batching_max_size. --- fastvideo/entrypoints/openai/batching.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fastvideo/entrypoints/openai/batching.py b/fastvideo/entrypoints/openai/batching.py index 7dc540091..39c269645 100644 --- a/fastvideo/entrypoints/openai/batching.py +++ b/fastvideo/entrypoints/openai/batching.py @@ -89,17 +89,17 @@ class VideoBatchScheduler: deadline = first.enqueue_time + delay_s while len(batch) < max_size: - if delay_s > 0: - timeout = deadline - time.perf_counter() - if timeout <= 0: - break + timeout = deadline - time.perf_counter() + if timeout > 0: try: candidate = await asyncio.wait_for(self._get_next_job(), timeout=timeout) except TimeoutError: break else: - # delay=0 means "don't wait": greedily drain whatever is - # already queued so max_size still coalesces. + # delay=0 means "don't wait", and an already-expired deadline + # means the first job waited behind an in-flight generation: + # in both cases greedily drain whatever is already queued so + # max_size still coalesces instead of dispatching batches of 1. if self._pending: candidate = self._pending.popleft() else: