From d74475ce06ea3a246cb6376bf28db494c23c6b1b Mon Sep 17 00:00:00 2001 From: Ishan Vaish Date: Mon, 31 Aug 2026 19:37:09 -0700 Subject: [PATCH] [feat] Let the OpenAI server plug in a non-CUDA generator --- fastvideo/entrypoints/openai/api_server.py | 12 +++++++++++- fastvideo/entrypoints/openai/serving_engine.py | 8 +++++++- fastvideo/entrypoints/openai/video_api.py | 2 ++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fastvideo/entrypoints/openai/api_server.py b/fastvideo/entrypoints/openai/api_server.py index 81d669e5b..6fa909d08 100644 --- a/fastvideo/entrypoints/openai/api_server.py +++ b/fastvideo/entrypoints/openai/api_server.py @@ -62,6 +62,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: default_request: GenerationRequest | None = getattr(app.state, "default_request", None) logger.info("Initializing %s generation runtime for %s ...", app.state.runtime, args.model_path) + # A non-CUDA runtime (e.g. MLX) supplies its own generator_factory instead + # of the default VideoGenerator/Executor path -- see MLXWanGenerator. factory = app.state.generator_factory generator = factory() if factory is not None else VideoGenerator.from_fastvideo_args(args) serving_engine = OpenAIServingEngine(generator, app.state.video_request_validator) @@ -98,7 +100,13 @@ def create_app( video_request_validator: Callable[[VideoGenerationRequest], None] | None = None, runtime: str = "cuda", ) -> FastAPI: - """Build the FastAPI application with all routers mounted""" + """Build the FastAPI application with all routers mounted. + + ``generator_factory``/``video_request_validator``/``runtime`` let a + non-CUDA backend (MLX) plug in its own generator and request rules + without a separate app-building path. Defaults reproduce the original + CUDA-only behavior exactly. + """ app = FastAPI( title="FastVideo OpenAI-Compatible API", @@ -158,6 +166,8 @@ def create_app( app.include_router(common_router) app.include_router(video_router) + # The MLX runtime only wires video-with-audio generation (see + # fastvideo/mlx_runtime/); image generation has no MLX backend yet. if runtime != "mlx": app.include_router(image_router) app.include_router(playground_router) diff --git a/fastvideo/entrypoints/openai/serving_engine.py b/fastvideo/entrypoints/openai/serving_engine.py index e01b1c99d..8d619b649 100644 --- a/fastvideo/entrypoints/openai/serving_engine.py +++ b/fastvideo/entrypoints/openai/serving_engine.py @@ -14,6 +14,11 @@ _T = TypeVar("_T") class ServingGenerator(Protocol): + """The minimal shape OpenAIServingEngine needs from a generator. + + VideoGenerator (CUDA/mp/ray) satisfies this structurally already; an MLX + generator (e.g. MLXWanGenerator) implements just these two methods. + """ def generate(self, request: GenerationRequest) -> Any: ... @@ -47,6 +52,7 @@ class OpenAIServingEngine: return self._generator def validate_video_request(self, request: VideoGenerationRequest) -> None: + """Runtime-specific request checks (e.g. MLX's supported-field allowlist).""" if self._video_request_validator is not None: self._video_request_validator(request) @@ -143,4 +149,4 @@ class OpenAIServingEngine: await asyncio.to_thread(self._generator.shutdown) -__all__ = ["OpenAIServingEngine"] +__all__ = ["OpenAIServingEngine", "ServingGenerator"] diff --git a/fastvideo/entrypoints/openai/video_api.py b/fastvideo/entrypoints/openai/video_api.py index 5c1c7a235..cdc83e9a8 100644 --- a/fastvideo/entrypoints/openai/video_api.py +++ b/fastvideo/entrypoints/openai/video_api.py @@ -331,6 +331,8 @@ async def _parse_video_request(raw_request: Request) -> VideoGenerationRequest: async def _adapt_request(request_id: str, request: VideoGenerationRequest) -> GenerationRequest: try: + # Runtime-specific checks (e.g. MLX's supported-field allowlist) run + # before the CUDA-oriented model/LoRA validation below. get_serving_engine().validate_video_request(request) validate_model_and_lora(request, get_server_args(), get_served_model_name()) await prepare_reference_media(request_id, request, get_output_dir())