[feat] Align multimodal OpenAI serving APIs (#1781)
This commit is contained in:
William Lin
@@ -21,6 +21,8 @@ surfaces:
|
||||
hsdp_shard_dim: generator.engine.parallelism.hsdp_shard_dim
|
||||
dist_timeout: generator.engine.parallelism.dist_timeout
|
||||
lora_path: generator.pipeline.components.lora_path
|
||||
lora_nickname: generator.pipeline.components.lora_nickname
|
||||
lora_strength: generator.pipeline.components.lora_strength
|
||||
dit_cpu_offload: generator.engine.offload.dit
|
||||
use_fsdp_inference: generator.engine.use_fsdp_inference
|
||||
dit_layerwise_offload: generator.engine.offload.dit_layerwise
|
||||
@@ -72,7 +74,6 @@ surfaces:
|
||||
compatibility_only:
|
||||
mode: "Legacy multi-mode FastVideoArgs switch; typed inference config should not expose execution mode."
|
||||
inference_mode: "Legacy boolean mirror of mode; kept only through adapters while FastVideoArgs remains."
|
||||
lora_nickname: "Legacy adapter-selection surface pending LoRA API cleanup."
|
||||
lora_target_modules: "Legacy LoRA configuration surface pending dedicated component API."
|
||||
output_type: "Legacy output formatting surface pending GenerationResult cleanup."
|
||||
VSA_sparsity: "Model-specific inference optimization not yet represented in the typed public schema."
|
||||
@@ -590,15 +591,31 @@ surfaces:
|
||||
openai_video_request:
|
||||
kept:
|
||||
model: "HTTP adapter model-routing field."
|
||||
user: "OpenAI-compatible caller tracking field."
|
||||
task: "SGLang-compatible MiniMax-H3 task selector validated against the startup pipeline."
|
||||
quality: "vLLM-Omni-compatible quality-intent field; model-specific."
|
||||
lora: "vLLM-Omni-compatible selector for the adapter fixed at server startup."
|
||||
moved:
|
||||
prompt: request.prompt
|
||||
input_reference: request.inputs.image_path
|
||||
reference_url: request.inputs.image_path
|
||||
image_reference: request.inputs.image_path,last_image,references
|
||||
video_reference: request.inputs.video_path,references
|
||||
audio_reference: request.inputs.references
|
||||
video_path: request.inputs.video_path
|
||||
video_url: request.inputs.video_path
|
||||
video_params: request.sampling.width,height,num_frames,fps
|
||||
size:
|
||||
target: request.sampling.width,height
|
||||
note: "Adapter parses OpenAI size strings as WIDTHxHEIGHT and forwards width then height."
|
||||
width: request.sampling.width
|
||||
height: request.sampling.height
|
||||
fps: request.sampling.fps
|
||||
num_frames: request.sampling.num_frames
|
||||
aspect_ratio: request.sampling.width,height
|
||||
short_edge: request.sampling.width,height
|
||||
num_outputs_per_prompt: request.sampling.num_videos_per_prompt
|
||||
n: request.sampling.num_videos_per_prompt
|
||||
seed: request.sampling.seed
|
||||
num_inference_steps: request.sampling.num_inference_steps
|
||||
guidance_scale: request.sampling.guidance_scale
|
||||
@@ -606,11 +623,22 @@ surfaces:
|
||||
true_cfg_scale: request.sampling.true_cfg_scale
|
||||
negative_prompt: request.negative_prompt
|
||||
enable_teacache: request.runtime.enable_teacache
|
||||
max_sequence_length: request.sampling.max_sequence_length
|
||||
boundary_ratio: request.sampling.boundary_ratio
|
||||
extra_params: request.extensions
|
||||
output_path: request.output.output_path
|
||||
compatibility_only:
|
||||
seconds:
|
||||
target: request.sampling.num_frames
|
||||
note: "HTTP adapter duration convenience field. If num_frames is omitted, the adapter computes num_frames = fps * seconds."
|
||||
start_time_seconds: "vLLM-Omni reference-video offset; rejected by pipelines that cannot represent it."
|
||||
flow_shift: "vLLM-Omni request field; accepted only when the selected model exposes a matching request parameter."
|
||||
generate_sound: "vLLM-Omni audio-output intent; accepted only by models with a matching request parameter."
|
||||
sound_duration: "vLLM-Omni audio-duration intent; accepted only by models with a matching request parameter."
|
||||
enable_frame_interpolation: "vLLM-Omni post-processing field; unavailable until FastVideo exposes a frame-interpolation stage."
|
||||
frame_interpolation_exp: "vLLM-Omni post-processing field; unavailable until FastVideo exposes a frame-interpolation stage."
|
||||
frame_interpolation_scale: "vLLM-Omni post-processing field; unavailable until FastVideo exposes a frame-interpolation stage."
|
||||
frame_interpolation_model_path: "vLLM-Omni post-processing field; unavailable until FastVideo exposes a frame-interpolation stage."
|
||||
|
||||
cli:
|
||||
notes:
|
||||
|
||||
@@ -1,116 +1,158 @@
|
||||
# OpenAI-compatible HTTP Contract
|
||||
# OpenAI-compatible HTTP contract
|
||||
|
||||
The stateless FastVideo HTTP server lives at
|
||||
[`fastvideo/entrypoints/openai/`](https://github.com/hao-ai-lab/FastVideo/tree/main/fastvideo/entrypoints/openai).
|
||||
Launch: `fastvideo serve --config serve.yaml`.
|
||||
FastVideo exposes one model-agnostic REST engine for image and video models.
|
||||
Launch it from a typed serve config:
|
||||
|
||||
```bash
|
||||
fastvideo serve --config examples/serving/openai_fasth3.yaml
|
||||
```
|
||||
|
||||
All generation routes share one serialized engine. FastVideo pipelines mutate
|
||||
per-request sampling state, and some adapters merge weights at load time, so a
|
||||
single loaded pipeline is never entered concurrently by image and video
|
||||
requests. HTTP handling and job polling remain asynchronous.
|
||||
|
||||
## Endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
| --- | --- | --- |
|
||||
| `POST` | `/v1/videos/generations` | Synchronous video generation |
|
||||
| `GET` | `/v1/videos` | List prior jobs held in the in-memory store |
|
||||
| `GET` | `/v1/videos/{id}` | Job status / result |
|
||||
| `GET` | `/v1/videos/{id}/content` | Download the MP4 once ready |
|
||||
| `POST` | `/v1/images/generations` | Synchronous image generation |
|
||||
| `GET` | `/v1/models` | Enumerate registered models |
|
||||
| `GET` | `/v1/models` | List the served model and optional startup adapter |
|
||||
| `GET` | `/v1/models/{model}` | Retrieve one served model card |
|
||||
| `POST` | `/v1/videos` | Submit an asynchronous video job |
|
||||
| `POST` | `/v1/videos/sync` | Generate and return an MP4 response directly |
|
||||
| `GET` | `/v1/videos` | List in-memory jobs with `after`, `limit`, and `order` |
|
||||
| `GET` | `/v1/videos/{id}` | Retrieve job status and metadata |
|
||||
| `GET` | `/v1/videos/{id}/content` | Download a completed MP4 |
|
||||
| `DELETE` | `/v1/videos/{id}` | Delete a job and its completed artifact |
|
||||
| `POST` | `/v1/images` | Generate an image |
|
||||
| `POST` | `/v1/images/edits` | Generate an image from image references |
|
||||
| `GET` | `/v1/images/{id}/content` | Download a generated image |
|
||||
| `GET` | `/health` | Liveness probe |
|
||||
|
||||
## `VideoGenerationsRequest` shape
|
||||
`POST /v1/videos/generations` remains an alias for older FastVideo clients.
|
||||
|
||||
Mirrors the OpenAI `POST /v1/videos/generations` shape:
|
||||
## Video requests
|
||||
|
||||
The canonical shape follows vLLM-Omni and accepts SGLang's common flat
|
||||
extensions. Fields that FastVideo cannot represent for the loaded model fail
|
||||
at admission with HTTP 400 instead of creating a job that later fails.
|
||||
|
||||
```json
|
||||
{
|
||||
"prompt": "a fox running through snow",
|
||||
"size": "1024x1536",
|
||||
"seconds": 5,
|
||||
"fps": 24,
|
||||
"num_frames": 121,
|
||||
"model": "fasth3",
|
||||
"prompt": "A fox runs through fresh snow.",
|
||||
"seconds": "5",
|
||||
"size": "1344x768",
|
||||
"video_params": {
|
||||
"fps": 24,
|
||||
"num_frames": 124
|
||||
},
|
||||
"seed": 42,
|
||||
"num_inference_steps": 8,
|
||||
"num_inference_steps": 5,
|
||||
"guidance_scale": 1.0,
|
||||
"negative_prompt": "blurry, low quality",
|
||||
"input_reference": "/path/to/init.png"
|
||||
}
|
||||
```
|
||||
|
||||
SGLang-compatible extensions carried today:
|
||||
`num_inference_steps`, `guidance_scale`, `guidance_scale_2`,
|
||||
`true_cfg_scale`, `negative_prompt`, `enable_teacache`, `output_path`.
|
||||
|
||||
## Merge precedence
|
||||
|
||||
The server builds a `GenerationRequest` each call using three layers,
|
||||
highest first:
|
||||
|
||||
1. **Request body (client-explicit)** — only fields carried in
|
||||
`request.model_fields_set` (Pydantic v2). Unset fields do not count,
|
||||
even if the Pydantic model has a schema default for them.
|
||||
2. **`ServeConfig.default_request` (operator-explicit)** — projected via
|
||||
[`explicit_request_updates()`](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/api/compat.py);
|
||||
only fields the operator actually wrote into the YAML count as
|
||||
defaults. Every other field inherits the schema default rather than
|
||||
being pinned.
|
||||
3. **Hardcoded fallback** — e.g. `fps = 24`.
|
||||
|
||||
The gate matters: both surfaces carry schema defaults. Without
|
||||
`model_fields_set` / explicit-path tracking, schema defaults would
|
||||
masquerade as intent and silently shadow the other side.
|
||||
|
||||
See [`video_api.py::_build_generation_kwargs`](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/entrypoints/openai/video_api.py)
|
||||
for the canonical implementation; the per-request assembly lives there,
|
||||
not in pipeline code.
|
||||
|
||||
## Continuation state
|
||||
|
||||
The stateless surface accepts an opaque `ContinuationState` round-trip.
|
||||
Clients that want continuation pass the prior `state` blob back on the
|
||||
next request, and receive a new one on the response when
|
||||
`request.output.return_state = true`.
|
||||
|
||||
Shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"state": {
|
||||
"kind": "ltx2.v1",
|
||||
"payload": { "schema_version": 1, "segment_index": 3, ... }
|
||||
"image_reference": [
|
||||
{"image_url": "https://example.com/first-frame.png"}
|
||||
],
|
||||
"extra_params": {
|
||||
"vsa_mode": "exempt"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Payload is always JSON-serializable. Large tensors may live in an
|
||||
opaque blob-store reference the client simply round-trips; see
|
||||
[`LTX2ContinuationState`](https://github.com/hao-ai-lab/FastVideo/blob/main/fastvideo/pipelines/basic/ltx2/continuation.py).
|
||||
Resolution precedence matches vLLM-Omni:
|
||||
|
||||
Continuation is not yet wired all the way through to
|
||||
`generator.generate_video(...)` — PR 7.6 (GPU pool upstream) is the
|
||||
pipeline-level consumer. PR 7 locked the envelope so this surface is
|
||||
stable ahead of that plumbing.
|
||||
1. `size`
|
||||
2. top-level `width` and `height`
|
||||
3. `video_params.width` and `video_params.height`
|
||||
|
||||
## Error codes
|
||||
Top-level `fps` and `num_frames` similarly take precedence over the nested
|
||||
block. If `num_frames` is absent, `seconds * fps` is used. FastVideo also keeps
|
||||
the legacy `input_reference`, `reference_url`, `video_path`, and `video_url`
|
||||
spellings.
|
||||
|
||||
| HTTP | Condition |
|
||||
| --- | --- |
|
||||
| `400 Bad Request` | Parse/validation failure (unknown field, type mismatch, incompatible preset/state) |
|
||||
| `404 Not Found` | `GET /v1/videos/{id}` for an unknown job |
|
||||
| `409 Conflict` | Job id already exists |
|
||||
| `500 Internal Server Error` | Pipeline raised; body mirrors upstream OpenAI error envelope |
|
||||
| `503 Service Unavailable` | No generator loaded, or shutdown in progress |
|
||||
Reference objects support URL or local-path strings through `image_url`,
|
||||
`video_url`, and `audio_url`. `file_id` references are schema-compatible but
|
||||
return HTTP 400 because FastVideo does not provide an OpenAI Files store.
|
||||
Multipart `input_reference` uploads are saved under the configured output
|
||||
directory.
|
||||
|
||||
Errors include a JSON body with
|
||||
`{"error": {"type": "...", "message": "..."}}` matching the OpenAI
|
||||
Python SDK's expectation.
|
||||
## Jobs and synchronous responses
|
||||
|
||||
## What does not cross this boundary
|
||||
An asynchronous submission returns a `video` object in `queued` state. Its
|
||||
status advances through `in_progress` to `completed` or `failed`. Completed
|
||||
jobs expose `file_name`, the FastVideo compatibility extension `file_path`,
|
||||
timings, and peak-memory metadata when the pipeline reports them.
|
||||
|
||||
* Flat legacy kwargs (`ltx2_refine_enabled`, `torch_compile_kwargs`,
|
||||
etc.) — these are init-time, configured via `ServeConfig.generator`,
|
||||
never per-request.
|
||||
* Private Dreamverse-only fields — those live in a private adapter on
|
||||
the Dreamverse side; the public FastVideo surface never promises
|
||||
backward compatibility for them.
|
||||
* Raw tensor payloads (`ltx2_audio_clean_latent` et al.) — these are
|
||||
derived by the pipeline from `ContinuationState`, never shipped as
|
||||
request fields.
|
||||
`POST /v1/videos/sync` returns `video/mp4` bytes. It includes
|
||||
`X-Request-Id`, `X-Model`, `X-Inference-Time-S`, `X-Stage-Durations`, and
|
||||
`X-Peak-Memory-MB` headers.
|
||||
|
||||
FastVideo's synchronous CUDA execution cannot be interrupted after launch.
|
||||
Deleting an in-progress resource removes it from the API immediately; the
|
||||
engine remains serialized until the call exits and then removes any artifact.
|
||||
|
||||
## Model and LoRA selection
|
||||
|
||||
`server.served_model_name` controls the public model id. If omitted, the
|
||||
checkpoint path is used. Requests that name another model fail with HTTP 400.
|
||||
|
||||
LoRAs are configured under
|
||||
`generator.pipeline.components.{lora_path,lora_nickname,lora_strength}`. The
|
||||
startup adapter appears in `/v1/models`, and requests can use either its model
|
||||
nickname or a vLLM-Omni selector:
|
||||
|
||||
```json
|
||||
{
|
||||
"prompt": "A fox runs through fresh snow.",
|
||||
"model": "fasth3-dense-datafree",
|
||||
"lora": {
|
||||
"name": "fasth3-dense-datafree",
|
||||
"path": "/models/adapter_model.safetensors",
|
||||
"scale": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The selector must match the adapter already loaded at startup. FastH3 adapter
|
||||
files can contain dense replacement tensors and VSA gates in addition to
|
||||
low-rank factors, so swapping them inside concurrent requests would corrupt
|
||||
shared pipeline state. A mismatch is rejected with HTTP 400.
|
||||
|
||||
## MiniMax-H3 and FastH3
|
||||
|
||||
FastH3 uses the same general routes and adapter. `task` is accepted for
|
||||
SGLang-compatible H3 clients:
|
||||
|
||||
- `t2va` uses text only.
|
||||
- `fl2va` takes one or two image references.
|
||||
- `ref2va` takes ordered image, video, and audio references and requires a
|
||||
server started with `MiniMaxH3Ref2VAModularPipeline`.
|
||||
|
||||
The released FastH3 pipeline generates one packed video/audio result per
|
||||
request, uses 24 fps, requires guidance scale 1, and accepts frame counts on
|
||||
its causal-VAE grid. The serving examples pin its five-point distilled sigma
|
||||
schedule (four DiT forwards).
|
||||
|
||||
## Defaults and errors
|
||||
|
||||
Incoming explicit fields override operator-explicit `default_request` fields,
|
||||
which override model preset defaults. Pydantic defaults do not masquerade as
|
||||
client intent; the transport uses `model_fields_set`, while typed config parsing
|
||||
tracks the exact paths written by the operator.
|
||||
|
||||
Errors use the OpenAI envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"message": "...",
|
||||
"type": "invalid_request_error",
|
||||
"param": null,
|
||||
"code": 400
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Parse, model-selection, startup-LoRA, and unsupported-parameter failures are
|
||||
HTTP 400; missing resources are HTTP 404; generation failures are stored on
|
||||
asynchronous jobs and returned as HTTP 500 when that job is retrieved.
|
||||
|
||||
Reference in New Issue
Block a user