427 lines
19 KiB
Python
427 lines
19 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""Translate OpenAI/vLLM-Omni requests into FastVideo's typed request API."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import math
|
|
import os
|
|
from typing import Any
|
|
|
|
from fastvideo.api.compat import (
|
|
REQUEST_BATCH_EXTRA_PASSTHROUGH_FIELDS,
|
|
explicit_request_updates,
|
|
legacy_generate_call_to_request,
|
|
request_to_sampling_param,
|
|
)
|
|
from fastvideo.api.schema import GenerationRequest
|
|
from fastvideo.entrypoints.openai.protocol import (
|
|
FileImageReference,
|
|
FileVideoReference,
|
|
UrlImageReference,
|
|
VideoGenerationRequest,
|
|
)
|
|
from fastvideo.entrypoints.openai.utils import save_image_to_path
|
|
from fastvideo.fastvideo_args import FastVideoArgs
|
|
from fastvideo.models.vision_utils import load_image
|
|
from fastvideo.registry import get_preset_selection
|
|
|
|
|
|
class RequestAdaptationError(ValueError):
|
|
"""The transport request cannot be represented by the loaded pipeline."""
|
|
|
|
|
|
def _as_list(value: Any | list[Any] | None) -> list[Any]:
|
|
if value is None:
|
|
return []
|
|
return value if isinstance(value, list) else [value]
|
|
|
|
|
|
def _image_sources(request: VideoGenerationRequest) -> list[str]:
|
|
sources: list[str] = []
|
|
for reference in _as_list(request.image_reference):
|
|
if isinstance(reference, FileImageReference):
|
|
raise RequestAdaptationError("file_id image references are not supported; provide image_url instead")
|
|
sources.append(reference.image_url)
|
|
legacy_sources = [source for source in (request.input_reference, request.reference_url) if source]
|
|
if len(legacy_sources) > 1:
|
|
raise RequestAdaptationError("Provide only one of input_reference or reference_url.")
|
|
legacy = legacy_sources[0] if legacy_sources else None
|
|
if legacy is not None:
|
|
if sources:
|
|
raise RequestAdaptationError("Provide only one of input_reference/reference_url or image_reference.")
|
|
sources.append(legacy)
|
|
return sources
|
|
|
|
|
|
async def prepare_reference_media(
|
|
request_id: str,
|
|
request: VideoGenerationRequest,
|
|
output_dir: str,
|
|
) -> None:
|
|
"""Materialize and decode image references before a job reaches workers."""
|
|
_image_sources(request)
|
|
uploads_dir = os.path.join(os.path.abspath(output_dir), "uploads")
|
|
|
|
async def materialize(source: str, index: int) -> str:
|
|
source = source.strip()
|
|
if not source:
|
|
raise RequestAdaptationError("Image references must not be empty.")
|
|
if source.lower().startswith(("http://", "https://", "data:image")):
|
|
target = os.path.join(uploads_dir, f"{request_id}_{index}")
|
|
try:
|
|
local_path = await save_image_to_path(source, target)
|
|
except Exception as error:
|
|
raise RequestAdaptationError(f"Unable to fetch image reference: {error}") from error
|
|
else:
|
|
local_path = os.path.abspath(os.path.expanduser(source))
|
|
if not os.path.isfile(local_path):
|
|
raise RequestAdaptationError(f"Image reference does not exist: {source}")
|
|
try:
|
|
await asyncio.to_thread(load_image, local_path)
|
|
except Exception as error:
|
|
raise RequestAdaptationError(f"Unable to decode image reference: {error}") from error
|
|
return local_path
|
|
|
|
index = 0
|
|
references = _as_list(request.image_reference)
|
|
for reference in references:
|
|
if isinstance(reference, UrlImageReference):
|
|
reference.image_url = await materialize(reference.image_url, index)
|
|
index += 1
|
|
if request.input_reference:
|
|
request.input_reference = await materialize(request.input_reference, index)
|
|
index += 1
|
|
elif request.input_reference == "":
|
|
request.input_reference = None
|
|
if request.reference_url:
|
|
request.reference_url = await materialize(request.reference_url, index)
|
|
elif request.reference_url == "":
|
|
request.reference_url = None
|
|
|
|
|
|
def _video_sources(request: VideoGenerationRequest) -> list[str]:
|
|
sources: list[str] = []
|
|
for reference in _as_list(request.video_reference):
|
|
if isinstance(reference, FileVideoReference):
|
|
raise RequestAdaptationError("file_id video references are not supported; provide video_url instead")
|
|
sources.append(reference.video_url)
|
|
direct = request.video_path or request.video_url
|
|
if direct is not None:
|
|
if sources:
|
|
raise RequestAdaptationError("Provide only one of video_reference or video_path/video_url.")
|
|
sources.append(direct)
|
|
return sources
|
|
|
|
|
|
def _audio_sources(request: VideoGenerationRequest) -> list[str]:
|
|
return [reference.audio_url for reference in _as_list(request.audio_reference)]
|
|
|
|
|
|
def _parse_aspect_ratio(value: str) -> tuple[float, float]:
|
|
try:
|
|
width, height = value.split(":", 1)
|
|
result = float(width), float(height)
|
|
except (AttributeError, TypeError, ValueError) as error:
|
|
raise RequestAdaptationError(f"Invalid aspect_ratio {value!r}; expected WIDTH:HEIGHT") from error
|
|
if not all(math.isfinite(term) for term in result) or result[0] <= 0 or result[1] <= 0:
|
|
raise RequestAdaptationError(f"Invalid aspect_ratio {value!r}; both terms must be positive")
|
|
return result
|
|
|
|
|
|
def _apply_aspect_ratio(
|
|
kwargs: dict[str, Any],
|
|
request: VideoGenerationRequest,
|
|
*,
|
|
model_family: str | None,
|
|
) -> None:
|
|
if request.aspect_ratio is None:
|
|
return
|
|
aspect_width, aspect_height = _parse_aspect_ratio(request.aspect_ratio)
|
|
if model_family == "minimax_h3":
|
|
from fastvideo.pipelines.basic.minimax_h3.packing import MINIMAX_H3_SHORT_EDGE, resolve_canvas_size
|
|
|
|
if request.short_edge is not None and request.short_edge != MINIMAX_H3_SHORT_EDGE:
|
|
raise RequestAdaptationError(
|
|
f"MiniMax-H3 currently uses a fixed short_edge={MINIMAX_H3_SHORT_EDGE}, got {request.short_edge}.")
|
|
try:
|
|
height, width = resolve_canvas_size(aspect_width, aspect_height)
|
|
except ValueError as error:
|
|
raise RequestAdaptationError(str(error)) from error
|
|
elif request.short_edge is not None:
|
|
if aspect_width >= aspect_height:
|
|
height = request.short_edge
|
|
width = round(request.short_edge * aspect_width / aspect_height)
|
|
else:
|
|
width = request.short_edge
|
|
height = round(request.short_edge * aspect_height / aspect_width)
|
|
else:
|
|
return
|
|
kwargs["width"], kwargs["height"] = width, height
|
|
|
|
|
|
def validate_model_and_lora(
|
|
request: VideoGenerationRequest,
|
|
args: FastVideoArgs,
|
|
served_model_name: str,
|
|
) -> None:
|
|
"""Validate vLLM-style model and LoRA selectors against startup state.
|
|
|
|
FastVideo's published FastH3 adapters include dense replacement tensors in
|
|
addition to low-rank factors. Those tensors are applied while the model is
|
|
loaded and cannot be swapped safely between concurrent requests. The API
|
|
accepts vLLM's selector shape, but it must identify the startup adapter.
|
|
"""
|
|
allowed_models = {args.lora_nickname} if args.lora_path else {served_model_name}
|
|
if request.model is not None and request.model not in allowed_models:
|
|
choices = ", ".join(sorted(allowed_models))
|
|
raise RequestAdaptationError(
|
|
f"Model mismatch: request specifies {request.model!r}; this server provides {choices}.")
|
|
|
|
if request.lora is None:
|
|
return
|
|
if not args.lora_path:
|
|
raise RequestAdaptationError(
|
|
"This server has no startup LoRA. Configure generator.pipeline.components.lora_path before using "
|
|
"the request lora selector.")
|
|
|
|
body = request.lora
|
|
name = next((body[key] for key in ("name", "lora_name", "adapter") if body.get(key) is not None), None)
|
|
path = next((body[key] for key in ("path", "lora_path", "local_path") if body.get(key) is not None), None)
|
|
scale = next((body[key] for key in ("scale", "lora_scale") if body.get(key) is not None), None)
|
|
if name is None and path is None:
|
|
raise RequestAdaptationError("lora must provide a name or path")
|
|
if name is not None and str(name) != args.lora_nickname:
|
|
raise RequestAdaptationError(f"Requested LoRA {name!r} is not the startup adapter {args.lora_nickname!r}.")
|
|
if path is not None and str(path) != args.lora_path:
|
|
raise RequestAdaptationError(
|
|
f"Requested LoRA path {path!r} does not match the startup adapter {args.lora_path!r}.")
|
|
if scale is not None:
|
|
try:
|
|
scale_value = float(scale)
|
|
except (TypeError, ValueError) as error:
|
|
raise RequestAdaptationError(f"Invalid LoRA scale {scale!r}") from error
|
|
if not math.isclose(scale_value, args.lora_strength, rel_tol=0.0, abs_tol=1e-8):
|
|
raise RequestAdaptationError(
|
|
f"Requested LoRA scale {scale_value:g} does not match startup strength {args.lora_strength:g}.")
|
|
|
|
|
|
def _apply_reference_inputs(
|
|
kwargs: dict[str, Any],
|
|
request: VideoGenerationRequest,
|
|
args: FastVideoArgs,
|
|
*,
|
|
model_family: str | None,
|
|
) -> None:
|
|
images = _image_sources(request)
|
|
videos = _video_sources(request)
|
|
audios = _audio_sources(request)
|
|
ref2va = model_family == "minimax_h3" and "ref2va" in (args.override_pipeline_cls_name or "").lower()
|
|
|
|
if model_family == "minimax_h3" and request.task is not None:
|
|
normalized_task = request.task.lower()
|
|
if normalized_task not in {"t2va", "fl2va", "ref2va"}:
|
|
raise RequestAdaptationError("MiniMax-H3 task must be one of t2va, fl2va, or ref2va.")
|
|
if normalized_task == "ref2va" and not ref2va:
|
|
raise RequestAdaptationError(
|
|
"MiniMax-H3 task='ref2va' requires MiniMaxH3Ref2VAModularPipeline at server startup.")
|
|
if normalized_task != "ref2va" and ref2va:
|
|
raise RequestAdaptationError(
|
|
f"This server is configured for MiniMax-H3 Ref2VA, not task={normalized_task!r}.")
|
|
if normalized_task == "t2va" and (images or videos or audios):
|
|
raise RequestAdaptationError("MiniMax-H3 task='t2va' does not accept reference media.")
|
|
if normalized_task == "fl2va" and not images:
|
|
raise RequestAdaptationError("MiniMax-H3 task='fl2va' requires one or two image references.")
|
|
|
|
if ref2va:
|
|
from fastvideo.pipelines.basic.minimax_h3 import MiniMaxH3Reference
|
|
from fastvideo.pipelines.basic.minimax_h3.reference import validate_references
|
|
|
|
references = [MiniMaxH3Reference(source=source, media_type="image") for source in images]
|
|
references.extend(MiniMaxH3Reference(source=source, media_type="video") for source in videos)
|
|
references.extend(MiniMaxH3Reference(source=source, media_type="audio") for source in audios)
|
|
if references:
|
|
try:
|
|
kwargs["references"] = validate_references(references)
|
|
except (TypeError, ValueError) as error:
|
|
raise RequestAdaptationError(str(error)) from error
|
|
return
|
|
|
|
if request.task is not None and model_family != "minimax_h3":
|
|
raise RequestAdaptationError("The task selector is only defined for MiniMax-H3 servers.")
|
|
|
|
if model_family == "minimax_h3" and (videos or audios):
|
|
raise RequestAdaptationError("MiniMax-H3 video/audio references require a server configured with "
|
|
"override_pipeline_cls_name=MiniMaxH3Ref2VAModularPipeline.")
|
|
if len(images) > 2:
|
|
raise RequestAdaptationError("The loaded pipeline accepts at most first and last image references.")
|
|
if images:
|
|
kwargs["image_path"] = images[0]
|
|
if len(images) == 2:
|
|
kwargs["last_image"] = load_image(images[1])
|
|
if len(videos) > 1:
|
|
raise RequestAdaptationError("The loaded pipeline accepts at most one video reference.")
|
|
if videos:
|
|
kwargs["video_path"] = videos[0]
|
|
if audios:
|
|
raise RequestAdaptationError("The loaded pipeline does not accept audio reference inputs.")
|
|
|
|
|
|
def build_generation_request(
|
|
request_id: str,
|
|
request: VideoGenerationRequest,
|
|
args: FastVideoArgs,
|
|
*,
|
|
served_model_name: str,
|
|
output_dir: str,
|
|
default_request: GenerationRequest | None = None,
|
|
) -> GenerationRequest:
|
|
"""Build one tracked FastVideo request using explicit-field precedence."""
|
|
validate_model_and_lora(request, args, served_model_name)
|
|
kwargs: dict[str, Any] = {}
|
|
if default_request is not None:
|
|
kwargs.update(explicit_request_updates(default_request))
|
|
|
|
body_set = request.model_fields_set
|
|
nested_set = request.video_params.model_fields_set if request.video_params is not None else set()
|
|
if "size" in body_set and request.size is not None:
|
|
width, height = request.size.split("x", 1)
|
|
kwargs["width"], kwargs["height"] = int(width), int(height)
|
|
else:
|
|
if "width" in body_set and request.width is not None:
|
|
kwargs["width"] = request.width
|
|
elif "video_params" in body_set and "width" in nested_set and request.video_params.width is not None:
|
|
kwargs["width"] = request.video_params.width
|
|
if "height" in body_set and request.height is not None:
|
|
kwargs["height"] = request.height
|
|
elif "video_params" in body_set and "height" in nested_set and request.video_params.height is not None:
|
|
kwargs["height"] = request.video_params.height
|
|
|
|
fps_explicit = ("fps" in body_set
|
|
and request.fps is not None) or ("video_params" in body_set and "fps" in nested_set
|
|
and request.video_params.fps is not None)
|
|
if fps_explicit:
|
|
fps = request.fps if "fps" in body_set else request.video_params.fps
|
|
if fps is not None:
|
|
kwargs["fps"] = fps
|
|
kwargs.setdefault("fps", 24)
|
|
|
|
frames_explicit = ("num_frames" in body_set
|
|
and request.num_frames is not None) or ("video_params" in body_set and "num_frames" in nested_set
|
|
and request.video_params.num_frames is not None)
|
|
if frames_explicit:
|
|
num_frames = request.num_frames if "num_frames" in body_set else request.video_params.num_frames
|
|
if num_frames is not None:
|
|
kwargs["num_frames"] = num_frames
|
|
elif "seconds" in body_set and request.seconds is not None:
|
|
kwargs["num_frames"] = int(request.seconds) * int(kwargs["fps"])
|
|
|
|
direct_fields = (
|
|
"seed",
|
|
"num_inference_steps",
|
|
"guidance_scale",
|
|
"guidance_scale_2",
|
|
"true_cfg_scale",
|
|
"negative_prompt",
|
|
"enable_teacache",
|
|
"max_sequence_length",
|
|
"boundary_ratio",
|
|
)
|
|
for name in direct_fields:
|
|
if name in body_set:
|
|
value = getattr(request, name)
|
|
if value is not None:
|
|
kwargs[name] = value
|
|
if "n" in body_set or "num_outputs_per_prompt" in body_set:
|
|
kwargs["num_videos_per_prompt"] = request.resolved_num_outputs
|
|
|
|
try:
|
|
_, model_family = get_preset_selection(args.model_path)
|
|
except (RuntimeError, ValueError):
|
|
model_family = None
|
|
if request.resolved_num_outputs != 1:
|
|
raise RequestAdaptationError("FastVideo serving currently supports exactly one video output per request.")
|
|
if "short_edge" in body_set and request.short_edge is not None and request.aspect_ratio is None:
|
|
raise RequestAdaptationError("short_edge requires aspect_ratio.")
|
|
_apply_aspect_ratio(kwargs, request, model_family=model_family)
|
|
_apply_reference_inputs(kwargs, request, args, model_family=model_family)
|
|
|
|
extension_fields = ("flow_shift", "sound_duration", "start_time_seconds")
|
|
for name in extension_fields:
|
|
if name in body_set and getattr(request, name) is not None:
|
|
kwargs[name] = getattr(request, name)
|
|
if "generate_sound" in body_set and request.generate_sound and model_family != "minimax_h3":
|
|
kwargs["generate_sound"] = True
|
|
if "enable_frame_interpolation" in body_set and request.enable_frame_interpolation:
|
|
kwargs["enable_frame_interpolation"] = True
|
|
for name in (
|
|
"frame_interpolation_exp",
|
|
"frame_interpolation_scale",
|
|
"frame_interpolation_model_path",
|
|
):
|
|
kwargs[name] = getattr(request, name)
|
|
if request.extra_params:
|
|
unknown_extra_params = sorted(set(request.extra_params) - set(REQUEST_BATCH_EXTRA_PASSTHROUGH_FIELDS))
|
|
if unknown_extra_params:
|
|
raise RequestAdaptationError("Unsupported extra_params fields: " + ", ".join(unknown_extra_params))
|
|
kwargs.update(request.extra_params)
|
|
|
|
width = kwargs.get("width")
|
|
height = kwargs.get("height")
|
|
if width is not None and (not isinstance(width, int) or width <= 0):
|
|
raise RequestAdaptationError(f"width must be a positive integer, got {width!r}")
|
|
if height is not None and (not isinstance(height, int) or height <= 0):
|
|
raise RequestAdaptationError(f"height must be a positive integer, got {height!r}")
|
|
|
|
if model_family == "minimax_h3":
|
|
from fastvideo.pipelines.basic.minimax_h3.packing import (
|
|
MINIMAX_H3_CANVAS_MULTIPLE,
|
|
MINIMAX_H3_MAX_PIXELS,
|
|
)
|
|
from fastvideo.pipelines.basic.minimax_h3.stages.minimax_h3_input_preparation import (
|
|
resolve_target_num_frames, )
|
|
|
|
if kwargs["fps"] != 24:
|
|
raise RequestAdaptationError(f"MiniMax-H3 requires fps=24, got {kwargs['fps']}.")
|
|
if width is None or height is None:
|
|
raise RequestAdaptationError("MiniMax-H3 requires both width and height.")
|
|
if width % MINIMAX_H3_CANVAS_MULTIPLE or height % MINIMAX_H3_CANVAS_MULTIPLE:
|
|
raise RequestAdaptationError("MiniMax-H3 width and height must be positive multiples of "
|
|
f"{MINIMAX_H3_CANVAS_MULTIPLE}, got {width}x{height}.")
|
|
if width * height > MINIMAX_H3_MAX_PIXELS:
|
|
raise RequestAdaptationError(
|
|
f"MiniMax-H3 canvas exceeds the {MINIMAX_H3_MAX_PIXELS}-pixel limit: {width}x{height}.")
|
|
try:
|
|
requested_num_frames = kwargs.get("num_frames")
|
|
aligned_num_frames = resolve_target_num_frames(requested_num_frames)
|
|
except (TypeError, ValueError) as error:
|
|
raise RequestAdaptationError(str(error)) from error
|
|
if frames_explicit and aligned_num_frames != requested_num_frames:
|
|
raise RequestAdaptationError("MiniMax-H3 num_frames must be on the causal-VAE grid (17 * n + 5); "
|
|
f"got {requested_num_frames}, next valid value is {aligned_num_frames}.")
|
|
kwargs["num_frames"] = aligned_num_frames
|
|
|
|
output_path = os.path.join(os.path.abspath(output_dir), "videos", f"{request_id}.mp4")
|
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
|
kwargs.update({
|
|
"output_path": output_path,
|
|
"save_video": True,
|
|
"return_frames": False,
|
|
})
|
|
generation_request = legacy_generate_call_to_request(request.prompt, None, legacy_kwargs=kwargs)
|
|
try:
|
|
# Resolve once at admission time so unsupported model-specific fields
|
|
# are a deterministic 400, rather than an asynchronous failed job.
|
|
request_to_sampling_param(generation_request, model_path=args.model_path)
|
|
except (TypeError, ValueError) as error:
|
|
raise RequestAdaptationError(str(error)) from error
|
|
return generation_request
|
|
|
|
|
|
__all__ = [
|
|
"RequestAdaptationError",
|
|
"build_generation_request",
|
|
"prepare_reference_media",
|
|
"validate_model_and_lora",
|
|
]
|