hao-ai-lab-FastVideo/fastvideo/entrypoints/openai/request_adapter.py

# SPDX-License-Identifier: Apache-2.0
"""Translate OpenAI/vLLM-Omni requests into FastVideo's typed request API."""

from __future__ import annotations

import asyncio
import math
import os
from typing import Any

from fastvideo.api.compat import (
    REQUEST_BATCH_EXTRA_PASSTHROUGH_FIELDS,
    explicit_request_raw,
    normalize_generation_request,
    request_to_sampling_param,
)
from fastvideo.api.schema import GenerationRequest
from fastvideo.entrypoints.openai.protocol import (
    FileImageReference,
    FileVideoReference,
    UrlImageReference,
    VideoGenerationRequest,
)
from fastvideo.entrypoints.openai.utils import save_image_to_path
from fastvideo.fastvideo_args import FastVideoArgs
from fastvideo.models.vision_utils import load_image
from fastvideo.registry import get_preset_selection


class RequestAdaptationError(ValueError):
    """The transport request cannot be represented by the loaded pipeline."""


# Field that build_generation_request collects from the OpenAI request -> its GenerationRequest path. Any other
# collected field is a model-specific value, which the pipeline reads from request.extensions.
_REQUEST_PATHS = {
    "negative_prompt": "negative_prompt",
    "enable_teacache": "runtime.enable_teacache",
    **{
        name: f"sampling.{name}"
        for name in ("width", "height", "fps", "num_frames", "seed", "num_inference_steps", "guidance_scale", "guidance_scale_2", "true_cfg_scale", "max_sequence_length", "boundary_ratio", "num_videos_per_prompt")
    },
    **{
        name: f"inputs.{name}"
        for name in ("image_path", "last_image", "video_path", "references")
    },
    **{
        name: f"output.{name}"
        for name in ("output_path", "save_video", "return_frames")
    },
}
# Sections of the serve config's default_request that apply to each request.
_DEFAULT_REQUEST_KEYS = ("negative_prompt", "inputs", "sampling", "runtime", "output", "stage_overrides", "extensions")


def _set_request_path(raw: dict[str, Any], path: str, value: Any) -> None:
    """Set one dotted path in a nested raw request mapping, creating the sections it goes through."""
    *sections, name = path.split(".")
    for section in sections:
        raw = raw.setdefault(section, {})
    raw[name] = value


def _as_list(value: Any | list[Any] | None) -> list[Any]:
    if value is None:
        return []
    return value if isinstance(value, list) else [value]


def _image_sources(request: VideoGenerationRequest) -> list[str]:
    sources: list[str] = []
    for reference in _as_list(request.image_reference):
        if isinstance(reference, FileImageReference):
            raise RequestAdaptationError("file_id image references are not supported; provide image_url instead")
        sources.append(reference.image_url)
    legacy_sources = [source for source in (request.input_reference, request.reference_url) if source]
    if len(legacy_sources) > 1:
        raise RequestAdaptationError("Provide only one of input_reference or reference_url.")
    legacy = legacy_sources[0] if legacy_sources else None
    if legacy is not None:
        if sources:
            raise RequestAdaptationError("Provide only one of input_reference/reference_url or image_reference.")
        sources.append(legacy)
    return sources


async def prepare_reference_media(
    request_id: str,
    request: VideoGenerationRequest,
    output_dir: str,
) -> None:
    """Materialize and decode image references before a job reaches workers."""
    _image_sources(request)
    uploads_dir = os.path.join(os.path.abspath(output_dir), "uploads")

    async def materialize(source: str, index: int) -> str:
        source = source.strip()
        if not source:
            raise RequestAdaptationError("Image references must not be empty.")
        if source.lower().startswith(("http://", "https://", "data:image")):
            target = os.path.join(uploads_dir, f"{request_id}_{index}")
            try:
                local_path = await save_image_to_path(source, target)
            except Exception as error:
                raise RequestAdaptationError(f"Unable to fetch image reference: {error}") from error
        else:
            local_path = os.path.abspath(os.path.expanduser(source))
            if not os.path.isfile(local_path):
                raise RequestAdaptationError(f"Image reference does not exist: {source}")
        try:
            await asyncio.to_thread(load_image, local_path)
        except Exception as error:
            raise RequestAdaptationError(f"Unable to decode image reference: {error}") from error
        return local_path

    index = 0
    references = _as_list(request.image_reference)
    for reference in references:
        if isinstance(reference, UrlImageReference):
            reference.image_url = await materialize(reference.image_url, index)
            index += 1
    if request.input_reference:
        request.input_reference = await materialize(request.input_reference, index)
        index += 1
    elif request.input_reference == "":
        request.input_reference = None
    if request.reference_url:
        request.reference_url = await materialize(request.reference_url, index)
    elif request.reference_url == "":
        request.reference_url = None


def _video_sources(request: VideoGenerationRequest) -> list[str]:
    sources: list[str] = []
    for reference in _as_list(request.video_reference):
        if isinstance(reference, FileVideoReference):
            raise RequestAdaptationError("file_id video references are not supported; provide video_url instead")
        sources.append(reference.video_url)
    direct = request.video_path or request.video_url
    if direct is not None:
        if sources:
            raise RequestAdaptationError("Provide only one of video_reference or video_path/video_url.")
        sources.append(direct)
    return sources


def _audio_sources(request: VideoGenerationRequest) -> list[str]:
    return [reference.audio_url for reference in _as_list(request.audio_reference)]


def _parse_aspect_ratio(value: str) -> tuple[float, float]:
    try:
        width, height = value.split(":", 1)
        result = float(width), float(height)
    except (AttributeError, TypeError, ValueError) as error:
        raise RequestAdaptationError(f"Invalid aspect_ratio {value!r}; expected WIDTH:HEIGHT") from error
    if not all(math.isfinite(term) for term in result) or result[0] <= 0 or result[1] <= 0:
        raise RequestAdaptationError(f"Invalid aspect_ratio {value!r}; both terms must be positive")
    return result


def _apply_aspect_ratio(
    kwargs: dict[str, Any],
    request: VideoGenerationRequest,
    *,
    model_family: str | None,
) -> None:
    if request.aspect_ratio is None:
        return
    aspect_width, aspect_height = _parse_aspect_ratio(request.aspect_ratio)
    if model_family == "minimax_h3":
        from fastvideo.pipelines.basic.minimax_h3.packing import MINIMAX_H3_SHORT_EDGE, resolve_canvas_size

        if request.short_edge is not None and request.short_edge != MINIMAX_H3_SHORT_EDGE:
            raise RequestAdaptationError(
                f"MiniMax-H3 currently uses a fixed short_edge={MINIMAX_H3_SHORT_EDGE}, got {request.short_edge}.")
        try:
            height, width = resolve_canvas_size(aspect_width, aspect_height)
        except ValueError as error:
            raise RequestAdaptationError(str(error)) from error
    elif request.short_edge is not None:
        if aspect_width >= aspect_height:
            height = request.short_edge
            width = round(request.short_edge * aspect_width / aspect_height)
        else:
            width = request.short_edge
            height = round(request.short_edge * aspect_height / aspect_width)
    else:
        return
    kwargs["width"], kwargs["height"] = width, height


def validate_served_model_name(
    model: str | None,
    args: FastVideoArgs,
    served_model_name: str,
) -> None:
    """Reject a request model id that is not the one this server loaded."""
    if model is None:
        return
    allowed_models = {args.lora_nickname} if args.lora_path else {served_model_name}
    if model not in allowed_models:
        choices = ", ".join(sorted(allowed_models))
        raise RequestAdaptationError(f"Model mismatch: request specifies {model!r}; this server provides {choices}.")


def validate_model_and_lora(
    request: VideoGenerationRequest,
    args: FastVideoArgs,
    served_model_name: str,
) -> None:
    """Validate vLLM-style model and LoRA selectors against startup state.

    FastVideo's published FastH3 adapters include dense replacement tensors in
    addition to low-rank factors. Those tensors are applied while the model is
    loaded and cannot be swapped safely between concurrent requests. The API
    accepts vLLM's selector shape, but it must identify the startup adapter.
    """
    validate_served_model_name(request.model, args, served_model_name)

    if request.lora is None:
        return
    if not args.lora_path:
        raise RequestAdaptationError(
            "This server has no startup LoRA. Configure generator.pipeline.components.lora_path before using "
            "the request lora selector.")

    body = request.lora
    name = next((body[key] for key in ("name", "lora_name", "adapter") if body.get(key) is not None), None)
    path = next((body[key] for key in ("path", "lora_path", "local_path") if body.get(key) is not None), None)
    scale = next((body[key] for key in ("scale", "lora_scale") if body.get(key) is not None), None)
    if name is None and path is None:
        raise RequestAdaptationError("lora must provide a name or path")
    if name is not None and str(name) != args.lora_nickname:
        raise RequestAdaptationError(f"Requested LoRA {name!r} is not the startup adapter {args.lora_nickname!r}.")
    if path is not None and str(path) != args.lora_path:
        raise RequestAdaptationError(
            f"Requested LoRA path {path!r} does not match the startup adapter {args.lora_path!r}.")
    if scale is not None:
        try:
            scale_value = float(scale)
        except (TypeError, ValueError) as error:
            raise RequestAdaptationError(f"Invalid LoRA scale {scale!r}") from error
        if not math.isclose(scale_value, args.lora_strength, rel_tol=0.0, abs_tol=1e-8):
            raise RequestAdaptationError(
                f"Requested LoRA scale {scale_value:g} does not match startup strength {args.lora_strength:g}.")


def _apply_reference_inputs(
    kwargs: dict[str, Any],
    request: VideoGenerationRequest,
    args: FastVideoArgs,
    *,
    model_family: str | None,
) -> None:
    images = _image_sources(request)
    videos = _video_sources(request)
    audios = _audio_sources(request)
    ref2va = model_family == "minimax_h3" and "ref2va" in (args.override_pipeline_cls_name or "").lower()

    if model_family == "minimax_h3" and request.task is not None:
        normalized_task = request.task.lower()
        if normalized_task not in {"t2va", "fl2va", "ref2va"}:
            raise RequestAdaptationError("MiniMax-H3 task must be one of t2va, fl2va, or ref2va.")
        if normalized_task == "ref2va" and not ref2va:
            raise RequestAdaptationError(
                "MiniMax-H3 task='ref2va' requires MiniMaxH3Ref2VAModularPipeline at server startup.")
        if normalized_task != "ref2va" and ref2va:
            raise RequestAdaptationError(
                f"This server is configured for MiniMax-H3 Ref2VA, not task={normalized_task!r}.")
        if normalized_task == "t2va" and (images or videos or audios):
            raise RequestAdaptationError("MiniMax-H3 task='t2va' does not accept reference media.")
        if normalized_task == "fl2va" and not images:
            raise RequestAdaptationError("MiniMax-H3 task='fl2va' requires one or two image references.")

    if ref2va:
        from fastvideo.pipelines.basic.minimax_h3 import MiniMaxH3Reference
        from fastvideo.pipelines.basic.minimax_h3.reference import validate_references

        references = [MiniMaxH3Reference(source=source, media_type="image") for source in images]
        references.extend(MiniMaxH3Reference(source=source, media_type="video") for source in videos)
        references.extend(MiniMaxH3Reference(source=source, media_type="audio") for source in audios)
        if references:
            try:
                kwargs["references"] = validate_references(references)
            except (TypeError, ValueError) as error:
                raise RequestAdaptationError(str(error)) from error
        return

    if request.task is not None and model_family != "minimax_h3":
        raise RequestAdaptationError("The task selector is only defined for MiniMax-H3 servers.")

    if model_family == "minimax_h3" and (videos or audios):
        raise RequestAdaptationError("MiniMax-H3 video/audio references require a server configured with "
                                     "override_pipeline_cls_name=MiniMaxH3Ref2VAModularPipeline.")
    if len(images) > 2:
        raise RequestAdaptationError("The loaded pipeline accepts at most first and last image references.")
    if images:
        kwargs["image_path"] = images[0]
    if len(images) == 2:
        kwargs["last_image"] = load_image(images[1])
    if len(videos) > 1:
        raise RequestAdaptationError("The loaded pipeline accepts at most one video reference.")
    if videos:
        kwargs["video_path"] = videos[0]
    if audios:
        raise RequestAdaptationError("The loaded pipeline does not accept audio reference inputs.")


def build_generation_request(
    request_id: str,
    request: VideoGenerationRequest,
    args: FastVideoArgs,
    *,
    served_model_name: str,
    output_dir: str,
    default_request: GenerationRequest | None = None,
) -> GenerationRequest:
    """Build one tracked FastVideo request using explicit-field precedence."""
    validate_model_and_lora(request, args, served_model_name)
    # The operator's default_request fields, with their sections; the request body's fields override them.
    raw: dict[str, Any] = {}
    if default_request is not None:
        raw = {
            key: value
            for key, value in explicit_request_raw(default_request).items() if key in _DEFAULT_REQUEST_KEYS
        }
    kwargs: dict[str, Any] = {}

    body_set = request.model_fields_set
    nested_set = request.video_params.model_fields_set if request.video_params is not None else set()
    if "size" in body_set and request.size is not None:
        width, height = request.size.split("x", 1)
        kwargs["width"], kwargs["height"] = int(width), int(height)
    else:
        if "width" in body_set and request.width is not None:
            kwargs["width"] = request.width
        elif "video_params" in body_set and "width" in nested_set and request.video_params.width is not None:
            kwargs["width"] = request.video_params.width
        if "height" in body_set and request.height is not None:
            kwargs["height"] = request.height
        elif "video_params" in body_set and "height" in nested_set and request.video_params.height is not None:
            kwargs["height"] = request.video_params.height

    fps_explicit = ("fps" in body_set
                    and request.fps is not None) or ("video_params" in body_set and "fps" in nested_set
                                                     and request.video_params.fps is not None)
    if fps_explicit:
        fps = request.fps if "fps" in body_set else request.video_params.fps
        if fps is not None:
            kwargs["fps"] = fps
    if "fps" not in kwargs and "fps" not in raw.get("sampling", {}):
        kwargs["fps"] = 24
    effective_fps = kwargs.get("fps", raw.get("sampling", {}).get("fps"))

    frames_explicit = ("num_frames" in body_set
                       and request.num_frames is not None) or ("video_params" in body_set and "num_frames" in nested_set
                                                               and request.video_params.num_frames is not None)
    if frames_explicit:
        num_frames = request.num_frames if "num_frames" in body_set else request.video_params.num_frames
        if num_frames is not None:
            kwargs["num_frames"] = num_frames
    elif "seconds" in body_set and request.seconds is not None:
        kwargs["num_frames"] = int(request.seconds) * int(effective_fps)

    direct_fields = (
        "seed",
        "num_inference_steps",
        "guidance_scale",
        "guidance_scale_2",
        "true_cfg_scale",
        "negative_prompt",
        "enable_teacache",
        "max_sequence_length",
        "boundary_ratio",
    )
    for name in direct_fields:
        if name in body_set:
            value = getattr(request, name)
            if value is not None:
                kwargs[name] = value
    if "n" in body_set or "num_outputs_per_prompt" in body_set:
        kwargs["num_videos_per_prompt"] = request.resolved_num_outputs

    try:
        _, model_family = get_preset_selection(args.model_path)
    except (RuntimeError, ValueError):
        model_family = None
    if request.resolved_num_outputs != 1:
        raise RequestAdaptationError("FastVideo serving currently supports exactly one video output per request.")
    if "short_edge" in body_set and request.short_edge is not None and request.aspect_ratio is None:
        raise RequestAdaptationError("short_edge requires aspect_ratio.")
    _apply_aspect_ratio(kwargs, request, model_family=model_family)
    _apply_reference_inputs(kwargs, request, args, model_family=model_family)

    extension_fields = ("flow_shift", "sound_duration", "start_time_seconds")
    for name in extension_fields:
        if name in body_set and getattr(request, name) is not None:
            kwargs[name] = getattr(request, name)
    if "generate_sound" in body_set and request.generate_sound and model_family != "minimax_h3":
        kwargs["generate_sound"] = True
    if "enable_frame_interpolation" in body_set and request.enable_frame_interpolation:
        kwargs["enable_frame_interpolation"] = True
        for name in (
                "frame_interpolation_exp",
                "frame_interpolation_scale",
                "frame_interpolation_model_path",
        ):
            kwargs[name] = getattr(request, name)
    if request.extra_params:
        unknown_extra_params = sorted(set(request.extra_params) - set(REQUEST_BATCH_EXTRA_PASSTHROUGH_FIELDS))
        if unknown_extra_params:
            raise RequestAdaptationError("Unsupported extra_params fields: " + ", ".join(unknown_extra_params))
        kwargs.update(request.extra_params)

    for name, value in kwargs.items():
        _set_request_path(raw, _REQUEST_PATHS.get(name, f"extensions.{name}"), value)
    # request_to_sampling_param applies stage overrides to the sampling fields of the same name, so a field the
    # request body sets must not stay in the operator's default stage overrides: the body value wins.
    for stage_name, overrides in list(raw.get("stage_overrides", {}).items()):
        kept = {key: value for key, value in overrides.items() if key not in kwargs}
        if kept:
            raw["stage_overrides"][stage_name] = kept
        else:
            del raw["stage_overrides"][stage_name]
    sampling = raw.setdefault("sampling", {})
    width = sampling.get("width")
    height = sampling.get("height")
    if width is not None and (not isinstance(width, int) or width <= 0):
        raise RequestAdaptationError(f"width must be a positive integer, got {width!r}")
    if height is not None and (not isinstance(height, int) or height <= 0):
        raise RequestAdaptationError(f"height must be a positive integer, got {height!r}")

    if model_family == "minimax_h3":
        from fastvideo.pipelines.basic.minimax_h3.packing import (
            MINIMAX_H3_CANVAS_MULTIPLE,
            MINIMAX_H3_MAX_PIXELS,
        )
        from fastvideo.pipelines.basic.minimax_h3.stages.minimax_h3_input_preparation import (
            resolve_target_num_frames, )

        if sampling["fps"] != 24:
            raise RequestAdaptationError(f"MiniMax-H3 requires fps=24, got {sampling['fps']}.")
        if width is None or height is None:
            raise RequestAdaptationError("MiniMax-H3 requires both width and height.")
        if width % MINIMAX_H3_CANVAS_MULTIPLE or height % MINIMAX_H3_CANVAS_MULTIPLE:
            raise RequestAdaptationError("MiniMax-H3 width and height must be positive multiples of "
                                         f"{MINIMAX_H3_CANVAS_MULTIPLE}, got {width}x{height}.")
        if width * height > MINIMAX_H3_MAX_PIXELS:
            raise RequestAdaptationError(
                f"MiniMax-H3 canvas exceeds the {MINIMAX_H3_MAX_PIXELS}-pixel limit: {width}x{height}.")
        try:
            requested_num_frames = sampling.get("num_frames")
            aligned_num_frames = resolve_target_num_frames(requested_num_frames)
        except (TypeError, ValueError) as error:
            raise RequestAdaptationError(str(error)) from error
        if frames_explicit and aligned_num_frames != requested_num_frames:
            raise RequestAdaptationError("MiniMax-H3 num_frames must be on the causal-VAE grid (17 * n + 5); "
                                         f"got {requested_num_frames}, next valid value is {aligned_num_frames}.")
        sampling["num_frames"] = aligned_num_frames

    output_path = os.path.join(os.path.abspath(output_dir), "videos", f"{request_id}.mp4")
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    raw.setdefault("output", {}).update({
        "output_path": output_path,
        "save_video": True,
        "return_frames": False,
    })
    generation_request = normalize_generation_request({"prompt": request.prompt, **raw})
    try:
        # Resolve once at admission time so unsupported model-specific fields
        # are a deterministic 400, rather than an asynchronous failed job.
        request_to_sampling_param(generation_request, model_path=args.model_path)
    except (TypeError, ValueError) as error:
        raise RequestAdaptationError(str(error)) from error
    return generation_request


__all__ = [
    "RequestAdaptationError",
    "build_generation_request",
    "prepare_reference_media",
    "validate_model_and_lora",
    "validate_served_model_name",
]