[feat] v2 VideoGenerator: typed fastvideo.api entrypoint over the v2 engine
Mirrors fastvideo.entrypoints.VideoGenerator (from_config(GeneratorConfig) -> generate( GenerationRequest) -> GenerationResult.video_path), reusing the OFFICIAL fastvideo.api config classes so a basic_dmd_new_api.py-style script differs only by importing VideoGenerator from v2. - model_path -> v2 card registry (Wan2.1 / FastWan -> wan21; SFWan -> wan_causal; LTX2 -> ltx2); snapshot_download + stamp_wan21_checkpoints + Engine(cuda) + program; generate maps SamplingConfig -> DiffusionParams -> make_request -> eng.run, saves the [C,T,H,W] decode as an mp4. - Lazy v2.__getattr__ keeps 'import v2' torch-free (verified) so the CPU mini stays green (202+2). - examples/inference/basic/v2_basic_new_api.py runs all three GPU models through this API. - Single-GPU, resident, TORCH_SDPA (EngineConfig offload/num_gpus>1/VSA accepted for parity, not applied). verified: wan21 from_config->generate->mp4 (frames (5,256,256,3) uint8, video_path written).
This commit is contained in:
SolitaryThinker
committed by
Peiyuan Zhang
Peiyuan Zhang
parent
d8af2e60d2
commit
f51497ee6d
@@ -0,0 +1,87 @@
|
||||
"""v2 typed-API inference example — mirrors ``basic_dmd_new_api.py`` but drives the **v2
|
||||
(recipe, runtime) substrate + real torch backend** for the three models brought up on GPU
|
||||
(Wan2.1, SF-causal Wan, LTX-2).
|
||||
|
||||
The ONLY delta from the upstream example is importing ``VideoGenerator`` from ``v2`` instead of
|
||||
``fastvideo`` — the typed config classes are the SAME ``fastvideo.api`` dataclasses.
|
||||
|
||||
Run (on a GPU box, with the v2 venv active):
|
||||
python examples/inference/basic/v2_basic_new_api.py
|
||||
|
||||
Notes vs upstream: the v2 bring-up runs single-GPU, resident, on the TORCH_SDPA backend (no
|
||||
fastvideo-kernel / VSA), so resolutions/steps are modest here for a quick runnable demo. LTX-2 loads
|
||||
an 18.88B DiT (slow first load).
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
|
||||
from v2 import VideoGenerator
|
||||
from fastvideo.api import (
|
||||
EngineConfig,
|
||||
GenerationRequest,
|
||||
GeneratorConfig,
|
||||
OffloadConfig,
|
||||
OutputConfig,
|
||||
SamplingConfig,
|
||||
)
|
||||
|
||||
OUTPUT_PATH = "v2_video_samples"
|
||||
|
||||
MODELS = [
|
||||
{
|
||||
"family": "wan21",
|
||||
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||
"prompt": "a red panda surfing on ocean waves at sunset, cinematic, highly detailed",
|
||||
"sampling": SamplingConfig(num_frames=25, height=480, width=832,
|
||||
num_inference_steps=30, guidance_scale=5.0, seed=1, fps=16),
|
||||
},
|
||||
{
|
||||
"family": "wan_causal",
|
||||
"model_path": "wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers",
|
||||
"prompt": "a cat walking through a sunlit garden, cinematic",
|
||||
"sampling": SamplingConfig(num_frames=25, height=480, width=832,
|
||||
num_inference_steps=4, guidance_scale=5.0, seed=1, fps=16),
|
||||
},
|
||||
{
|
||||
"family": "ltx2",
|
||||
"model_path": "FastVideo/LTX2-Distilled-Diffusers",
|
||||
"prompt": "surfers riding ocean waves at sunset, cinematic, highly detailed",
|
||||
"sampling": SamplingConfig(num_frames=9, height=512, width=768,
|
||||
num_inference_steps=8, guidance_scale=1.0, seed=1, fps=16),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def run_one(m: dict) -> None:
|
||||
generator_config = GeneratorConfig(
|
||||
model_path=m["model_path"],
|
||||
engine=EngineConfig(
|
||||
num_gpus=1,
|
||||
use_fsdp_inference=False,
|
||||
offload=OffloadConfig(text_encoder=False, dit=False, vae=False, pin_cpu_memory=False),
|
||||
),
|
||||
)
|
||||
load_start = time.perf_counter()
|
||||
generator = VideoGenerator.from_config(generator_config)
|
||||
load_time = time.perf_counter() - load_start
|
||||
|
||||
request = GenerationRequest(
|
||||
prompt=m["prompt"],
|
||||
sampling=m["sampling"],
|
||||
output=OutputConfig(output_path=OUTPUT_PATH, output_video_name=f"v2_{m['family']}",
|
||||
save_video=True, return_frames=False),
|
||||
)
|
||||
gen_start = time.perf_counter()
|
||||
result = generator.generate(request)
|
||||
gen_time = time.perf_counter() - gen_start
|
||||
|
||||
print(f"[{m['family']:10s}] load={load_time:6.1f}s gen={gen_time:6.1f}s -> {result.video_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
for m in MODELS:
|
||||
run_one(m)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -54,5 +54,15 @@ __all__ = [
|
||||
"Engine", "AsyncEngine", "Program", "ProgramKind", "ComponentNode", "ModelLoopNode", "when_task", "when_opt",
|
||||
"Request", "Session", "Output", "make_request", "TaskType", "SamplingParams", "DiffusionParams",
|
||||
"LoopKind", "WorkUnitKind", "ConsistencyLevel", "ExecutionProfile", "Capability",
|
||||
"VideoGenerator",
|
||||
"__version__",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
# Lazy: the GPU entrypoint imports torch / fastvideo, so resolve it only on access — plain
|
||||
# ``import v2`` (and the CPU-only mini) stay torch-free.
|
||||
if name == "VideoGenerator":
|
||||
from .video_generator import VideoGenerator
|
||||
return VideoGenerator
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""v2 ``VideoGenerator`` — a typed entrypoint mirroring ``fastvideo.entrypoints.VideoGenerator``,
|
||||
wrapping the v2 ``Engine`` so a ``basic_dmd_new_api.py``-style script runs against the v2
|
||||
(recipe, runtime) substrate + the real torch backend on a GPU box.
|
||||
|
||||
It consumes the OFFICIAL ``fastvideo.api`` typed configs (``GeneratorConfig`` / ``GenerationRequest`` /
|
||||
``EngineConfig`` / ``OutputConfig`` / ...) — so the only delta from the upstream example is importing
|
||||
``VideoGenerator`` from ``v2`` instead of ``fastvideo``. ``from_config(GeneratorConfig)`` →
|
||||
``generate(GenerationRequest)`` → ``GenerationResult`` with ``.video_path``.
|
||||
|
||||
All torch / fastvideo / huggingface imports are lazy (inside methods) so ``import v2`` stays CPU-clean.
|
||||
|
||||
Scope: the three representative models brought up on GPU (Wan2.1, SF-causal Wan, LTX-2). Multi-GPU
|
||||
(num_gpus>1), CPU/FSDP offload, and VSA attention are accepted for API parity but the bring-up runs
|
||||
single-GPU, resident, on the TORCH_SDPA backend (no fastvideo-kernel) — see GPU_BRINGUP.md.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
# model_path (HF id, from fastvideo/tests/ssim/*) -> v2 model family. FastWan / DMD-distilled Wan
|
||||
# reuse the wan21 card (same WanTransformer3DModel arch); SF-causal -> wan_causal; LTX-2 -> ltx2.
|
||||
_FAMILY_BY_PATH = {
|
||||
"Wan-AI/Wan2.1-T2V-1.3B-Diffusers": "wan21",
|
||||
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers": "wan21",
|
||||
"wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers": "wan_causal",
|
||||
"FastVideo/LTX2-Distilled-Diffusers": "ltx2",
|
||||
"FastVideo/LTX-2.3-Distilled-Diffusers": "ltx2",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_family(model_path: str) -> str:
|
||||
if model_path in _FAMILY_BY_PATH:
|
||||
return _FAMILY_BY_PATH[model_path]
|
||||
p = model_path.lower()
|
||||
if "ltx" in p:
|
||||
return "ltx2"
|
||||
if "sfwan" in p or "self-forcing" in p or "causal" in p:
|
||||
return "wan_causal"
|
||||
if "wan2.1-t2v-1.3b" in p or "fastwan2.1-t2v-1.3b" in p:
|
||||
return "wan21"
|
||||
raise ValueError(
|
||||
f"v2 VideoGenerator: no v2 card mapped for model_path {model_path!r}. "
|
||||
f"Supported: {sorted(_FAMILY_BY_PATH)} (or names containing ltx / sfwan / wan2.1-t2v-1.3b).")
|
||||
|
||||
|
||||
def _build_card_and_program(family: str):
|
||||
if family == "wan21":
|
||||
from .models.wan21 import build_wan21_card, build_wan_t2v_program
|
||||
return build_wan21_card(), build_wan_t2v_program()
|
||||
if family == "wan_causal":
|
||||
from .models.wan_causal import build_wan_causal_card, build_wan_causal_program
|
||||
return build_wan_causal_card(), build_wan_causal_program()
|
||||
if family == "ltx2":
|
||||
from .models.ltx2 import build_ltx2_card, build_ltx2_program
|
||||
return build_ltx2_card(), build_ltx2_program()
|
||||
raise ValueError(f"unknown family {family!r}")
|
||||
|
||||
|
||||
class VideoGenerator:
|
||||
"""Typed t2v entrypoint over a single resident v2 ``ModelInstance`` (mirrors fastvideo's)."""
|
||||
|
||||
def __init__(self, engine: Any, model_id: str, family: str) -> None:
|
||||
self._engine = engine
|
||||
self._model_id = model_id
|
||||
self._family = family
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
@classmethod
|
||||
def from_pretrained(cls, model_path: str, **kwargs: Any) -> "VideoGenerator":
|
||||
"""Convenience: build from an HF id (+ optional GeneratorConfig kwargs)."""
|
||||
from fastvideo.api import GeneratorConfig
|
||||
return cls.from_config(GeneratorConfig(model_path=model_path, **kwargs))
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: Any) -> "VideoGenerator":
|
||||
"""``config``: ``fastvideo.api.GeneratorConfig``. Resolves the v2 card from ``model_path``,
|
||||
stamps the (downloaded) checkpoint onto it, and registers it on a fresh v2 ``Engine`` whose
|
||||
``Platform.detect()`` returns cuda on a GPU box."""
|
||||
# The v2 bring-up runs dense attention via SDPA (fastvideo-kernel / VSA not built). Honor an
|
||||
# explicit override but default to SDPA so PipelineSelection.experimental VSA knobs don't break.
|
||||
os.environ.setdefault("FASTVIDEO_ATTENTION_BACKEND", "TORCH_SDPA")
|
||||
|
||||
model_path = config.model_path
|
||||
family = _resolve_family(model_path)
|
||||
if getattr(config.engine, "num_gpus", 1) != 1:
|
||||
import warnings
|
||||
warnings.warn("v2 VideoGenerator bring-up is single-GPU; ignoring num_gpus>1.", stacklevel=2)
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
root = snapshot_download(model_path, revision=getattr(config, "revision", None))
|
||||
|
||||
card, program = _build_card_and_program(family)
|
||||
from .models.wan21 import stamp_wan21_checkpoints # transformer/vae/text_encoder subfolder layout
|
||||
stamp_wan21_checkpoints(card, root)
|
||||
|
||||
from .card import load_card
|
||||
from .cache import CacheManager
|
||||
from .runtime import Engine
|
||||
inst = load_card(card, cache_manager=CacheManager.from_card(card)) # platform auto-detect -> cuda
|
||||
eng = Engine()
|
||||
eng.register(card.model_id, inst, program)
|
||||
return cls(eng, card.model_id, family)
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
def generate(self, request: Any) -> Any:
|
||||
"""``request``: ``fastvideo.api.GenerationRequest``. Returns a ``GenerationResult`` (or a list
|
||||
for multi-prompt) with ``.video_path`` (saved mp4) and optionally ``.frames``."""
|
||||
from .request import DiffusionParams, TaskType, make_request
|
||||
s = request.sampling
|
||||
prompts = request.prompt if isinstance(request.prompt, list) else [request.prompt]
|
||||
results = []
|
||||
for idx, prompt in enumerate(prompts):
|
||||
diff = DiffusionParams(
|
||||
num_steps=s.num_inference_steps, seed=s.seed, num_frames=s.num_frames,
|
||||
height=s.height, width=s.width, guidance_scale=s.guidance_scale,
|
||||
negative_prompt=request.negative_prompt or "",
|
||||
sigmas=tuple(s.sigmas) if getattr(s, "sigmas", None) else None)
|
||||
req = make_request(TaskType.T2V, self._model_id, prompt, diffusion=diff)
|
||||
out = self._engine.run(req)
|
||||
results.append(self._result(out, request.output, s.fps, idx))
|
||||
return results[0] if len(results) == 1 else results
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
def _result(self, out: Any, output: Any, fps: int, idx: int) -> Any:
|
||||
import numpy as np
|
||||
from fastvideo.api import GenerationResult
|
||||
frames = np.asarray(out.artifacts["video"].frames, dtype="float32") # [C,T,H,W] in [-1,1]
|
||||
# -> [T,H,W,C] uint8 (the on-disk / return convention)
|
||||
vid = ((np.clip(frames, -1.0, 1.0) + 1.0) / 2.0 * 255.0).astype("uint8")
|
||||
vid = vid.transpose(1, 2, 3, 0) if vid.ndim == 4 else vid
|
||||
video_path = None
|
||||
if getattr(output, "save_video", True):
|
||||
os.makedirs(output.output_path, exist_ok=True)
|
||||
name = output.output_video_name or f"v2_{self._family}_{idx}"
|
||||
name = name if name.endswith(".mp4") else f"{name}.mp4"
|
||||
video_path = os.path.join(output.output_path, name)
|
||||
import imageio.v2 as imageio
|
||||
imageio.mimsave(video_path, list(vid), fps=int(fps), format="mp4")
|
||||
return GenerationResult(frames=(vid if getattr(output, "return_frames", True) else None),
|
||||
video_path=video_path)
|
||||
Reference in New Issue
Block a user