diff --git a/examples/inference/basic/V2_PORTING_STATUS.md b/examples/inference/basic/V2_PORTING_STATUS.md new file mode 100644 index 000000000..85f018ad6 --- /dev/null +++ b/examples/inference/basic/V2_PORTING_STATUS.md @@ -0,0 +1,49 @@ +# v2 porting status — fastvideo models → the v2 (recipe, runtime) substrate + +Goal: drive each `examples/inference/basic/` model through the **v2 `VideoGenerator`** (typed +`fastvideo.api` configs + the real torch backend) and generate video, with a matching `v2_*` example. + +The v2 mini implements a small set of loops/samplers (**flow-match Euler**, causal **chunk_rollout**, +**FlowGRPO SDE**) and adapters (Wan DiT/VAE/UMT5, causal Wan, LTX-2 DiT/VAE/Gemma). A model ports +**cleanly** only when its architecture *and* sampler match what v2 implements; otherwise it needs new +per-model work (a card, an adapter for the DiT/VAE/encoder forward, and/or a new sampler), and some are +blocked by the environment (gated weights, local-only weights, non-video modality, interactive input). + +The v2 bring-up runs **single-GPU, resident, on the `TORCH_SDPA` backend** (no fastvideo-kernel / VSA / +FP4), so distilled models that depend on VSA/SLA/FP4 kernels or non-flow-match samplers won't match. + +## Working today (verified, real video) — committed on `v2` +| Official example(s) | Model | v2 card | v2 example | +|---|---|---|---| +| `basic.py`, `basic_mps.py`*, `basic_ray.py`* | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | wan21 | `v2_basic.py`, `v2_basic_new_api.py` | +| `basic_self_forcing_causal.py` | `wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers` | wan_causal | `v2_basic_self_forcing_causal.py`, `v2_basic_new_api.py` | +| `basic_ltx2_distilled.py`, `basic_ltx2_distilled_fast_profile.py` | `FastVideo/LTX2-Distilled-Diffusers` | ltx2 | `v2_basic_new_api.py` | + +\* `basic_mps.py` (Apple MPS) and `basic_ray.py` (ray executor) are device/executor variants of the +same Wan2.1 model — the model itself is covered by wan21; those runtime modes are out of v2 scope. + +## Needs per-model work (architecture/sampler matches partially) +| Official example(s) | Model | What v2 needs | +|---|---|---| +| `basic_dmd.py`, `basic_dmd_new_api.py` | FastWan2.1-T2V-1.3B (WanDMDPipeline) | **Blocked via generic reuse:** DMD checkpoint's `to_gate_compress` param mapping differs from the generic `WanTransformer3DModel` load; needs WanDMD param-mapping/config handling (+ ideally the DMD/UniPC few-step schedule). | +| `basic_turbodiffusion*.py` | TurboWan2.1/2.2 (RCM) | **New sampler:** Reparameterized Consistency Model — v2 has no consistency loop (flow-match Euler won't match). | +| `basic_wan2_2.py`, `basic_wan2_2_ti2v.py`, `basic_wan2_2_i2v.py`, `basic_wan2_2_Fun.py` | Wan2.2 (A14B MoE / TI2V-5B / I2V / Fun) | **New card:** MoE boundary-timestep expert routing (2 transformers; v2 has `ExpertRouting`), TI2V-5B's distinct VAE, I2V image conditioning, Fun control. Large (~56GB A14B). | +| `basic_ltx2.py` | `Davids048/LTX2-Base-Diffusers` | Base (non-distilled): base sigma schedule + more steps (the v2 ltx2 card hardcodes the distilled 8+3). | +| `basic_ltx2_3_distilled_i2v*.py` | LTX-2.3-Distilled I2V | Image conditioning on the ltx2 program (encode + condition). | +| `basic_lucy_edit.py` | `decart-ai/Lucy-Edit-Dev` | Wan-family video-edit: input-video conditioning. | + +## New families — each needs a new card + adapters (DiT/VAE/encoder forwards) + maybe a sampler +`basic_cosmos2_5_*` (Cosmos-Predict2.5), `basic_gen3c.py` (GEN3C — **local `converted_weights/` only**), +`basic_hy15*.py` / `basic_gamecraft.py` / `basic_hyworld.py` (Hunyuan family), +`basic_longcat_*.py` (LongCat + LoRA), `basic_lingbotworld_base_cam.py`, +`basic_sd35_t2i.py` (SD3.5 image), `basic_flux2*.py` (**gated** black-forest-labs). + +## Different modality / interactive (out of the t2v VideoGenerator surface) +`basic_stable_audio*.py` (audio), `basic_matrixgame2*.py` / `basic_matrixgame3.py` +(interactive mouse/keyboard-conditioned world models). + +## How to add a model to the v2 VideoGenerator +1. Ensure a v2 card exists (`v2/models//`) whose `load_id`s resolve to real `fastvideo.models.*`. +2. Register the HF id → family in `v2/video_generator.py:_FAMILY_BY_PATH` (or rely on the name fallback). +3. If the DiT/VAE/encoder forward differs, add an adapter + class-detection in `torch_adapters.py`. +4. Add `examples/inference/basic/v2_.py` mirroring the official example; generate to confirm. diff --git a/examples/inference/basic/v2_basic.py b/examples/inference/basic/v2_basic.py new file mode 100644 index 000000000..754e94d24 --- /dev/null +++ b/examples/inference/basic/v2_basic.py @@ -0,0 +1,36 @@ +"""v2 port of basic.py — Wan2.1-T2V-1.3B through the v2 VideoGenerator. + +Same convenience API as upstream (from_pretrained + generate_video); only delta is importing +VideoGenerator from v2. v2 bring-up: single-GPU, resident, SDPA; modest res/frames for a quick run. +""" +from v2 import VideoGenerator + +OUTPUT_PATH = "v2_video_samples" + + +def main() -> None: + generator = VideoGenerator.from_pretrained( + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=1, + use_fsdp_inference=False, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=False, + pin_cpu_memory=False, + ) + common = dict(output_path=OUTPUT_PATH, save_video=True, + num_frames=25, height=480, width=832, num_inference_steps=30, guidance_scale=5.0) + + prompt = ("A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes wide with " + "interest. The playful yet serene atmosphere is complemented by soft natural light " + "filtering through the petals. Mid-shot, warm and cheerful tones.") + video = generator.generate_video(prompt, output_video_name="wan21_raccoon", **common) + + prompt2 = ("A majestic lion strides across the golden savanna, its powerful frame glistening under " + "the warm afternoon sun. Low angle, steady tracking shot, cinematic.") + video2 = generator.generate_video(prompt2, output_video_name="wan21_lion", **common) + print(f"Outputs: {video.video_path} , {video2.video_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/inference/basic/v2_basic_self_forcing_causal.py b/examples/inference/basic/v2_basic_self_forcing_causal.py new file mode 100644 index 000000000..5180eeb85 --- /dev/null +++ b/examples/inference/basic/v2_basic_self_forcing_causal.py @@ -0,0 +1,30 @@ +"""v2 port of basic_self_forcing_causal.py — SF-causal Wan2.1 (CausalWanTransformer3DModel) through +the v2 VideoGenerator (chunk_rollout loop). + +Same convenience API as upstream; only delta is importing VideoGenerator from v2. NOTE: the v2 causal +loop runs per-chunk few-step (not the upstream kv-cache streaming + SF schedule), so output is coherent +but lower-fidelity (a documented gap). num_frames is set by the card's chunk schedule; height/width +drive the latent geometry. +""" +from v2 import VideoGenerator +from fastvideo.api.sampling_param import SamplingParam + +OUTPUT_PATH = "v2_video_samples_causal" + + +def main() -> None: + model_name = "wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers" + generator = VideoGenerator.from_pretrained( + model_name, num_gpus=1, text_encoder_cpu_offload=False, dit_cpu_offload=False) + sampling_param = SamplingParam.from_pretrained(model_name) + + prompt = ("A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes wide with " + "interest. The playful yet serene atmosphere is complemented by soft natural light " + "filtering through the petals. Mid-shot, warm and cheerful tones.") + video = generator.generate_video(prompt, output_path=OUTPUT_PATH, output_video_name="causal_raccoon", + save_video=True, sampling_param=sampling_param, height=480, width=832) + print(f"Output: {video.video_path}") + + +if __name__ == "__main__": + main() diff --git a/v2/video_generator.py b/v2/video_generator.py index ceb83e2fe..f1af51ad8 100644 --- a/v2/video_generator.py +++ b/v2/video_generator.py @@ -68,9 +68,20 @@ class VideoGenerator: # --------------------------------------------------------------------- # @classmethod def from_pretrained(cls, model_path: str, **kwargs: Any) -> "VideoGenerator": - """Convenience: build from an HF id (+ optional GeneratorConfig kwargs).""" - from fastvideo.api import GeneratorConfig - return cls.from_config(GeneratorConfig(model_path=model_path, **kwargs)) + """Convenience constructor (mirrors fastvideo's): accepts the legacy ``from_pretrained`` kwargs + (``num_gpus`` / ``use_fsdp_inference`` / ``*_cpu_offload`` / ``pin_cpu_memory`` / ``VSA_sparsity`` + / ...). The v2 bring-up runs single-GPU, resident, on SDPA, so these are accepted for parity but + not all applied.""" + from fastvideo.api import EngineConfig, GeneratorConfig, OffloadConfig + engine = EngineConfig( + num_gpus=int(kwargs.get("num_gpus", 1)), + use_fsdp_inference=bool(kwargs.get("use_fsdp_inference", False)), + offload=OffloadConfig( + text_encoder=bool(kwargs.get("text_encoder_cpu_offload", False)), + dit=bool(kwargs.get("dit_cpu_offload", False)), + vae=bool(kwargs.get("vae_cpu_offload", False)), + pin_cpu_memory=bool(kwargs.get("pin_cpu_memory", False)))) + return cls.from_config(GeneratorConfig(model_path=model_path, engine=engine)) @classmethod def from_config(cls, config: Any) -> "VideoGenerator": @@ -121,6 +132,41 @@ class VideoGenerator: results.append(self._result(out, request.output, s.fps, idx)) return results[0] if len(results) == 1 else results + # --------------------------------------------------------------------- # + def generate_video(self, prompt: Any = None, sampling_param: Any = None, **kwargs: Any) -> Any: + """Convenience API mirroring ``fastvideo.VideoGenerator.generate_video``. Builds a + ``GenerationRequest`` from loose kwargs (+ an optional ``SamplingParam`` override) and calls + ``generate()``. kwargs: num_inference_steps / seed / num_frames / height / width / + guidance_scale / fps / sigmas / negative_prompt / output_path / output_video_name / + save_video / return_frames.""" + from fastvideo.api import GenerationRequest, OutputConfig, SamplingConfig + sp = sampling_param + + def pick(key: str, default: Any) -> Any: + if key in kwargs and kwargs[key] is not None: + return kwargs[key] + if sp is not None and getattr(sp, key, None) is not None: + return getattr(sp, key) + return default + + sampling = SamplingConfig( + num_inference_steps=int(pick("num_inference_steps", 30)), + seed=int(pick("seed", 1024)), + num_frames=int(pick("num_frames", 25)), + height=int(pick("height", 480)), + width=int(pick("width", 832)), + guidance_scale=float(pick("guidance_scale", 5.0)), + fps=int(pick("fps", 16)), + sigmas=pick("sigmas", None)) + output = OutputConfig( + output_path=kwargs.get("output_path", "outputs/"), + output_video_name=kwargs.get("output_video_name"), + save_video=bool(kwargs.get("save_video", True)), + return_frames=bool(kwargs.get("return_frames", False))) + req = GenerationRequest(prompt=prompt, negative_prompt=pick("negative_prompt", None), + sampling=sampling, output=output) + return self.generate(req) + # --------------------------------------------------------------------- # def _result(self, out: Any, output: Any, fps: int, idx: int) -> Any: import numpy as np