From e82951f1c4105342ef86bcb7caa6a9fd2c670dcc Mon Sep 17 00:00:00 2001 From: Mac Lee Date: Mon, 29 Jun 2026 04:36:49 +0000 Subject: [PATCH] [docs]: document InterleaveThinker inference workflow --- docs/design/interleave_thinker.md | 78 +++++++++++++ examples/interleave/README.md | 48 ++++++++ .../interleave/interleave_single_prompt.py | 107 ++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 234 insertions(+) create mode 100644 docs/design/interleave_thinker.md create mode 100644 examples/interleave/README.md create mode 100644 examples/interleave/interleave_single_prompt.py diff --git a/docs/design/interleave_thinker.md b/docs/design/interleave_thinker.md new file mode 100644 index 000000000..314c3a8c1 --- /dev/null +++ b/docs/design/interleave_thinker.md @@ -0,0 +1,78 @@ +# InterleaveThinker Inference Workflow + +This page summarizes the inference half of the InterleaveThinker integration. +It adds reusable Python helpers for interleaved image generation/editing without +adding new `fastvideo` CLI commands or HTTP API routes. + +The training half is intentionally deferred to a follow-up PR after this +inference PR lands upstream. + +## Scope + +This PR adds: + +- typed request/response and trace schemas for interleaved workflows; +- a planner -> generator/edit -> critic orchestration loop; +- a FastVideo image backend adapter for `VideoGenerator`; +- an optional Nano Banana image backend with lazy Google SDK imports; +- config-driven single-prompt runs and prompt-set evaluation helpers; +- saved-trace metrics and report helpers; +- a runnable single-prompt example under `examples/interleave/`. + +This PR does not add: + +- Qwen3-VL planner or critic model wrappers; +- InterleaveThinker SFT or GRPO training methods; +- training YAML configs; +- standalone FastVideo subcommands or a new HTTP service. + +## Integrated Surfaces + +| Surface | Purpose | +|---------|---------| +| `fastvideo.workflow.interleave_thinker.schema` | Typed edit requests, generated-image records, planner/critic inputs, attempts, and traces. | +| `fastvideo.workflow.interleave_thinker.orchestrator` | Provider-based planner/generator/critic loop with retry/refinement support. | +| `fastvideo.workflow.interleave_thinker.generator` | Image backend protocol plus FastVideo and Nano Banana implementations. | +| `fastvideo.workflow.interleave_thinker.config` | Typed config loading for reusable interleave runs. | +| `fastvideo.workflow.interleave_thinker.runner` | Config-driven runner for the inference-only `single_prompt` planner and `accept_all`/`none` critics. | +| `fastvideo.workflow.interleave_thinker.evaluation` | Prompt-set execution and resumable trace writing. | +| `fastvideo.workflow.interleave_thinker.trace_eval` | Saved-trace metric extraction and JSON/HTML reports. | + +The Qwen-backed `interleave_thinker` planner and critic config kinds are +reserved for the follow-up training PR. In this PR, selecting those kinds raises +a clear runtime error instead of importing training-only modules. + +## Runtime Flow + +1. A user supplies an instruction through a typed config or Python call. +2. A planner provider returns one or more `PlannedInterleaveStep` objects. The + inference PR ships `SinglePromptPlanner`, which turns the instruction into a + single generator step. +3. An image backend translates the step into a `GenerationRequest` or external + image API request. +4. An optional critic reviews the generated image and can request a refined + prompt for another attempt. +5. The orchestrator records every attempt in an `InterleaveTrace`. +6. Trace helpers write JSON without base64 image payloads by default. + +## Validation Expectations + +Before the PR is marked ready: + +- run `pre-commit run --all-files` in an authoritative environment; +- run focused workflow tests covering backend translation, orchestration, + config loading, prompt-set execution, and trace evaluation; +- run targeted Wan T2V SSIM on Modal L40S, or document why the workflow helper + changes are N/A for Wan T2V pixel output; +- rely on upstream CI for pre-commit, Fastcheck, and full-suite gating before + merge. + +## Review Checklist + +- Confirm no standalone `fastvideo` CLI command or HTTP API route was added. +- Confirm workflow imports do not require training-only InterleaveThinker model + modules. +- Confirm optional Google SDK usage is lazy and only required for Nano Banana + execution. +- Confirm fake-provider tests cover planner, generator, critic, trace, and + evaluation behavior without credentials. diff --git a/examples/interleave/README.md b/examples/interleave/README.md new file mode 100644 index 000000000..e1fbd935a --- /dev/null +++ b/examples/interleave/README.md @@ -0,0 +1,48 @@ +# FastVideo Interleave Examples + +This directory contains a small Python example for the reusable Interleave +orchestration helpers. It does not add FastVideo CLI commands or HTTP routes. + +## Single-Prompt Trace + +Run: + +```bash +FASTVIDEO_ATTENTION_BACKEND=TORCH_SDPA \ + python examples/interleave/interleave_single_prompt.py \ + --model-path black-forest-labs/FLUX.2-klein-4B \ + --prompt "a brushed steel espresso machine on a marble counter, morning window light" \ + --output-dir outputs/interleave_single_prompt +``` + +The script uses `VideoGenerator` directly with a fallback single-prompt planner +and accept-all critic. It writes an image plus `trace.json`; the trace records +planner/generator/critic attempts and omits base64 image payloads by default. + +## Planner And Critic Follow-Up + +This inference PR ships the reusable workflow layer and the fallback +single-prompt planner. The Qwen3-VL InterleaveThinker planner/critic model +wrappers and training configs are intentionally deferred to a follow-up +training PR. + +## Optional Gemini Backends + +The workflow helper can use closed-source Google image models through lazy +wrappers: + +- `fastvideo.workflow.interleave_thinker.generator.NanoBananaImageGeneratorBackend` + implements the same image backend protocol as the local FastVideo generator. + +Install the optional SDK and provide a key only when using these API backends: + +```bash +uv pip install -e ".[eval-judge]" +export GEMINI_API_KEY=... +``` + +Supported Nano Banana aliases are: + +- `nano-banana` -> `gemini-2.5-flash-image` +- `nano-banana-pro` -> `gemini-3-pro-image` +- `nano-banana-2` -> `gemini-3.1-flash-image` diff --git a/examples/interleave/interleave_single_prompt.py b/examples/interleave/interleave_single_prompt.py new file mode 100644 index 000000000..b5350d7ef --- /dev/null +++ b/examples/interleave/interleave_single_prompt.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Run a one-step interleaved generation trace through FastVideo. + +This is intentionally small: it uses the fallback single-prompt planner and an +accept-all critic, so it exercises the native Interleave helper layer without +requiring InterleaveThinker planner/critic checkpoints. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +from fastvideo import VideoGenerator +from fastvideo.api.schema import ( + EngineConfig, + GeneratorConfig, + OffloadConfig, + PipelineSelection, +) +from fastvideo.workflow.interleave_thinker import ( + AcceptAllCritic, + FastVideoImageGeneratorBackend, + InterleaveOrchestrator, + SinglePromptPlanner, + save_trace, +) + + +DEFAULT_PROMPT = "a brushed steel espresso machine on a marble counter, morning window light" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run a one-step FastVideo interleave trace.") + parser.add_argument( + "--model-path", + default="black-forest-labs/FLUX.2-klein-4B", + help="HF id or local diffusers-format image model directory.", + ) + parser.add_argument("--prompt", default=DEFAULT_PROMPT, help="Interleaved generation instruction.") + parser.add_argument("--output-dir", default="outputs/interleave_single_prompt", help="Output directory.") + parser.add_argument("--trace-path", default=None, help="Trace JSON path. Defaults under output-dir.") + parser.add_argument("--seed", type=int, default=0, help="Generation seed.") + parser.add_argument("--height", type=int, default=1024, help="Output image height.") + parser.add_argument("--width", type=int, default=1024, help="Output image width.") + parser.add_argument("--steps", type=int, default=4, help="Number of denoising steps.") + parser.add_argument("--num-gpus", type=int, default=1, help="Number of GPUs to use.") + parser.add_argument( + "--backend", + default=None, + help="Set FASTVIDEO_ATTENTION_BACKEND, for example TORCH_SDPA.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.backend: + os.environ["FASTVIDEO_ATTENTION_BACKEND"] = args.backend + + output_dir = Path(args.output_dir) + trace_path = Path(args.trace_path) if args.trace_path else output_dir / "trace.json" + + generator_config = GeneratorConfig( + model_path=args.model_path, + engine=EngineConfig( + num_gpus=args.num_gpus, + use_fsdp_inference=False, + offload=OffloadConfig( + dit=False, + vae=True, + text_encoder=True, + pin_cpu_memory=False, + ), + ), + pipeline=PipelineSelection(workload_type="t2i"), + ) + + generator = VideoGenerator.from_config(generator_config) + try: + backend = FastVideoImageGeneratorBackend( + generator, + output_dir=str(output_dir), + ) + orchestrator = InterleaveOrchestrator( + planner=SinglePromptPlanner(), + generator=backend, + critic=AcceptAllCritic(), + width=args.width, + height=args.height, + num_inference_steps=args.steps, + guidance_scale=1.0, + seed=args.seed, + ) + trace = orchestrator.run(args.prompt) + save_trace(trace, trace_path) + if trace.final_image is None or trace.final_image.file_path is None: + raise RuntimeError("Interleave run completed without a final image path") + print(f"Image: {trace.final_image.file_path}") + print(f"Trace: {trace_path}") + finally: + generator.shutdown() + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index e8a349cf2..fdb58d570 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -191,6 +191,7 @@ nav: - Debugging: utilities/debugging.md - Design: - Overview: design/overview.md + - InterleaveThinker Inference Workflow: design/interleave_thinker.md - Training Architecture: design/training_architecture.md - Server Contracts: - Overview: design/server_contracts/index.md