Will Lin
1fe50c0092
The v2 top level had grown to ~27 dirs + 13 loose files — half of them
single-concept abstraction shells (memory/ 86 LOC, transport/ 222, parity/ 178,
extend/ 226) and half vendored fastvideo code sitting as peers to the actual v2
design. Regroup to mirror README section 3 "Planes & dependency order":
core/ enums+types, card, loop, program, parity, request, parallel
(the model-native contracts; no kernels)
runtime/ + folded-in substrate: cache, memory, transport, extend
(the import graph shows only runtime consumes them; compile/cudagraph
already lived here)
serving/ + deploy/ (products / fleet)
_vendor/ all copied fastvideo: models, layers, attention, configs, distributed,
platforms, api, hooks, logging_utils, third_party + fastvideo_args/
utils/logger/envs/forward_context — internal layout unchanged, still
mirrors upstream for diffing
28 dirs + 13 root files -> 8 dirs + 6 root files. Pure mechanical move: 947
absolute-import paths rewritten (v2.X -> v2.{core,runtime,serving,_vendor}.X),
boundary-anchored so platform/ (native dispatch) and platforms/ (vendored CUDA
detect) no longer collide and neither does hooks/ vs extend/. Deleted the empty
v2/loader/. README section 3 + 16 updated; stale test count corrected
(34 files/216 tests -> 22 files/143 tests).
Validated: `import v2` stays torch-free; `v2/run_tests.py` and `pytest v2/tests/`
-> 143 passed, 2 skipped on the numpy toy backend. (On a GPU box force the toy
backend with CUDA_VISIBLE_DEVICES="" or detect() picks cuda.) No external importer
changed — examples use the re-exported `from v2 import VideoGenerator`.
95 lines
2.9 KiB
Python
95 lines
2.9 KiB
Python
"""ModalPart — typed multimodal request inputs.
|
|
|
|
Requests carry a list of typed modality parts instead of a god-batch with
|
|
per-model fields, avoiding a ``ForwardBatch`` blackboard / ``extra[...]`` pattern
|
|
at the input boundary; artifacts (outputs) do the same on the way out (see
|
|
``artifacts.py``).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
|
|
from v2.core.types import TensorLike
|
|
|
|
|
|
class Modality(str, Enum):
|
|
TEXT = "text"
|
|
IMAGE = "image"
|
|
VIDEO = "video"
|
|
AUDIO = "audio"
|
|
ACTION = "action"
|
|
LATENT = "latent"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ModalPart:
|
|
modality: Modality
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TextPart(ModalPart):
|
|
text: str = ""
|
|
|
|
def __init__(self, text: str = ""):
|
|
object.__setattr__(self, "modality", Modality.TEXT)
|
|
object.__setattr__(self, "text", text)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ImagePart(ModalPart):
|
|
pixels: TensorLike = None # HxWxC or CxHxW (backend decides), or a path
|
|
path: str | None = None
|
|
|
|
def __init__(self, pixels: TensorLike = None, path: str | None = None):
|
|
object.__setattr__(self, "modality", Modality.IMAGE)
|
|
object.__setattr__(self, "pixels", pixels)
|
|
object.__setattr__(self, "path", path)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VideoPart(ModalPart):
|
|
frames: TensorLike = None # T x H x W x C
|
|
fps: float = 16.0
|
|
path: str | None = None
|
|
|
|
def __init__(self, frames: TensorLike = None, fps: float = 16.0, path: str | None = None):
|
|
object.__setattr__(self, "modality", Modality.VIDEO)
|
|
object.__setattr__(self, "frames", frames)
|
|
object.__setattr__(self, "fps", fps)
|
|
object.__setattr__(self, "path", path)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AudioPart(ModalPart):
|
|
samples: TensorLike = None
|
|
sample_rate: int = 44100
|
|
|
|
def __init__(self, samples: TensorLike = None, sample_rate: int = 44100):
|
|
object.__setattr__(self, "modality", Modality.AUDIO)
|
|
object.__setattr__(self, "samples", samples)
|
|
object.__setattr__(self, "sample_rate", sample_rate)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ActionPart(ModalPart):
|
|
"""Camera / mouse / keyboard / pose conditioning for interactive world models."""
|
|
tensor: TensorLike = None
|
|
kind: str = "camera" # camera | mouse | keyboard | pose
|
|
|
|
def __init__(self, tensor: TensorLike = None, kind: str = "camera"):
|
|
object.__setattr__(self, "modality", Modality.ACTION)
|
|
object.__setattr__(self, "tensor", tensor)
|
|
object.__setattr__(self, "kind", kind)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LatentPart(ModalPart):
|
|
latent: TensorLike = None
|
|
of_modality: Modality = Modality.VIDEO
|
|
|
|
def __init__(self, latent: TensorLike = None, of_modality: Modality = Modality.VIDEO):
|
|
object.__setattr__(self, "modality", Modality.LATENT)
|
|
object.__setattr__(self, "latent", latent)
|
|
object.__setattr__(self, "of_modality", of_modality)
|